Merge pull request #644 from DataDog/fricounet/referrer-detect-shared-layer

[mountRemote] include intermediate snapshots in mount slice during referrer detection

.github/codecov.yml

@@ -0,0 +1,23 @@
+coverage:
+  status:
+    patch: off
+    project:
+      default:
+        enabled: yes
+        target: auto # auto compares coverage to the previous base commit
+        # adjust accordingly based on how flaky your tests are
+        # this allows a 0.3% drop from the previous base commit coverage
+        threshold: 0.3%
+
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: true # if true: only post the comment if coverage changes
+
+codecov:
+  require_ci_to_pass: false
+  notify:
+    wait_for_ci: true
+# When modifying this file, please validate using
+
+# curl -X POST --data-binary @codecov.yml https://codecov.io/validate

.github/workflows/ci.yml

@@ -2,116 +2,113 @@ name: CI
 
 on:
   push:
-    branches: ["*"]
+    branches: ["**", "stable/**"]
   pull_request:
-    branches: [main]
+    branches: ["**", "stable/**"]
+
+env:
+  CARGO_TERM_COLOR: always
 
 jobs:
   build:
-    name: Build
+    name: Build and Lint
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.17'
       - name: Check out code
-        uses: actions/checkout@v1
-      - name: cache go mod
-        uses: actions/cache@v2
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.59
+          skip-cache: true
       - name: Build
         run: |
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.43.0
-          export PATH=$PATH:$(go env GOPATH)/bin
           make
           make test
-          make check
-
+  build-optimizer:
+    name: Build optimizer
+    timeout-minutes: 10
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: Build
+        run: |
+          rustup component add rustfmt clippy
+          make build-optimizer
   smoke:
     name: Smoke
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.17'
       - name: Check out code
-        uses: actions/checkout@v1
-      - name: cache go mod
-        uses: actions/cache@v2
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: Set up containerd
+        uses: crazy-max/ghaction-setup-containerd@v3
       - name: Build
         run: |
           # Download nydus components
-          NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
+          NYDUS_VER=v$(curl -fsSL --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//')
+          wget -q https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
           tar xzvf nydus-static-$NYDUS_VER-linux-amd64.tgz
           mkdir -p /usr/bin
-          sudo mv nydus-static/nydus-image nydus-static/nydusd-fusedev nydus-static/nydusify /usr/bin/
-
+          sudo mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/
           export PATH=$PATH:$(go env GOPATH)/bin
           make smoke
 
   cross-build-test:
     name: Cross Build Test
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        GOOS: ['linux', 'windows', 'darwin']
-        GOARCH: ['amd64', 'arm64']
+        GOOS: ["linux", "windows", "darwin"]
+        GOARCH: ["amd64", "arm64"]
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.17'
       - name: Check out code
-        uses: actions/checkout@v1
-      - name: cache go mod
-        uses: actions/cache@v2
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
       - name: Build
         run: |
-          make -e GOOS=${{ matrix.GOOS }} GOARCH=${{ matrix.GOARCH }}
+          make -e GOOS=${{ matrix.GOOS }} GOARCH=${{ matrix.GOARCH }} converter
 
   coverage:
     name: Code coverage
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [build]
     steps:
-      - name: Install Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.17'
       - name: Checkout code
-        uses: actions/checkout@v2
-      - name: cache go mod
-        uses: actions/cache@v2
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
       - name: Run unit tests.
         run: make cover
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
           files: ./coverage.txt

.github/workflows/e2e.yml

@@ -0,0 +1,66 @@
+name: integration test
+
+on:
+  push:
+    branches:
+      - "main"
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+  pull_request:
+    branches:
+      - "main"
+  schedule:
+    # Trigger test every day at 00:03 clock UTC
+    - cron: "3 0 * * *"
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+  REGISTRY: ghcr.io
+
+jobs:
+  run-e2e-for-cgroups-v1:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: Log in to container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run e2e test
+        run: |
+          TAG=$GITHUB_REF_NAME
+          [ "$TAG" == "main" ] && TAG="latest"
+          [ "$GITHUB_EVENT_NAME" == "pull_request" ] && TAG="local"
+          make integration
+  
+  run-e2e-for-cgroups-v2:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: Log in to container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run e2e test
+        run: |
+          TAG=$GITHUB_REF_NAME
+          [ "$TAG" == "main" ] && TAG="latest"
+          [ "$GITHUB_EVENT_NAME" == "pull_request" ] && TAG="local"
+          make integration

.github/workflows/image.yml

@@ -1,173 +0,0 @@
-name: nydus-snapshotter image
-
-on:
-  push:
-    branches:
-      - 'main'
-    tags:
-      - v[0-9]+.[0-9]+.[0-9]+
-  pull_request:
-    branches:
-      - 'main'
-
-env:
-  CARGO_TERM_COLOR: always
-  REGISTRY: ghcr.io
-  ORGANIZATION: ${{ github.repository }}
-  CNI_VERSION: v1.1.0
-  CRICTL_VERSION: v1.23.0
-
-jobs:
-  build-image:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.17'
-      - name: Checkout repository
-        uses: actions/checkout@v2
-      - name: cache go mod
-        uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
-      - name: Build nydus-snapshotter
-        run: |
-          make static-release
-          cp bin/containerd-nydus-grpc misc/snapshotter
-      - name: Log in to container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and push nydus-snapshotter image
-        run: |
-          TAG=$GITHUB_REF_NAME
-          [ "$TAG" == "main" ] && TAG="latest"
-          [ "$GITHUB_EVENT_NAME" == "pull_request" ] && TAG="local"
-          cd misc/snapshotter
-          docker build -t ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/nydus-snapshotter:$TAG .
-          # Only push for non pull_request
-          if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then
-              docker push ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/nydus-snapshotter:$TAG
-          fi
-      - name: Prepare nydus-snapshotter image test
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          sudo mkdir -p /var/lib/containerd-test/ /etc/containerd/ /opt/cni/bin/ /etc/cni/net.d/
-          sudo cp misc/example/containerd-test-config.toml /etc/containerd/
-          sudo cp misc/example/crictl.yaml /etc/
-          sudo cp misc/example/10-containerd-net.conflist /etc/cni/net.d/
-          # install cni plugin
-          wget https://github.com/containernetworking/plugins/releases/download/${{ env.CNI_VERSION }}/cni-plugins-linux-amd64-${{ env.CNI_VERSION }}.tgz
-          sudo tar xzf cni-plugins-linux-amd64-${{ env.CNI_VERSION }}.tgz -C /opt/cni/bin/
-          # install crictl
-          wget https://github.com/kubernetes-sigs/cri-tools/releases/download/${{ env.CRICTL_VERSION }}/crictl-${{ env.CRICTL_VERSION }}-linux-amd64.tar.gz
-          tar xzf crictl-${{ env.CRICTL_VERSION }}-linux-amd64.tar.gz -C /usr/local/bin/
-          # install nydus-overlayfs
-          NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
-          tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz
-          sudo cp nydus-static/nydus-overlayfs /usr/local/sbin/
-          sudo cp nydus-static/ctr-remote /usr/local/sbin/
-          # install containerd
-          #CONTAINERD_VER=$(curl -s "https://api.github.com/repos/containerd/containerd/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          # Let's use fixed containerd version for now.
-          CONTAINERD_VER=1.6.1
-          wget https://github.com/containerd/containerd/releases/download/v$CONTAINERD_VER/containerd-$CONTAINERD_VER-linux-amd64.tar.gz
-          tar xzf containerd-$CONTAINERD_VER-linux-amd64.tar.gz
-          sudo cp bin/* /usr/local/bin/
-      - name: Test nydus-snapshotter image
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          # start nydus-snapshotter
-          docker run -d --name snapshotter --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined -e CONTAINERD_ROOT=/var/lib/containerd-test -v /var/lib/containerd-test:/var/lib/containerd-test:shared -v ~/.docker/config.json:/root/.docker/config.json:shared ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/nydus-snapshotter:local
-          # start containerd
-          sudo /usr/local/bin/containerd --config /etc/containerd/containerd-test-config.toml -l debug &
-          # wait for containerd to start up
-          sleep 10
-          echo "Pull nydus-latest image"
-          sudo crictl pull ghcr.io/dragonflyoss/image-service/ubuntu:nydus-latest
-          docker logs `docker ps |grep  "nydus-snapshotter"|cut -d ' ' -f1`
-          echo "create new pod with nydus snapshotter"
-          sudo crictl run misc/example/container.yaml misc/example/pod.yaml
-          container=$(sudo crictl ps -q)
-          echo "check container liveness"
-          sudo crictl exec $container ls
-          echo "delete pod"
-          sudo crictl rmp -fa
-      - name: Cleanup containerd and snapshotter
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          echo "stop snapshotter"
-          docker stop snapshotter
-          docker rm snapshotter
-          echo "stop containerd"
-          sudo pkill -f /usr/local/bin/containerd
-          echo "delete containerd dir"
-          # After the snapshotter container is stopped, it seems that Nydusd doesn't umount it
-          # so we need to umount it here, otherwise you cannot delete this directory. 
-          # Frankly, I don't know why Nydusd didn't clean up these resources.
-          sudo umount -f /var/lib/containerd-test/io.containerd.snapshotter.v1.nydus/mnt
-          sudo rm -rf /var/lib/containerd-test/*
-      - name: Test nydus-snapshotter image in localfs mode
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          # start nydus-snapshotter
-          docker run -d --name snapshotter --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined -e BACKEND_TYPE=localfs -e CONTAINERD_ROOT=/var/lib/containerd-test -v /var/lib/containerd-test:/var/lib/containerd-test:shared -v ~/.docker/config.json:/root/.docker/config.json:shared ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/nydus-snapshotter:local
-          docker_snapshotter=$(docker ps -q)
-          # start containerd
-          sudo /usr/local/bin/containerd --config /etc/containerd/containerd-test-config.toml -l debug &
-          # wait for containerd to start up
-          sleep 10
-          echo "Pull nydus-latest image"
-          sudo crictl pull ghcr.io/dragonflyoss/image-service/ubuntu:nydus-latest
-          docker logs `docker ps |grep  "nydus-snapshotter"|cut -d ' ' -f1`
-          echo "create new pod with nydus snapshotter"
-          sudo crictl run misc/example/container.yaml misc/example/pod.yaml
-          container=$(sudo crictl ps -q)
-          echo "check container liveness"
-          sudo crictl exec $container ls
-          echo "delete pod"
-          sudo crictl rmp -fa
-          echo "delete nydus-latest image"
-          sudo crictl rmi ghcr.io/dragonflyoss/image-service/ubuntu:nydus-latest
-          echo "check whether the blobs of nydus-latest image has been deleted"
-          docker exec -i $docker_snapshotter sh -c 'if [ "`ls -A /tmp/blobs/`" != "" ]; then exit 1; fi'
-      - name: Cleanup containerd and snapshotter
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          echo "stop snapshotter"
-          docker stop snapshotter
-          docker rm snapshotter
-          echo "stop containerd"
-          sudo pkill -f /usr/local/bin/containerd
-          echo "delete containerd dir"
-          # After the snapshotter container is stopped, it seems that Nydusd doesn't umount it
-          # so we need to umount it here, otherwise you cannot delete this directory. 
-          # Frankly, I don't know why Nydusd didn't clean up these resources.
-          sudo umount -f /var/lib/containerd-test/io.containerd.snapshotter.v1.nydus/mnt
-          sudo rm -rf /var/lib/containerd-test/*
-      - name: Test nydus-snapshotter with estargz image
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          # start nydus-snapshotter
-          docker run -d --name snapshotter --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined -e CONTAINERD_ROOT=/var/lib/containerd-test -v /var/lib/containerd-test:/var/lib/containerd-test:shared -v ~/.docker/config.json:/root/.docker/config.json:shared ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/nydus-snapshotter:local
-          # start containerd
-          sudo /usr/local/bin/containerd --config /etc/containerd/containerd-test-config.toml -l debug &
-          # wait for containerd to start up
-          sleep 10
-          echo "pull estargz image"
-          sudo crictl pull ghcr.io/stargz-containers/wordpress:5.9.2-esgz
-          docker logs `docker ps |grep  "nydus-snapshotter"|cut -d ' ' -f1`
-          echo "create new pod with nydus snapshotter"
-          sudo crictl run misc/example/container.yaml misc/example/pod.yaml
-          container=$(sudo crictl ps -q)
-          echo "check container liveness"
-          sudo crictl exec $container ls
-          echo "delete pod"
-          sudo crictl rmp -fa

.github/workflows/k8s-e2e-run.yml

@@ -0,0 +1,21 @@
+name: E2E Test With Kubernetes
+
+on:
+  push:
+    branches:
+      - "main"
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+  pull_request:
+    branches: [main]
+
+jobs:
+  cri_auth:
+    uses: ./.github/workflows/k8s-e2e.yml
+    with:
+      auth-type: cri
+
+  kubeconf_auth:
+    uses: ./.github/workflows/k8s-e2e.yml
+    with:
+      auth-type: kubeconf

.github/workflows/k8s-e2e.yml

@@ -0,0 +1,63 @@
+name: E2E Test With Kubernetes Template
+
+on:
+  workflow_call:
+    inputs:
+      auth-type:
+        required: true
+        type: string
+
+env:
+  DOCKER_USER: testuser
+  DOCKER_PASSWORD: testpassword
+  NAMESPACE: nydus-system
+
+jobs:
+  e2e_tests_k8s:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: Test
+        run: |
+          AUTH_TYPE='${{ inputs.auth-type }}'
+          ./tests/helpers/kind.sh
+      - name: Dump logs
+        if: failure()
+        continue-on-error: true
+        run: |
+          log_dir="/tmp/nydus-log"
+          mkdir -p $log_dir
+          for p in `kubectl --namespace "$NAMESPACE" get pods --no-headers -o custom-columns=NAME:metadata.name`; do
+           kubectl --namespace "$NAMESPACE" get pod $p -o yaml >> $log_dir/nydus-pods.conf
+           kubectl --namespace "$NAMESPACE" describe pod $p  >> $log_dir/nydus-pods.conf
+           kubectl --namespace "$NAMESPACE" logs $p -c nydus-snapshotter >> $log_dir/nydus-snapshotter.log || echo "failed to get snapshotter log"
+          done
+          kubectl --namespace "$NAMESPACE" get secrets -o yaml >> $log_dir/nydus-secrets.log
+
+          docker exec kind-control-plane cat /etc/containerd/config.toml >> $log_dir/containerd-config.toml
+          docker exec kind-control-plane containerd config dump >> $log_dir/containerd-config-dump.toml
+
+          docker exec kind-control-plane journalctl --no-pager -u containerd >> $log_dir/containerd.log
+          docker exec kind-control-plane journalctl --no-pager -u kubelet >> $log_dir/kubelet.log
+          docker exec kind-control-plane ps -ef >> $log_dir/psef.log
+
+          kubectl get pod test-pod -o yaml >> $log_dir/test-pod.log || echo "test-pod may be deleted or not created"
+
+          cat ~/.docker/config.json > $log_dir/docker.config.json || echo "~/.docker/config.json  not found"
+      - name: Upload Logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: k8s-e2e-tests-logs
+          path: |
+            /tmp/nydus-log
+          overwrite: true

.github/workflows/optimizer.yml

@@ -0,0 +1,109 @@
+name: optimizer test
+
+on:
+  push:
+    branches:
+      - "main"
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+  pull_request:
+    branches:
+      - "main"
+  schedule:
+    # Trigger test every day at 00:03 clock UTC
+    - cron: "3 0 * * *"
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  run_optimizer:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            tools/optimizer-server/target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('tools/optimizer-server/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo
+      - name: containerd runc and crictl
+        run: |
+          sudo wget -q https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.0/crictl-v1.26.0-linux-amd64.tar.gz
+          sudo tar zxvf ./crictl-v1.26.0-linux-amd64.tar.gz -C /usr/local/bin
+          sudo install -D -m 755 misc/optimizer/crictl.yaml /etc/crictl.yaml
+          sudo wget -q https://github.com/containerd/containerd/releases/download/v1.7.0/containerd-static-1.7.0-linux-amd64.tar.gz
+          sudo systemctl stop containerd
+          sudo tar -zxf ./containerd-static-1.7.0-linux-amd64.tar.gz -C /usr/
+          sudo install -D -m 755 misc/optimizer/containerd-config.toml /etc/containerd/config.toml
+          sudo systemctl restart containerd
+          sudo wget -q https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 -O /usr/bin/runc
+          sudo chmod +x /usr/bin/runc
+      - name: Setup CNI
+        run: |
+          wget -q https://github.com/containernetworking/plugins/releases/download/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+          sudo mkdir -p /opt/cni/bin
+          sudo tar xzf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/
+          sudo install -D -m 755 misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist
+      - name: Build and install optimizer
+        run: |
+          rustup component add rustfmt clippy
+          make optimizer
+          sudo chown -R $(id -un):$(id -gn) . ~/.cargo/
+          pwd
+          ls -lh bin/*optimizer*
+          sudo make install-optimizer
+          sudo install -D -m 755 misc/example/optimizer-nri-plugin.conf /etc/nri/conf.d/02-optimizer-nri-plugin.conf
+          sudo systemctl restart containerd
+          systemctl status containerd --no-pager -l
+      - name: Wait containerd ready
+        run: |
+          unset READY
+          for i in $(seq 30); do
+            if eval "timeout 180 ls /run/containerd/containerd.sock"; then
+                READY=true
+                break
+            fi
+            echo "Fail(${i}). Retrying..."
+            sleep 1
+          done
+          if [ "$READY" != "true" ];then
+            echo "containerd is not ready"
+            exit 1
+          fi
+      - name: Generate accessed files list
+        run: |
+          sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
+          sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
+          sleep 20
+          sudo crictl rmp  -f --all
+          tree /opt/nri/optimizer/results/
+          count=$(cat /opt/nri/optimizer/results/library/nginx:1.23.3 | wc -l)
+          expected=$(cat misc/optimizer/script/file_list.txt | wc -l)
+          echo "count: $count expected minimum value: $expected"
+          if [ $count -lt $expected ]; then
+            echo "failed to generate accessed files list for nginx:1.23.3"
+            cat misc/optimizer/script/file_list.txt
+            exit 1
+          fi
+          cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv
+      - name: Dump logs
+        if: failure()
+        continue-on-error: true
+        run: |
+          systemctl status containerd --no-pager -l
+          journalctl -xeu containerd --no-pager
+          

.github/workflows/release.yml

@@ -7,66 +7,180 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - build-os: linux
+            build-arch: amd64
+            build-linker: x86-64
+          - build-os: linux
+            build-arch: arm64
+            build-linker: aarch64
+          - build-os: linux
+            build-arch: s390x
+            build-linker: s390x
+          - build-os: linux
+            build-arch: ppc64le
+            build-linker: powerpc64le
+          - build-os: linux
+            build-arch: riscv64
+            build-linker: riscv64
+          - build-os: linux
+            build-arch: static
+            build-linker: static
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.17'
+          go-version: "1.22.5"
       - name: cache go mod
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+          key: ${{ matrix.build-os }}-go-${{ hashFiles('go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go
-      - name: build nydus-snapshotter
-        run: |
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.43.0
-          export PATH=$PATH:$(go env GOPATH)/bin
-          make
-      - name: upload artifacts
-        uses: actions/upload-artifact@v2
+            ${{ matrix.build-os }}-go
+      - name: cache cargo
+        uses: actions/cache@v4
         with:
-          name: nydus-snapshotter_artifacts
           path: |
-            bin/containerd-nydus-grpc
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            tools/optimizer-server/target/
+          key: ${{ matrix.build-os }}-cargo-${{ hashFiles('tools/optimizer-server/Cargo.lock') }}
+          restore-keys: |
+            ${{ matrix.build-os }}-cargo
+      - name: install gnu gcc linker
+        run: |
+          if [ "${{ matrix.build-linker }}" != "static" ]; then
+            sudo apt-get install -qq gcc-${{ matrix.build-linker }}-linux-gnu
+          fi
+      - name: build nydus-snapshotter and optimizer
+        run: |
+          go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2
+          export PATH=$PATH:$(go env GOPATH)/bin
+          if [ "${{ matrix.build-arch }}" == "static" ]; then
+            make static-package
+          else
+            make package GOOS=${{ matrix.build-os }} GOARCH=${{ matrix.build-arch }}
+          fi
+      - name: upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-tars-${{ matrix.build-os }}-${{ matrix.build-arch }}
+          path: |
+            package/*.tar.gz*
+          overwrite: true
   upload:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [build]
     steps:
-      - uses: actions/checkout@v2
-      - name: install hub
-        run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub'
-          sudo mv hub /usr/local/bin/hub
-      - name: download artifacts
-        uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
         with:
-          name: nydus-snapshotter_artifacts
-          path: nydus-snapshotter
-      - name: upload artifacts
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="nydus-snapshotter-$tag-x86_64.tgz"
-          chmod +x nydus-snapshotter/*
-          tar cf - nydus-snapshotter | gzip > ${tarball}
-          echo "tag=$tag" >> $GITHUB_ENV
-          echo "tarball=$tarball" >> $GITHUB_ENV
-
-          tarball_shasum="$tarball.sha256sum"
-          sha256sum $tarball > $tarball_shasum
-          echo "tarball_shasum=$tarball_shasum" >> $GITHUB_ENV
+          path: builds
       - name: Release
         uses: softprops/action-gh-release@v1
         with:
-          name: "Nydus Snapshotter ${{ env.tag }} Release"
+          name: "Nydus Snapshotter ${{ github.ref_name }} Release"
           generate_release_notes: true
-          files:
-            ${{ env.tarball }}
-            ${{ env.tarball_shasum }}
+          files: |
+            builds/release-tars-**/*
+
+  publish-image:
+    runs-on: ubuntu-22.04
+    needs: [build]
+    strategy:
+      matrix:
+        include:
+          - build-os: linux
+            build-arch: amd64
+          - build-os: linux
+            build-arch: arm64
+          - build-os: linux
+            build-arch: s390x
+          - build-os: linux
+            build-arch: ppc64le
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: release-tars-${{ matrix.build-os }}-${{ matrix.build-arch }}
+          path: misc/snapshotter
+      - name: unpack static release
+        run: |
+          cd misc/snapshotter && tar -zxf *.tar.gz && mv bin/* . && rm -rf bin
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Get the nydusd version
+        run: |
+          export NYDUS_STABLE_VER=$(curl -fsSL https://api.github.com/repos/dragonflyoss/nydus/releases/latest | jq -r .tag_name)
+          echo "NYDUS_STABLE_VER=$NYDUS_STABLE_VER" >> "$GITHUB_ENV"
+          printf 'nydus version is: %s\n' "$NYDUS_STABLE_VER"
+      - name: build and push nydus-snapshotter image
+        uses: docker/build-push-action@v5
+        with:
+          context: misc/snapshotter
+          file: misc/snapshotter/Dockerfile
+          push: true
+          platforms: ${{ matrix.build-os }}/${{ matrix.build-arch }}
+          provenance: false
+          tags: |
+            ${{ fromJSON(steps.meta.outputs.json).tags[0] }}-${{ matrix.build-arch }}
+            ${{ fromJSON(steps.meta.outputs.json).tags[1] }}-${{ matrix.build-arch }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: NYDUS_VER=${{ env.NYDUS_STABLE_VER }}
+
+  publish-manifest:
+    runs-on: ubuntu-22.04
+    needs: [publish-image]
+    steps:
+      - name: Log in to the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Publish manifest for multi-arch image
+        run: |
+          IFS=',' read -ra tags <<< "$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ',')"
+          for tag in "${tags[@]}"; do
+            docker manifest create "${tag}" \
+              --amend "${tag}-amd64" \
+              --amend "${tag}-arm64" \
+              --amend "${tag}-s390x" \
+              --amend "${tag}-ppc64le"
+            docker manifest push "${tag}"
+          done

.gitignore

@@ -2,3 +2,7 @@ bin/
 pkg/filesystem/stargz/testdata/db/
 coverage.txt
 .vscode/
+tests/output/
+smoke.tests
+tools/optimizer-server/target
+vendor/

.golangci.yml

@@ -1,23 +1,77 @@
 # https://golangci-lint.run/usage/configuration#config-file
 
+run:
+  concurrency: 4
+  timeout: 5m
+  issues-exit-code: 1
+  tests: true
+
+issues:
+  exclude-dirs:
+    - misc
+    # The package is ported from containerd project, let's skip it.
+    - pkg/remote/remotes
+
+linters-settings:
+  depguard:
+    rules:
+      main:
+        deny:
+          - pkg: "github.com/containerd/containerd/errdefs"
+            desc: The containerd errdefs package was migrated to a separate module. Use github.com/containerd/errdefs instead.
+          - pkg: "github.com/containerd/containerd/log"
+            desc: The containerd log package was migrated to a separate module. Use github.com/containerd/log instead.
+          - pkg: "github.com/containerd/containerd/platforms"
+            desc: The containerd platforms package was migrated to a separate module. Use github.com/containerd/platforms instead.
+          - pkg: "github.com/containerd/containerd/reference/docker"
+            desc: The containerd platforms package was migrated to a separate module. Use github.com/distribution/reference instead.
+  #   govet:
+  #     check-shadowing: true
+  #     enable:
+  #       - fieldalignment
+  funlen:
+    # Checks the number of lines in a function.
+    # If lower than 0, disable the check.
+    # Default: 60
+    lines: 100
+    # Checks the number of statements in a function.
+    # If lower than 0, disable the check.
+    # Default: 40
+    statements: 80
+  nilnil:
+    checked-types:
+      - ptr
+      - func
+      - iface
+      - map
+      - chan
+
 linters:
   enable:
-    - structcheck
-    - varcheck
+    - depguard      # Checks for imports that shouldn't be used.
     - staticcheck
     - unconvert
     - gofmt
     - goimports
     - revive
     - ineffassign
-    - vet
+    - govet
     - unused
     - misspell
-  disable:
+    - bodyclose
+    # - cyclop
+    - dogsled
+    - nilnil
+    - unparam
+    - nilerr
+    # - goerr113
+    - exportloopref
+    # - gosec
+    - gocritic
+    - prealloc
+    - tenv
+    # - funlen
+    - exhaustive
     - errcheck
-
-run:
-  deadline: 4m
-  skip-dirs:
-    - misc
-
+  disable:
+    - gosec

MAINTAINERS

@@ -7,7 +7,10 @@
 # GitHub ID, Name, Email address
 changweige, Changwei Ge, changweige@gmail.com
 eryugey, Eryu Guan, eguan@linux.alibaba.com
+imeoer, Yan Song, yansong.ys@antgroup.com
+
 #
 # REVIEWERS
 # GitHub ID, Name, Email address
-imeoer, Yan Song, imeoer@linux.alibaba.com
+sctb512, Bin Tang, tangbin.bin@bytedance.com
+

Makefile

@@ -1,15 +1,33 @@
-all: clear build
+all: clean build
+optimizer: clean-optimizer build-optimizer
 
-VERSION=$(shell git rev-parse --verify HEAD --short=7)
-BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
+PKG = github.com/containerd/nydus-snapshotter
 PACKAGES ?= $(shell go list ./... | grep -v /tests)
 SUDO = $(shell which sudo)
 GO_EXECUTABLE_PATH ?= $(shell which go)
 NYDUS_BUILDER ?= /usr/bin/nydus-image
-NYDUS_NYDUSD ?= /usr/bin/nydusd-fusedev
+NYDUS_NYDUSD ?= /usr/bin/nydusd
 GOOS ?= linux
-GOARCH ?= amd64
-#GOPROXY ?= https://goproxy.io
+GOARCH ?= $(shell go env GOARCH)
+KERNEL_VER = $(shell uname -r)
+
+# Used to populate variables in version package.
+BUILD_TIMESTAMP=$(shell date '+%Y-%m-%dT%H:%M:%S')
+VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
+REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
+
+
+RELEASE=nydus-snapshotter-v$(VERSION:v%=%)-${GOOS}-${GOARCH}
+STATIC_RELEASE=nydus-snapshotter-v$(VERSION:v%=%)-${GOOS}-static
+
+# Relpace test target images for e2e tests.
+ifdef E2E_TEST_TARGET_IMAGES_FILE
+ENV_TARGET_IMAGES_FILE = --env-file ${E2E_TEST_TARGET_IMAGES_FILE}
+endif
+
+ifdef E2E_DOWNLOADS_MIRROR
+BUILD_ARG_E2E_DOWNLOADS_MIRROR = --build-arg DOWNLOADS_MIRROR=${E2E_DOWNLOADS_MIRROR}
+endif
 
 ifdef GOPROXY
 PROXY := GOPROXY="${GOPROXY}"
@@ -21,25 +39,100 @@ else
 FS_DRIVER = fusedev
 endif
 
+SNAPSHOTTER_CONFIG=/etc/nydus/config.toml
+SOURCE_SNAPSHOTTER_CONFIG=misc/snapshotter/config.toml
+NYDUSD_CONFIG=/etc/nydus/nydusd-config.${FS_DRIVER}.json
+SOURCE_NYDUSD_CONFIG=misc/snapshotter/nydusd-config.${FS_DRIVER}.json
 
+SNAPSHOTTER_SYSTEMD_UNIT_SERVICE=misc/snapshotter/nydus-snapshotter.${FS_DRIVER}.service
+
+LDFLAGS = -s -w -X ${PKG}/version.Version=${VERSION} -X ${PKG}/version.Revision=$(REVISION) -X ${PKG}/version.BuildTimestamp=$(BUILD_TIMESTAMP)
+DEBUG_LDFLAGS = -X ${PKG}/version.Version=${VERSION} -X ${PKG}/version.Revision=$(REVISION) -X ${PKG}/version.BuildTimestamp=$(BUILD_TIMESTAMP)
+
+CARGO ?= $(shell which cargo)
+OPTIMIZER_SERVER = tools/optimizer-server
+OPTIMIZER_SERVER_TOML = ${OPTIMIZER_SERVER}/Cargo.toml
+OPTIMIZER_SERVER_BIN = ${OPTIMIZER_SERVER}/bin/optimizer-server
 .PHONY: build
 build:
-	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags="-s -w -X 'main.Version=${VERSION}'" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+
+.PHONY: static
+static:
+	CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+
+debug:
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(DEBUG_LDFLAGS)" -gcflags "-N -l" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(DEBUG_LDFLAGS)" -gcflags "-N -l" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+
+.PHONY: build-optimizer
+build-optimizer:
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/optimizer-nri-plugin ./cmd/optimizer-nri-plugin
+	make -C tools/optimizer-server release OS=$(GOOS) ARCH=$(GOARCH) && cp ${OPTIMIZER_SERVER_BIN} ./bin
 
 static-release:
-	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-s -w -X "main.Version=${VERSION}" -extldflags "-static"' -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/optimizer-nri-plugin ./cmd/optimizer-nri-plugin
+	make -C tools/optimizer-server static-release && cp ${OPTIMIZER_SERVER_BIN} ./bin
 
-.PHONY: clear
-clear:
+package/$(RELEASE).tar.gz: build build-optimizer
+	mkdir -p package
+	rm -rf package/$(RELEASE) package/$(RELEASE).tar.gz
+	tar -czf package/$(RELEASE).tar.gz bin
+	rm -rf package/$(RELEASE)
+
+package/$(STATIC_RELEASE).tar.gz: static-release
+	@mkdir -p package
+	@rm -rf package/$(STATIC_RELEASE) package/$(STATIC_RELEASE).tar.gz
+	@tar -czf package/$(STATIC_RELEASE).tar.gz bin
+	@rm -rf package/$(STATIC_RELEASE)
+
+package: package/$(RELEASE).tar.gz
+	cd package && sha256sum $(RELEASE).tar.gz >$(RELEASE).tar.gz.sha256sum
+
+static-package: package/$(STATIC_RELEASE).tar.gz
+	@cd package && sha256sum $(STATIC_RELEASE).tar.gz >$(STATIC_RELEASE).tar.gz.sha256sum
+
+# Majorly for cross build for converter package since it is imported by other projects
+converter:
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/converter ./cmd/converter
+
+.PHONY: clean
+clean:
 	rm -f bin/*
 	rm -rf _out
 
+.PHONY: clean-optimizer
+clean-optimizer:
+	rm -rf bin/optimizer-nri-plugin bin/optimizer-server
+	make -C tools/optimizer-server clean
+
 .PHONY: install
-install: static-release
-	sudo install -D -m 755 bin/containerd-nydus-grpc /usr/local/bin/containerd-nydus-grpc
-	sudo install -D -m 755 misc/snapshotter/nydusd-config.${FS_DRIVER}.json /etc/nydus/config.json
-	sudo install -D -m 644 misc/snapshotter/nydus-snapshotter.${FS_DRIVER}.service /etc/systemd/system/nydus-snapshotter.service
-	sudo systemctl enable /etc/systemd/system/nydus-snapshotter.service
+install:
+	@echo "+ $@ bin/containerd-nydus-grpc"
+	@sudo install -D -m 755 bin/containerd-nydus-grpc /usr/local/bin/containerd-nydus-grpc
+	@echo "+ $@ bin/nydus-overlayfs"
+	@sudo install -D -m 755 bin/nydus-overlayfs /usr/local/bin/nydus-overlayfs
+
+	@if [ ! -e ${NYDUSD_CONFIG} ]; then echo "+ $@ SOURCE_NYDUSD_CONFIG"; sudo install -D -m 664 ${SOURCE_NYDUSD_CONFIG} ${NYDUSD_CONFIG}; fi
+	@if [ ! -e ${SNAPSHOTTER_CONFIG} ]; then echo "+ $@ ${SOURCE_SNAPSHOTTER_CONFIG}"; sudo install -D -m 664 ${SOURCE_SNAPSHOTTER_CONFIG} ${SNAPSHOTTER_CONFIG}; fi
+	@sudo ln -f -s /etc/nydus/nydusd-config.${FS_DRIVER}.json /etc/nydus/nydusd-config.json
+
+	@echo "+ $@ ${SNAPSHOTTER_SYSTEMD_UNIT_SERVICE}"
+	@sudo install -D -m 644 ${SNAPSHOTTER_SYSTEMD_UNIT_SERVICE} /etc/systemd/system/nydus-snapshotter.service
+
+	@sudo mkdir -p /etc/nydus/certs.d
+	@if which systemctl >/dev/null; then sudo systemctl enable /etc/systemd/system/nydus-snapshotter.service; sudo systemctl restart nydus-snapshotter; fi
+
+install-optimizer:
+	sudo install -D -m 755 bin/optimizer-nri-plugin /opt/nri/plugins/02-optimizer-nri-plugin
+	sudo install -D -m 755 bin/optimizer-server /usr/local/bin/optimizer-server
+	sudo install -D -m 755 misc/example/optimizer-nri-plugin.conf /etc/nri/conf.d/02-optimizer-nri-plugin.conf
+
+	@sudo mkdir -p /opt/nri/optimizer/results
 
 .PHONY: vet
 vet:
@@ -58,6 +151,18 @@ cover:
 	go test -v -covermode=atomic -coverprofile=coverage.txt $(PACKAGES)
 	go tool cover -func=coverage.txt
 
+# make smoke TESTS=TestPack
 smoke:
-	$(SUDO) NYDUS_BUILDER=${NYDUS_BUILDER} NYDUS_NYDUSD=${NYDUS_NYDUSD} ${GO_EXECUTABLE_PATH} test -race -v ./tests
-	$(SUDO) NYDUS_BUILDER=${NYDUS_BUILDER} NYDUS_NYDUSD=${NYDUS_NYDUSD} ${GO_EXECUTABLE_PATH} test -race -v ./tests -args -fs-version=6
+	${GO_EXECUTABLE_PATH} test -o smoke.tests -c -race -v -cover ./tests
+	$(SUDO) -E NYDUS_BUILDER=${NYDUS_BUILDER} NYDUS_NYDUSD=${NYDUS_NYDUSD} ./smoke.tests -test.v -test.timeout 10m -test.parallel=8 -test.run=$(TESTS)
+
+.PHONY: integration
+integration:
+	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+	$(SUDO) docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile .
+	$(SUDO) docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \
+	-v `go env GOCACHE`:/root/.cache/go-build -v `pwd`:/nydus-snapshotter \
+	-v /usr/src/linux-headers-${KERNEL_VER}:/usr/src/linux-headers-${KERNEL_VER} \
+	${ENV_TARGET_IMAGES_FILE}  \
+	nydus-snapshotter-e2e:0.1

README.md

@@ -1,14 +1,37 @@
+[**[⬇️ Download]**](https://github.com/containerd/nydus-snapshotter/releases)
+[**[📖 Website]**](https://nydus.dev/)
+[**[☸ Quick Start (Kubernetes)**]](https://github.com/containerd/nydus-snapshotter/blob/main/docs/run_nydus_in_kubernetes.md)
+[**[🤓 Quick Start (nerdctl)**]](https://github.com/containerd/nerdctl/blob/master/docs/nydus.md)
+[**[❓ FAQs & Troubleshooting]**](https://github.com/dragonflyoss/nydus/wiki/FAQ)
+
 # Nydus Snapshotter
 
-<p><img src="https://github.com/dragonflyoss/image-service/blob/master/misc/logo.svg" width="170"></p>
+<p><img src="https://github.com/dragonflyoss/nydus/blob/master/misc/logo.svg" width="170"></p>
+
+[![Release Version](https://img.shields.io/github/v/release/containerd/nydus-snapshotter?style=flat)](https://github.com/containerd/nydus-snapshotter/releases)
+[![LICENSE](https://img.shields.io/github/license/containerd/nydus-snapshotter.svg?style=flat)](https://github.com/containerd/nydus-snapshotter/blob/main/LICENSE)
+![CI](https://github.com/containerd/nydus-snapshotter/actions/workflows/ci.yml/badge.svg?event=push)
+[![Go Report Card](https://goreportcard.com/badge/github.com/containerd/nydus-snapshotter?style=flat)](https://goreportcard.com/report/github.com/containerd/nydus-snapshotter)
+[![Twitter](https://img.shields.io/twitter/url?style=social&url=https%3A%2F%2Ftwitter.com%2Fdragonfly_oss)](https://twitter.com/dragonfly_oss)
+[![Nydus Stars](https://img.shields.io/github/stars/dragonflyoss/nydus?label=Nydus%20Stars&style=social)](https://github.com/dragonflyoss/nydus)
 
 Nydus-snapshotter is a **non-core** sub-project of containerd.
 
 Nydus snapshotter is an external plugin of containerd for [Nydus image service](https://nydus.dev) which implements a chunk-based content-addressable filesystem on top of a called `RAFS (Registry Acceleration File System)` format that improves the current OCI image specification, in terms of container launching speed, image space, and network bandwidth efficiency, as well as data integrity with several runtime backends: FUSE, virtiofs and in-kernel [EROFS](https://www.kernel.org/doc/html/latest/filesystems/erofs.html).
 
-Nydus supports lazy pulling feature since pulling image is one of the time-consuming steps in the container lifecycle. Lazy pulling here means a container can run even the image is partially available and necessary chunks of the image are fetched on-demand. Apart from that, Nydus also supports [(e)Stargz](https://github.com/containerd/stargz-snapshotter) lazy pulling directly **WITHOUT** any explicit conversion.
+Nydus supports lazy pulling feature since pulling image is one of the time-consuming steps in the container lifecycle. Lazy pulling here means a container can run even the image is partially available and necessary chunks of the image are fetched on-demand. Apart from that, Nydus also supports [(e)Stargz](https://github.com/containerd/stargz-snapshotter) and OCI (by using [zran](https://github.com/dragonflyoss/nydus/blob/master/docs/nydus-zran.md)) lazy pulling directly **WITHOUT** any explicit conversion.
 
-For more details about how to build Nydus container image, please refer to [nydusify](https://github.com/dragonflyoss/image-service/blob/master/docs/nydusify.md) conversion tool and [acceld](https://github.com/goharbor/acceleration-service).
+For more details about how to build Nydus container image, please refer to [nydusify](https://github.com/dragonflyoss/nydus/blob/master/docs/nydusify.md) conversion tool and [acceld](https://github.com/goharbor/acceleration-service).
+
+## Architecture
+
+### Architecture Based on FUSE
+
+![fuse arch](./docs/diagram/nydus_fuse_arch.svg)
+
+### Architecture Based on Fscache/Erofs
+
+![fscache arch](./docs/diagram/nydus_fscache_erofs_arch.svg)
 
 ## Building
 
@@ -20,6 +43,8 @@ make
 
 ## Integrate Nydus-snapshotter into Containerd
 
+The following document will describe how to manually configure containerd + Nydus snapshotter. If you want to run Nydus snapshotter in Kubernetes cluster, you can try to use helm or run nydus snapshotter as a container. You can refer to [this documentation](./docs/run_nydus_in_kubernetes.md).
+
 Containerd provides a general mechanism to exploit different types of snapshotters. Please ensure your containerd's version is 1.4.0 or above.
 Add Nydus as a proxy plugin into containerd's configuration file which may be located at `/etc/containerd/config.toml`.
 
@@ -37,48 +62,39 @@ Restart your containerd service making the change take effect. Assume that your
 systemctl restart containerd
 ```
 
-## Get Nydusd Binary
+### Get Nydus Binaries
 
-Find a suitable `nydusd` release for you from [nydus releases page](https://github.com/dragonflyoss/image-service/releases).
+Get `nydusd` `nydus-image` and `nydusctl` binaries from [nydus releases page](https://github.com/dragonflyoss/nydus/releases).
+It's suggested to install the binaries to your system path. `nydusd` is FUSE userspace daemon and a vhost-user-fs backend. Nydus-snapshotter
+will fork a nydusd process when necessary.
 
-`nydusd-fusedev` is FUSE userspace daemon handling linux kernel FUSE requests from `/dev/fuse` frontend.
-`nydusd-virtiofs` is a virtiofs daemon handling guest kernel FUSE requests.
+### Configure Nydus
 
-## Configure Nydus
+Please follow instructions to [configure nydus](./docs/configure_nydus.md) in order to make it work properly in your environment.
 
-Nydus is configured by a json file which is required now. Since Nydus container images are likely stored in a registry, where auth has to be provided.
-Please follow instructions to [configure nydus](./docs/configure_nydus.md) configure Nydus in order to make it work properly in your environment.
-
-## Start Nydus Snapshotter
+### Start Nydus Snapshotter
 
 Nydus-snapshotter is implemented as a [proxy plugin](https://github.com/containerd/containerd/blob/04985039cede6aafbb7dfb3206c9c4d04e2f924d/PLUGINS.md#proxy-plugins) (`containerd-nydus-grpc`) for containerd.
 
-Assume your server systemd based, install nydus-snapshotter:
+Assume your server is systemd based, install nydus-snapshotter:
 Note: `nydusd` and `nydus-image` should be found from $PATH.
+
 ```bash
 make install
 systemctl restart containerd
 ```
 
 Or you can start nydus-snapshotter manually.
-```bash
-# `nydusd-path` is the path to nydusd binary
-# `address` is the domain socket that you configured in containerd configuration file
-# `root` is the path to Nydus snapshotter
-# `config-path` is the path to Nydus configuration file
-$ ./containerd-nydus-grpc \
-    --config-path /etc/nydusd-config.json \
-    --shared-daemon \
-    --log-level info \
-    --root /var/lib/containerd/io.containerd.snapshotter.v1.nydus \
-    --cache-dir /var/lib/nydus/cache \
-    --address /run/containerd-nydus/containerd-nydus-grpc.sock \
-    --nydusd-path /usr/local/bin/nydusd \
-    --nydusimg-path /usr/local/bin/nydus-image \
-    --log-to-stdout
-```
 
-You could append `--enable-stargz` to the command line above in order to enable (e)Stargz support.
+```bash
+# `--nydusd` specifies the path to nydusd binary. If `nydusd` and `nydus-image` are installed, `--nydusd` and `--nydus-image`can be omitted.
+# Otherwise, provide them in below command line.
+# `address` is the domain socket that you configured in containerd configuration file
+# `--nydusd-config` is the path to `nydusd` configuration file
+# The default nydus-snapshotter work directory is located at `/var/lib/containerd/io.containerd.snapshotter.v1.nydus`
+
+$ sudo ./containerd-nydus-grpc --config /etc/nydus/config.toml --nydusd-config /etc/nydus/nydusd-config.json --log-to-stdout
+```
 
 ### Validate Nydus-snapshotter Setup
 
@@ -90,37 +106,25 @@ TYPE                            ID                       PLATFORMS      STATUS
 io.containerd.snapshotter.v1    nydus                    -              ok
 ```
 
-## Quickly Start Container with Lazy Pulling
+### Optimize Nydus Image as per Workload
 
-### Start Container on Node
+Nydus usually prefetch image data to local filesystem before a real user on-demand read. It helps to improve the performance and availability. A containerd NRI plugin [container image optimizer](docs/optimize_nydus_image.md) can be used to generate nydus image building suggestions to optimize your nydus image making the nydusd runtime match your workload IO pattern. The optimized nydus image has
+a better performance.
 
-Containerd can start container with specified snapshotter, so `nerdctl` or `ctr` needs to specify the Nydus snapshotter when start container.
+## Quickstart Container with Lazy Pulling
 
-A CLI tool [ctr-remote](https://github.com/dragonflyoss/image-service/tree/master/contrib/ctr-remote) is alongside. Use Nydus `ctr-remote` to pull Nydus image or start container based on nydus image.
+### Start Container on single Node
+
+Start container using `nerdctl` (>=v0.22) which has native nydus support with `nydus-snapshotter`.
 
 ```bash
-$ sudo ctr-remote image rpull ghcr.io/dragonflyoss/image-service/nginx:nydus-latest
-fetching sha256:75002dfe... application/vnd.oci.image.manifest.v1+json
-fetching sha256:5a42e21c... application/vnd.oci.image.config.v1+json
-fetching sha256:eb1af2e1... application/vnd.oci.image.layer.v1.tar+gzip
-
-# Start container by `ctr-remote`
-$ sudo ctr-remote run --snapshotter nydus ghcr.io/dragonflyoss/image-service/nginx:nydus-latest awesome-nydus
-
 # Start container by `nerdctl`
 nerdctl --snapshotter nydus run ghcr.io/dragonflyoss/image-service/nginx:nydus-latest
 ```
 
-In addition that, `nerdctl` can now directly pull Nydus or (e)Stargz images with Nydus snapshotter without `ctr-remote` involved:
+### Start Container in Kubernetes Cluster
 
-```bash
-# Start an eStargz container with Nydus snapshotter by `nerdctl`
-nerdctl --snapshotter nydus run -it --rm ghcr.io/stargz-containers/fedora:35-esgz
-```
-
-### Start Container in Kubernetes
-
-**NOTE:** A potential drawback using CRI is that we can hardly specify snapshotter to `nydus-snapshotter`. So we have to change containerd's default snapshotter in its configuration file and enable snapshot annotations like below:
+Change containerd's CRI configuration:
 
 ```toml
 [plugins."io.containerd.grpc.v1.cri".containerd]
@@ -131,11 +135,71 @@ nerdctl --snapshotter nydus run -it --rm ghcr.io/stargz-containers/fedora:35-esg
 Use `crictl` to debug starting container via Kubernetes CRI. Dry run [steps](./docs/crictl_dry_run.md) of using `crictl` can be found in [documents](./docs).
 
 ### Setup with nydus-snapshotter image
-We can also use the `nydus-snapshotter` container image when we want to put Nydus stuffs inside a container. See the [nydus-snapshotter exmple](./misc/example/README.md) for how to setup and use it.
+
+We can also use the `nydus-snapshotter` container image when we want to put Nydus stuffs inside a container. See the [nydus-snapshotter example](./misc/example/README.md) for how to setup and use it.
+
+## Integrate with Dragonfly to Distribute Images by P2P
+
+Nydus is a sub-project of [Dragonfly](https://github.com/dragonflyoss/dragonfly). So it closely works with Dragonfly to distribute container images in a fast and efficient P2P fashion to reduce network latency and lower the pressure on a single-point of the registry.
+
+### Quickstart Dragonfly & Nydus in Kubernetes
+
+We recommend using the Dragonfly P2P data distribution system to further improve the runtime performance of Nydus images.
+
+If you want to deploy Dragonfly and Nydus at the same time, please refer to this **[Quick Start](https://github.com/dragonflyoss/helm-charts/blob/main/INSTALL.md)**.
+
+### Config Dragonfly mode
+
+Dragonfly supports both **mirror** mode and HTTP **proxy** mode to boost the containers startup. It is suggested to use Dragonfly mirror mode. To integrate with Dragonfly in the mirror mode, please provide registry mirror in nydusd's json configuration file in section `device.backend.mirrors`
+
+```json
+{
+  "mirrors": [
+    {
+      "host": "http://127.0.0.1:65001",
+      "headers": "https://index.docker.io/v1/"
+    }
+  ]
+}
+```
+
+### Hot updating mirror configurations
+
+In addition to setting the registry mirror in nydusd's json configuration file, `nydus-snapshotter` also supports hot updating mirror configurations. You can set the configuration directory in nudus-snapshotter's toml configuration file with `remote.mirrors_config.dir`. The empty `remote.mirrors_config.dir` means disabling it.
+
+```toml
+[remote.mirrors_config]
+dir = "/etc/nydus/certs.d"
+```
+
+Configuration file is compatible with containerd's configuration file in toml format.
+
+```toml
+[host]
+  [host."http://127.0.0.1:65001"]
+    [host."http://127.0.0.1:65001".header]
+      # NOTE: For Dragonfly, the HTTP scheme must be explicitly specified.
+      X-Dragonfly-Registry = ["https://p2p-nydus.com"]
+```
+
+Mirror configurations loaded from nydusd's json file will be overwritten before pulling image if the valid mirror configuration items loaded from `remote.mirrors_config.dir` are greater than 0.
 
 ## Community
 
 Nydus aims to form a **vendor-neutral opensource** image distribution solution to all communities.
 Questions, bug reports, technical discussion, feature requests and contribution are always welcomed!
 
-Join our Slack [workspace](https://join.slack.com/t/nydusimageservice/shared_invite/zt-pz4qvl4y-WIh4itPNILGhPS8JqdFm_w)
+We're very pleased to hear your use cases any time.
+Feel free to reach/join us via Slack and/or Dingtalk.
+
+- **Slack:** [Nydus Workspace](https://join.slack.com/t/nydusimageservice/shared_invite/zt-pz4qvl4y-WIh4itPNILGhPS8JqdFm_w)
+- **Twitter:** [@dragonfly_oss](https://twitter.com/dragonfly_oss)
+- **Dingtalk:** [34971767](https://qr.dingtalk.com/action/joingroup?code=v1,k1,ioWGzuDZEIO10Bf+/ohz4RcQqAkW0MtOwoG1nbbMxQg=&_dt_no_comment=1&origin=11)
+
+<img src="https://github.com/dragonflyoss/nydus/blob/master/misc/dingtalk.jpg" width="250" height="300"/>
+
+- **Technical Meeting:** Every Wednesday at 06:00 UTC (Beijing, Shanghai 14:00), please see our [HackMD](https://hackmd.io/@Nydus/Bk8u2X0p9) page for more information.
+
+## License
+
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcontainerd%2Fnydus-snapshotter.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcontainerd%2Fnydus-snapshotter?ref=badge_large)

cmd/containerd-nydus-grpc/app/snapshotter/main.go

@@ -1,34 +0,0 @@
-/*
- * Copyright (c) 2020. Ant Group. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package snapshotter
-
-import (
-	"context"
-
-	"github.com/pkg/errors"
-
-	"github.com/containerd/nydus-snapshotter/config"
-	"github.com/containerd/nydus-snapshotter/pkg/auth"
-	"github.com/containerd/nydus-snapshotter/pkg/utils/signals"
-	"github.com/containerd/nydus-snapshotter/snapshot"
-)
-
-func Start(ctx context.Context, cfg config.Config) error {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	rs, err := snapshot.NewSnapshotter(ctx, &cfg)
-	if err != nil {
-		return errors.Wrap(err, "failed to initialize snapshotter")
-	}
-
-	stopSignal := signals.SetupSignalHandler()
-	opt := ServeOptions{
-		ListeningSocketPath: cfg.Address,
-	}
-	auth.InitKubeSecretListener(ctx, cfg.KubeconfigPath)
-	return Serve(ctx, rs, opt, stopSignal)
-}

cmd/containerd-nydus-grpc/app/snapshotter/serve.go

@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2020. Ant Group. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package snapshotter
-
-import (
-	"context"
-	"net"
-	"os"
-	"path/filepath"
-
-	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
-	"github.com/containerd/containerd/contrib/snapshotservice"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/snapshots"
-	"github.com/pkg/errors"
-	"google.golang.org/grpc"
-)
-
-type ServeOptions struct {
-	ListeningSocketPath string
-}
-
-func Serve(ctx context.Context, rs snapshots.Snapshotter, options ServeOptions, stop <-chan struct{}) error {
-	err := ensureSocketNotExists(options.ListeningSocketPath)
-	if err != nil {
-		return err
-	}
-	rpc := grpc.NewServer()
-	snapshotsapi.RegisterSnapshotsServer(rpc, snapshotservice.FromSnapshotter(rs))
-	l, err := net.Listen("unix", options.ListeningSocketPath)
-	if err != nil {
-		return errors.Wrapf(err, "error on listen socket %q", options.ListeningSocketPath)
-	}
-	go func() {
-		sig := <-stop
-		log.G(ctx).Infof("caught signal %s: shutting down", sig)
-		err := l.Close()
-		if err != nil {
-			log.G(ctx).Errorf("failed to close listener %s, err: %v", options.ListeningSocketPath, err)
-		}
-	}()
-	return rpc.Serve(l)
-}
-
-func ensureSocketNotExists(listeningSocketPath string) error {
-	if err := os.MkdirAll(filepath.Dir(listeningSocketPath), 0700); err != nil {
-		return errors.Wrapf(err, "failed to create directory %q", filepath.Dir(listeningSocketPath))
-	}
-	finfo, err := os.Stat(listeningSocketPath)
-	// err is nil means listening socket path exists, remove before serve
-	if err == nil {
-		if finfo.Mode()&os.ModeSocket == 0 {
-			return errors.Errorf("file %s is not a socket", listeningSocketPath)
-		}
-		err := os.Remove(listeningSocketPath)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

cmd/containerd-nydus-grpc/app/snapshotter/testdata/happypath/config.json

@@ -1,29 +0,0 @@
-{
-  "device": {
-    "backend": {
-      "type": "registry",
-      "config": {
-        "scheme": "https",
-        "auth": "",
-        "timeout": 5,
-        "connect_timeout": 5,
-        "retry_limit": 0
-      }
-    },
-    "cache": {
-      "type": "blobcache",
-      "config": {
-        "work_dir": "/tmp/nydus-rs/cache"
-      }
-    }
-  },
-  "mode": "direct",
-  "digest_validate": true,
-  "iostats_files": true,
-  "enable_xattr": true,
-  "fs_prefetch": {
-    "enable": true,
-    "threads_count": 10,
-    "merging_size": 131072
-  }
-}

cmd/containerd-nydus-grpc/main.go

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2020. Ant Group. All rights reserved.
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -7,45 +8,103 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
-	"github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/app/snapshotter"
-	"github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/command"
-	"github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/logging"
 	"github.com/containerd/nydus-snapshotter/config"
+	"github.com/containerd/nydus-snapshotter/internal/flags"
+	"github.com/containerd/nydus-snapshotter/internal/logging"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/version"
 )
 
 func main() {
-	flags := command.NewFlags()
+	flags := flags.NewFlags()
 	app := &cli.App{
-		Name:    "containerd-nydus-grpc",
-		Usage:   "Nydus remote snapshotter for containerd",
-		Version: Version,
-		Flags:   flags.F,
-		Action: func(c *cli.Context) error {
-			var cfg config.Config
-			if err := command.Validate(flags.Args, &cfg); err != nil {
-				return errors.Wrap(err, "invalid argument")
+		Name:        "containerd-nydus-grpc",
+		Usage:       "Nydus remote snapshotter for containerd",
+		Version:     version.Version,
+		Flags:       flags.F,
+		HideVersion: true,
+		Action: func(_ *cli.Context) error {
+			if flags.Args.PrintVersion {
+				fmt.Println("Version:    ", version.Version)
+				fmt.Println("Revision:   ", version.Revision)
+				fmt.Println("Go version: ", version.GoVersion)
+				fmt.Println("Build time: ", version.BuildTimestamp)
+				return nil
+			}
+
+			snapshotterConfigPath := flags.Args.SnapshotterConfigPath
+			var defaultSnapshotterConfig config.SnapshotterConfig
+			var snapshotterConfig config.SnapshotterConfig
+
+			if err := defaultSnapshotterConfig.FillUpWithDefaults(); err != nil {
+				return errors.New("failed to generate nydus default configuration")
+			}
+
+			// Once snapshotter's configuration file is provided, parse it and let command line parameters override it.
+			if snapshotterConfigPath != "" {
+				if c, err := config.LoadSnapshotterConfig(snapshotterConfigPath); err == nil {
+					// Command line parameters override the snapshotter's configurations for backwards compatibility
+					if err := config.ParseParameters(flags.Args, c); err != nil {
+						return errors.Wrap(err, "failed to parse commandline options")
+					}
+					snapshotterConfig = *c
+				} else {
+					return errors.Wrapf(err, "failed to load snapshotter configuration from %q", snapshotterConfigPath)
+				}
+			} else {
+				if err := config.ParseParameters(flags.Args, &snapshotterConfig); err != nil {
+					return errors.Wrap(err, "failed to parse commandline options")
+				}
+			}
+
+			if err := config.MergeConfig(&snapshotterConfig, &defaultSnapshotterConfig); err != nil {
+				return errors.Wrap(err, "failed to merge configurations")
+			}
+
+			if err := config.ValidateConfig(&snapshotterConfig); err != nil {
+				return errors.Wrapf(err, "failed to validate configurations")
+			}
+
+			if err := config.ProcessConfigurations(&snapshotterConfig); err != nil {
+				return errors.Wrap(err, "failed to process configurations")
+			}
+
+			if err := config.SetUpEnvironment(&snapshotterConfig); err != nil {
+				return errors.Wrap(err, "failed to setup environment")
 			}
 
 			ctx := logging.WithContext()
-			if err := logging.SetUp(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir, flags.Args.RootDir); err != nil {
-				return errors.Wrap(err, "failed to prepare logger")
+			logConfig := &snapshotterConfig.LoggingConfig
+			logRotateArgs := &logging.RotateLogArgs{
+				RotateLogMaxSize:    logConfig.RotateLogMaxSize,
+				RotateLogMaxBackups: logConfig.RotateLogMaxBackups,
+				RotateLogMaxAge:     logConfig.RotateLogMaxAge,
+				RotateLogLocalTime:  logConfig.RotateLogLocalTime,
+				RotateLogCompress:   logConfig.RotateLogCompress,
 			}
 
-			return snapshotter.Start(ctx, cfg)
+			if err := logging.SetUp(logConfig.LogLevel, logConfig.LogToStdout, logConfig.LogDir, logRotateArgs); err != nil {
+				return errors.Wrap(err, "failed to setup logger")
+			}
+
+			log.L.Infof("Start nydus-snapshotter. Version: %s, PID: %d, FsDriver: %s, DaemonMode: %s",
+				version.Version, os.Getpid(), config.GetFsDriver(), snapshotterConfig.DaemonMode)
+
+			return Start(ctx, &snapshotterConfig)
 		},
 	}
 	if err := app.Run(os.Args); err != nil {
 		if errdefs.IsConnectionClosed(err) {
-			log.L.Info("nydus snapshotter exited")
+			log.L.Info("nydus-snapshotter exited")
 		} else {
-			log.L.WithError(err).Fatal("failed to start nydus snapshotter:\n\r")
+			log.L.WithError(err).Fatal("failed to start nydus-snapshotter")
 		}
 	}
 }

cmd/containerd-nydus-grpc/pkg/command/flags.go

@@ -1,288 +0,0 @@
-/*
- * Copyright (c) 2020. Ant Group. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package command
-
-import (
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/logging"
-	"github.com/containerd/nydus-snapshotter/config"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-const (
-	defaultAddress   = "/run/containerd-nydus/containerd-nydus-grpc.sock"
-	defaultLogLevel  = logrus.InfoLevel
-	defaultRootDir   = "/var/lib/containerd-nydus-grpc"
-	defaultGCPeriod  = "24h"
-	defaultPublicKey = "/signing/nydus-image-signing-public.key"
-)
-
-type Args struct {
-	Address              string
-	LogLevel             string
-	LogDir               string
-	ConfigPath           string
-	RootDir              string
-	CacheDir             string
-	GCPeriod             string
-	ValidateSignature    bool
-	PublicKeyFile        string
-	ConvertVpcRegistry   bool
-	NydusdBinaryPath     string
-	NydusImageBinaryPath string
-	SharedDaemon         bool
-	DaemonMode           string
-	FsDriver             string
-	SyncRemove           bool
-	EnableMetrics        bool
-	MetricsFile          string
-	EnableStargz         bool
-	DisableCacheManager  bool
-	LogToStdout          bool
-	EnableNydusOverlayFS bool
-	NydusdThreadNum      int
-	CleanupOnClose       bool
-	KubeconfigPath       string
-}
-
-type Flags struct {
-	Args *Args
-	F    []cli.Flag
-}
-
-func buildFlags(args *Args) []cli.Flag {
-	return []cli.Flag{
-		&cli.StringFlag{
-			Name:        "address",
-			Value:       defaultAddress,
-			Usage:       "set `PATH` for gRPC socket",
-			Destination: &args.Address,
-		},
-		&cli.StringFlag{
-			Name:        "cache-dir",
-			Value:       "",
-			Aliases:     []string{"C"},
-			Usage:       "set `DIRECTORY` to store/cache downloaded image data",
-			Destination: &args.CacheDir,
-		},
-		&cli.BoolFlag{
-			Name:        "cleanup-on-close",
-			Value:       false,
-			Usage:       "whether to clean up on exit",
-			Destination: &args.CleanupOnClose,
-		},
-		&cli.StringFlag{
-			Name:        "config-path",
-			Required:    true,
-			Aliases:     []string{"c", "config"},
-			Usage:       "path to the configuration `FILE`",
-			Destination: &args.ConfigPath,
-		},
-		&cli.BoolFlag{
-			Name:        "convert-vpc-registry",
-			Usage:       "whether to automatically convert the image to vpc registry to accelerate image pulling",
-			Destination: &args.ConvertVpcRegistry,
-		},
-		&cli.StringFlag{
-			Name:        "daemon-mode",
-			Value:       config.DefaultDaemonMode,
-			Aliases:     []string{"M"},
-			Usage:       "set daemon working `MODE`, one of \"multiple\", \"shared\" or \"none\"",
-			Destination: &args.DaemonMode,
-		},
-		&cli.BoolFlag{
-			Name:        "disable-cache-manager",
-			Usage:       "whether to disable blob cache manager",
-			Destination: &args.DisableCacheManager,
-		},
-		&cli.BoolFlag{
-			Name:        "enable-metrics",
-			Usage:       "whether to collect metrics",
-			Destination: &args.EnableMetrics,
-		},
-		&cli.BoolFlag{
-			Name:        "enable-nydus-overlayfs",
-			Usage:       "whether to enable nydus-overlayfs",
-			Destination: &args.EnableNydusOverlayFS,
-		},
-		&cli.BoolFlag{
-			Name:        "enable-stargz",
-			Usage:       "whether to enable support of estargz image (experimental)",
-			Destination: &args.EnableStargz,
-		},
-		&cli.StringFlag{
-			Name:        "fs-driver",
-			Value:       config.FsDriverFusedev,
-			Aliases:     []string{"daemon-backend"},
-			Usage:       "backend `DRIVER` to serve the filesystem, one of \"fusedev\", \"fscache\"",
-			Destination: &args.FsDriver,
-		},
-		&cli.StringFlag{
-			Name:        "gc-period",
-			Value:       defaultGCPeriod,
-			Usage:       "blob cache garbage collection `INTERVAL`, duration string(for example, 1m, 2h)",
-			Destination: &args.GCPeriod,
-		},
-		&cli.StringFlag{
-			Name:        "log-dir",
-			Value:       "",
-			Aliases:     []string{"L"},
-			Usage:       "set `DIRECTORY` to store log files",
-			Destination: &args.LogDir,
-		},
-		&cli.StringFlag{
-			Name:        "log-level",
-			Value:       defaultLogLevel.String(),
-			Aliases:     []string{"l"},
-			Usage:       "set the logging `LEVEL` [trace, debug, info, warn, error, fatal, panic]",
-			Destination: &args.LogLevel,
-		},
-		&cli.BoolFlag{
-			Name:        "log-to-stdout",
-			Usage:       "log messages to standard out rather than files.",
-			Destination: &args.LogToStdout,
-		},
-		&cli.StringFlag{
-			Name:        "metrics-file",
-			Usage:       "path to the metrics output `FILE`",
-			Destination: &args.MetricsFile,
-		},
-		&cli.StringFlag{
-			Name:        "nydus-image",
-			Value:       "",
-			Aliases:     []string{"nydusimg-path"},
-			Usage:       "set `PATH` to the nydus-image binary, default to lookup nydus-image in $PATH",
-			Destination: &args.NydusImageBinaryPath,
-		},
-		&cli.StringFlag{
-			Name:        "nydusd",
-			Value:       "",
-			Aliases:     []string{"nydusd-path"},
-			Usage:       "set `PATH` to the nydusd binary, default to lookup nydusd in $PATH",
-			Destination: &args.NydusdBinaryPath,
-		},
-		&cli.IntFlag{
-			Name:        "nydusd-thread-num",
-			Usage:       "set worker thread number for nydusd, default to the number of CPUs",
-			Destination: &args.NydusdThreadNum,
-		},
-		&cli.StringFlag{
-			Name:        "publickey-file",
-			Value:       defaultPublicKey,
-			Usage:       "path to the publickey `FILE` for signature validation",
-			Destination: &args.PublicKeyFile,
-		},
-		&cli.StringFlag{
-			Name:        "root",
-			Value:       defaultRootDir,
-			Aliases:     []string{"R"},
-			Usage:       "set `DIRECTORY` to store snapshotter working state",
-			Destination: &args.RootDir,
-		},
-		&cli.BoolFlag{
-			Name:        "shared-daemon",
-			Usage:       "Deprecated, equivalent to \"--daemon-mode shared\"",
-			Destination: &args.SharedDaemon,
-		},
-		&cli.BoolFlag{
-			Name:        "sync-remove",
-			Usage:       "whether to clean up snapshots in synchronous mode, default to asynchronous mode",
-			Destination: &args.SyncRemove,
-		},
-		&cli.BoolFlag{
-			Name:        "validate-signature",
-			Usage:       "whether to validate integrity of image bootstrap",
-			Destination: &args.ValidateSignature,
-		},
-		&cli.StringFlag{
-			Name:        "kubeconfig-path",
-			Value:       "",
-			Usage:       "path to the kubeconfig file",
-			Destination: &args.KubeconfigPath,
-		},
-	}
-}
-
-func NewFlags() *Flags {
-	var args Args
-	return &Flags{
-		Args: &args,
-		F:    buildFlags(&args),
-	}
-}
-
-func Validate(args *Args, cfg *config.Config) error {
-	var daemonCfg config.DaemonConfig
-	if err := config.LoadConfig(args.ConfigPath, &daemonCfg); err != nil {
-		return errors.Wrapf(err, "failed to load config file %q", args.ConfigPath)
-	}
-	cfg.DaemonCfg = daemonCfg
-
-	if args.ValidateSignature {
-		if args.PublicKeyFile == "" {
-			return errors.New("need to specify publicKey file for signature validation")
-		} else if _, err := os.Stat(args.PublicKeyFile); err != nil {
-			return errors.Wrapf(err, "failed to find publicKey file %q", args.PublicKeyFile)
-		}
-	}
-	cfg.PublicKeyFile = args.PublicKeyFile
-	cfg.ValidateSignature = args.ValidateSignature
-
-	// Give --shared-daemon higher priority
-	cfg.DaemonMode = args.DaemonMode
-	if args.SharedDaemon {
-		cfg.DaemonMode = config.DaemonModeShared
-	}
-	if args.FsDriver == config.FsDriverFscache && args.DaemonMode != config.DaemonModeShared {
-		return errors.New("`fscache` backend driver only supports `shared` daemon mode")
-	}
-
-	cfg.RootDir = args.RootDir
-	if len(cfg.RootDir) == 0 {
-		return errors.New("invalid empty root directory")
-	}
-
-	cfg.CacheDir = args.CacheDir
-	if len(cfg.CacheDir) == 0 {
-		cfg.CacheDir = filepath.Join(cfg.RootDir, "cache")
-	}
-
-	cfg.LogLevel = args.LogLevel
-	// Always let options from CLI override those from configuration file.
-	cfg.LogToStdout = args.LogToStdout
-	cfg.LogDir = args.LogDir
-	if len(cfg.LogDir) == 0 {
-		cfg.LogDir = filepath.Join(cfg.RootDir, logging.DefaultLogDirName)
-	}
-
-	d, err := time.ParseDuration(args.GCPeriod)
-	if err != nil {
-		return errors.Wrapf(err, "parse gc period %v failed", args.GCPeriod)
-	}
-	cfg.GCPeriod = d
-
-	cfg.Address = args.Address
-	cfg.CleanupOnClose = args.CleanupOnClose
-	cfg.ConvertVpcRegistry = args.ConvertVpcRegistry
-	cfg.DisableCacheManager = args.DisableCacheManager
-	cfg.EnableMetrics = args.EnableMetrics
-	cfg.EnableStargz = args.EnableStargz
-	cfg.EnableNydusOverlayFS = args.EnableNydusOverlayFS
-	cfg.FsDriver = args.FsDriver
-	cfg.MetricsFile = args.MetricsFile
-	cfg.NydusdBinaryPath = args.NydusdBinaryPath
-	cfg.NydusImageBinaryPath = args.NydusImageBinaryPath
-	cfg.NydusdThreadNum = args.NydusdThreadNum
-	cfg.SyncRemove = args.SyncRemove
-	cfg.KubeconfigPath = args.KubeconfigPath
-	return cfg.SetupNydusBinaryPaths()
-}

cmd/containerd-nydus-grpc/snapshotter.go

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2020. Ant Group. All rights reserved.
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package main
+
+import (
+	"context"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+
+	"github.com/containerd/nydus-snapshotter/config"
+	"github.com/containerd/nydus-snapshotter/pkg/auth"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/signals"
+	"github.com/containerd/nydus-snapshotter/snapshot"
+
+	api "github.com/containerd/containerd/api/services/snapshots/v1"
+	"github.com/containerd/containerd/v2/contrib/snapshotservice"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/log"
+	"google.golang.org/grpc"
+)
+
+func Start(ctx context.Context, cfg *config.SnapshotterConfig) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	rs, err := snapshot.NewSnapshotter(ctx, cfg)
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize snapshotter")
+	}
+
+	stopSignal := signals.SetupSignalHandler()
+	opt := ServeOptions{
+		ListeningSocketPath: cfg.Address,
+		EnableCRIKeychain:   cfg.RemoteConfig.AuthConfig.EnableCRIKeychain,
+		ImageServiceAddress: cfg.RemoteConfig.AuthConfig.ImageServiceAddress,
+	}
+
+	if cfg.RemoteConfig.AuthConfig.EnableKubeconfigKeychain {
+		if err := auth.InitKubeSecretListener(ctx, cfg.RemoteConfig.AuthConfig.KubeconfigPath); err != nil {
+			return err
+		}
+	}
+
+	return Serve(ctx, rs, opt, stopSignal)
+}
+
+type ServeOptions struct {
+	ListeningSocketPath string
+	EnableCRIKeychain   bool
+	ImageServiceAddress string
+}
+
+func Serve(ctx context.Context, sn snapshots.Snapshotter, options ServeOptions, stop <-chan struct{}) error {
+	err := ensureSocketNotExists(options.ListeningSocketPath)
+	if err != nil {
+		return err
+	}
+	rpc := grpc.NewServer()
+	if rpc == nil {
+		return errors.New("start gRPC server")
+	}
+	api.RegisterSnapshotsServer(rpc, snapshotservice.FromSnapshotter(sn))
+	listener, err := net.Listen("unix", options.ListeningSocketPath)
+	if err != nil {
+		return errors.Wrapf(err, "listen socket %q", options.ListeningSocketPath)
+	}
+
+	if options.EnableCRIKeychain {
+		auth.AddImageProxy(ctx, rpc, options.ImageServiceAddress)
+	}
+
+	go func() {
+		<-stop
+
+		log.L.Infof("Shutting down nydus-snapshotter!")
+
+		if err := sn.Close(); err != nil {
+			log.L.WithError(err).Errorf("Closing snapshotter error")
+		}
+
+		if err := listener.Close(); err != nil {
+			log.L.Errorf("Failed to close listener %s, err: %v", options.ListeningSocketPath, err)
+		}
+	}()
+
+	return rpc.Serve(listener)
+}
+
+func ensureSocketNotExists(listeningSocketPath string) error {
+	if err := os.MkdirAll(filepath.Dir(listeningSocketPath), 0700); err != nil {
+		return errors.Wrapf(err, "failed to create directory %q", filepath.Dir(listeningSocketPath))
+	}
+	finfo, err := os.Stat(listeningSocketPath)
+	// err is nil means listening socket path exists, remove before serve
+	if err == nil {
+		if finfo.Mode()&os.ModeSocket == 0 {
+			return errors.Errorf("file %s is not a socket", listeningSocketPath)
+		}
+		err := os.Remove(listeningSocketPath)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/containerd-nydus-grpc/version.go

@@ -1,11 +0,0 @@
-/*
- * Copyright (c) 2020. Ant Group. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package main
-
-var (
-	Version = "development"
-)

cmd/converter/main.go

@@ -0,0 +1,10 @@
+package main
+
+// Import the converter package so that it can be compiled during
+// `go build` to ensure cross-compilation compatibility.
+import (
+	_ "github.com/containerd/nydus-snapshotter/pkg/converter"
+)
+
+func main() {
+}

cmd/nydus-overlayfs/main.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// Extra mount option to pass Nydus specific information from snapshotter to runtime through containerd.
+	extraOptionKey = "extraoption="
+	// Kata virtual volume infmation passed from snapshotter to runtime through containerd, superset of `extraOptionKey`.
+	// Please refer to `KataVirtualVolume` in https://github.com/kata-containers/kata-containers/blob/main/src/libs/kata-types/src/mount.rs
+	kataVolumeOptionKey = "io.katacontainers.volume="
+)
+
+var (
+	Version   = "v0.1"
+	BuildTime = "unknown"
+)
+
+/*
+containerd run fuse.mount format: nydus-overlayfs overlay /tmp/ctd-volume107067851
+-o lowerdir=/foo/lower2:/foo/lower1,upperdir=/foo/upper,workdir=/foo/work,extraoption={...},dev,suid]
+*/
+type mountArgs struct {
+	fsType  string
+	target  string
+	options []string
+}
+
+func parseArgs(args []string) (*mountArgs, error) {
+	margs := &mountArgs{
+		fsType: args[0],
+		target: args[1],
+	}
+	if margs.fsType != "overlay" {
+		return nil, errors.Errorf("invalid filesystem type %s for overlayfs", margs.fsType)
+	}
+	if len(margs.target) == 0 {
+		return nil, errors.New("empty overlayfs mount target")
+	}
+
+	if args[2] == "-o" && len(args[3]) != 0 {
+		for _, opt := range strings.Split(args[3], ",") {
+			// filter Nydus specific options
+			if strings.HasPrefix(opt, extraOptionKey) || strings.HasPrefix(opt, kataVolumeOptionKey) {
+				continue
+			}
+			margs.options = append(margs.options, opt)
+		}
+	}
+	if len(margs.options) == 0 {
+		return nil, errors.New("empty overlayfs mount options")
+	}
+
+	return margs, nil
+}
+
+func parseOptions(options []string) (int, string) {
+	flagsTable := map[string]int{
+		"async":         unix.MS_SYNCHRONOUS,
+		"atime":         unix.MS_NOATIME,
+		"bind":          unix.MS_BIND,
+		"defaults":      0,
+		"dev":           unix.MS_NODEV,
+		"diratime":      unix.MS_NODIRATIME,
+		"dirsync":       unix.MS_DIRSYNC,
+		"exec":          unix.MS_NOEXEC,
+		"mand":          unix.MS_MANDLOCK,
+		"noatime":       unix.MS_NOATIME,
+		"nodev":         unix.MS_NODEV,
+		"nodiratime":    unix.MS_NODIRATIME,
+		"noexec":        unix.MS_NOEXEC,
+		"nomand":        unix.MS_MANDLOCK,
+		"norelatime":    unix.MS_RELATIME,
+		"nostrictatime": unix.MS_STRICTATIME,
+		"nosuid":        unix.MS_NOSUID,
+		"rbind":         unix.MS_BIND | unix.MS_REC,
+		"relatime":      unix.MS_RELATIME,
+		"remount":       unix.MS_REMOUNT,
+		"ro":            unix.MS_RDONLY,
+		"rw":            unix.MS_RDONLY,
+		"strictatime":   unix.MS_STRICTATIME,
+		"suid":          unix.MS_NOSUID,
+		"sync":          unix.MS_SYNCHRONOUS,
+	}
+
+	var (
+		flags int
+		data  []string
+	)
+	for _, o := range options {
+		if f, exist := flagsTable[o]; exist {
+			flags |= f
+		} else {
+			data = append(data, o)
+		}
+	}
+	return flags, strings.Join(data, ",")
+}
+
+func run(args cli.Args) error {
+	margs, err := parseArgs(args.Slice())
+	if err != nil {
+		return errors.Wrap(err, "parse mount options")
+	}
+
+	flags, data := parseOptions(margs.options)
+	err = syscall.Mount(margs.fsType, margs.target, margs.fsType, uintptr(flags), data)
+	if err != nil {
+		return errors.Wrapf(err, "mount overlayfs by syscall")
+	}
+	return nil
+}
+
+func main() {
+	app := &cli.App{
+		Name:      "NydusOverlayfs",
+		Usage:     "FUSE mount helper for containerd to filter out Nydus specific options",
+		Version:   fmt.Sprintf("%s.%s", Version, BuildTime),
+		UsageText: "[Usage]: nydus-overlayfs overlay <target> -o <options>",
+		Action: func(c *cli.Context) error {
+			return run(c.Args())
+		},
+		Before: func(c *cli.Context) error {
+			if c.NArg() != 4 {
+				cli.ShowAppHelpAndExit(c, 1)
+			}
+			return nil
+		},
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Exit(0)
+}

cmd/optimizer-nri-plugin/main.go

@@ -0,0 +1,285 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/syslog"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/containerd/log"
+	distribution "github.com/distribution/reference"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/containerd/nri/pkg/api"
+	"github.com/containerd/nri/pkg/stub"
+	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/fanotify"
+	"github.com/containerd/nydus-snapshotter/version"
+	"github.com/pelletier/go-toml"
+)
+
+const (
+	defaultEvents     = "StartContainer,StopContainer"
+	defaultServerPath = "/usr/local/bin/optimizer-server"
+	defaultPersistDir = "/opt/nri/optimizer/results"
+)
+
+type PluginConfig struct {
+	Events []string `toml:"events"`
+
+	ServerPath string `toml:"server_path"`
+	PersistDir string `toml:"persist_dir"`
+	Readable   bool   `toml:"readable"`
+	Timeout    int    `toml:"timeout"`
+	Overwrite  bool   `toml:"overwrite"`
+}
+
+type PluginArgs struct {
+	PluginName   string
+	PluginIdx    string
+	PluginEvents string
+	Config       PluginConfig
+}
+
+type Flags struct {
+	Args *PluginArgs
+	F    []cli.Flag
+}
+
+func buildFlags(args *PluginArgs) []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "name",
+			Usage:       "plugin name to register to NRI",
+			Destination: &args.PluginName,
+		},
+		&cli.StringFlag{
+			Name:        "idx",
+			Usage:       "plugin index to register to NRI",
+			Destination: &args.PluginIdx,
+		},
+		&cli.StringFlag{
+			Name:        "events",
+			Value:       defaultEvents,
+			Usage:       "the events that containerd subscribes to. DO NOT CHANGE THIS.",
+			Destination: &args.PluginEvents,
+		},
+		&cli.StringFlag{
+			Name:        "server-path",
+			Value:       defaultServerPath,
+			Usage:       "the path of optimizer server binary",
+			Destination: &args.Config.ServerPath,
+		},
+		&cli.StringFlag{
+			Name:        "persist-dir",
+			Value:       defaultPersistDir,
+			Usage:       "the directory to persist accessed files list for container",
+			Destination: &args.Config.PersistDir,
+		},
+		&cli.BoolFlag{
+			Name:        "readable",
+			Value:       false,
+			Usage:       "whether to make the csv file human readable",
+			Destination: &args.Config.Readable,
+		},
+		&cli.IntFlag{
+			Name:        "timeout",
+			Value:       0,
+			Usage:       "the timeout to kill optimizer server, 0 to disable it",
+			Destination: &args.Config.Timeout,
+		},
+		&cli.BoolFlag{
+			Name:        "overwrite",
+			Usage:       "whether to overwrite the existed persistent files",
+			Destination: &args.Config.Overwrite,
+		},
+	}
+}
+
+func NewPluginFlags() *Flags {
+	var args PluginArgs
+	return &Flags{
+		Args: &args,
+		F:    buildFlags(&args),
+	}
+}
+
+type plugin struct {
+	stub stub.Stub
+	mask stub.EventMask
+}
+
+var (
+	cfg                  PluginConfig
+	logWriter            *syslog.Writer
+	globalFanotifyServer = make(map[string]*fanotify.Server)
+
+	_ = stub.ConfigureInterface(&plugin{})
+	_ = stub.StartContainerInterface(&plugin{})
+	_ = stub.StopContainerInterface(&plugin{})
+)
+
+const (
+	imageNameLabel = "io.kubernetes.cri.image-name"
+)
+
+func (p *plugin) Configure(ctx context.Context, config, runtime, version string) (stub.EventMask, error) {
+	log.G(ctx).Infof("got configuration data: %q from runtime %s %s", config, runtime, version)
+	if config == "" {
+		return p.mask, nil
+	}
+
+	tree, err := toml.Load(config)
+	if err != nil {
+		return 0, errors.Wrap(err, "parse TOML")
+	}
+	if err := tree.Unmarshal(&cfg); err != nil {
+		return 0, err
+	}
+
+	p.mask, err = api.ParseEventMask(cfg.Events...)
+	if err != nil {
+		return 0, errors.Wrap(err, "parse events in configuration")
+	}
+
+	log.G(ctx).Infof("configuration: %#v", cfg)
+
+	return p.mask, nil
+}
+
+func (p *plugin) StartContainer(_ context.Context, _ *api.PodSandbox, container *api.Container) error {
+	dir, imageName, err := GetImageName(container.Annotations)
+	if err != nil {
+		return err
+	}
+
+	persistDir := filepath.Join(cfg.PersistDir, dir)
+	if err := os.MkdirAll(persistDir, os.ModePerm); err != nil {
+		return err
+	}
+
+	persistFile := filepath.Join(persistDir, imageName)
+	if cfg.Timeout > 0 {
+		persistFile = fmt.Sprintf("%s.timeout%ds", persistFile, cfg.Timeout)
+	}
+
+	fanotifyServer := fanotify.NewServer(cfg.ServerPath, container.Pid, imageName, persistFile, cfg.Readable, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter)
+
+	if err := fanotifyServer.RunServer(); err != nil {
+		return err
+	}
+
+	globalFanotifyServer[imageName] = fanotifyServer
+
+	return nil
+}
+
+func (p *plugin) StopContainer(_ context.Context, _ *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) {
+	var update = []*api.ContainerUpdate{}
+	_, imageName, err := GetImageName(container.Annotations)
+	if err != nil {
+		return update, err
+	}
+	if fanotifyServer, ok := globalFanotifyServer[imageName]; ok {
+		fanotifyServer.StopServer()
+	} else {
+		return nil, errors.New("can not find fanotify server for container image " + imageName)
+	}
+
+	return update, nil
+}
+
+func GetImageName(annotations map[string]string) (string, string, error) {
+	named, err := distribution.ParseDockerRef(annotations[imageNameLabel])
+	if err != nil {
+		return "", "", err
+	}
+	nameTagged := named.(distribution.NamedTagged)
+	repo := distribution.Path(nameTagged)
+
+	dir := filepath.Dir(repo)
+	image := filepath.Base(repo)
+
+	imageName := image + ":" + nameTagged.Tag()
+
+	return dir, imageName, nil
+}
+
+func (p *plugin) onClose() {
+	for _, fanotifyServer := range globalFanotifyServer {
+		fanotifyServer.StopServer()
+	}
+	os.Exit(0)
+}
+
+func main() {
+	flags := NewPluginFlags()
+	app := &cli.App{
+		Name:        "optimizer-nri-plugin",
+		Usage:       "Optimizer client for NRI plugin to manage optimizer server",
+		Version:     version.Version,
+		Flags:       flags.F,
+		HideVersion: true,
+		Action: func(_ *cli.Context) error {
+			var (
+				opts []stub.Option
+				err  error
+			)
+
+			cfg = flags.Args.Config
+
+			// FIXME(thaJeztah): ucontainerd's log does not set "PadLevelText: true"
+			_ = log.SetFormat(log.TextFormat)
+			ctx := log.WithLogger(context.Background(), log.L)
+
+			logWriter, err = syslog.New(syslog.LOG_INFO, "optimizer-nri-plugin")
+			if err == nil {
+				log.G(ctx).Logger.SetOutput(io.MultiWriter(os.Stdout, logWriter))
+			}
+
+			if flags.Args.PluginName != "" {
+				opts = append(opts, stub.WithPluginName(flags.Args.PluginName))
+			}
+			if flags.Args.PluginIdx != "" {
+				opts = append(opts, stub.WithPluginIdx(flags.Args.PluginIdx))
+			}
+
+			p := &plugin{}
+
+			if p.mask, err = api.ParseEventMask(flags.Args.PluginEvents); err != nil {
+				log.G(ctx).Fatalf("failed to parse events: %v", err)
+			}
+			cfg.Events = strings.Split(flags.Args.PluginEvents, ",")
+
+			if p.stub, err = stub.New(p, append(opts, stub.WithOnClose(p.onClose))...); err != nil {
+				log.G(ctx).Fatalf("failed to create plugin stub: %v", err)
+			}
+
+			err = p.stub.Run(context.Background())
+			if err != nil {
+				log.G(ctx).Errorf("plugin exited with error %v", err)
+				os.Exit(1)
+			}
+
+			return nil
+		},
+	}
+	if err := app.Run(os.Args); err != nil {
+		if errdefs.IsConnectionClosed(err) {
+			log.L.Info("optimizer NRI plugin exited")
+		} else {
+			log.L.WithError(err).Fatal("failed to start optimizer NRI plugin")
+		}
+	}
+}

cmd/prefetchfiles-nri-plugin/main.go

@@ -0,0 +1,209 @@
+/*
+* Copyright (c) 2023. Nydus Developers. All rights reserved.
+*
+* SPDX-License-Identifier: Apache-2.0
+ */
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/syslog"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/containerd/log"
+	"github.com/containerd/nri/pkg/api"
+	"github.com/containerd/nri/pkg/stub"
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/version"
+)
+
+const (
+	endpointPrefetch               = "/api/v1/prefetch"
+	defaultEvents                  = "RunPodSandbox"
+	defaultSystemControllerAddress = "/run/containerd-nydus/system.sock"
+	defaultPrefetchConfigDir       = "/etc/nydus"
+	nydusPrefetchAnnotation        = "containerd.io/nydus-prefetch"
+)
+
+type PluginArgs struct {
+	PluginName    string
+	PluginIdx     string
+	SocketAddress string
+}
+
+type Flags struct {
+	Args *PluginArgs
+	Flag []cli.Flag
+}
+
+func buildFlags(args *PluginArgs) []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "name",
+			Usage:       "plugin name to register to NRI",
+			Destination: &args.PluginName,
+		},
+		&cli.StringFlag{
+			Name:        "idx",
+			Usage:       "plugin index to register to NRI",
+			Destination: &args.PluginIdx,
+		},
+		&cli.StringFlag{
+			Name:        "socket-addr",
+			Value:       defaultSystemControllerAddress,
+			Usage:       "unix domain socket address. If defined in the configuration file, there is no need to add ",
+			Destination: &args.SocketAddress,
+		},
+	}
+}
+
+func NewPluginFlags() *Flags {
+	var args PluginArgs
+	return &Flags{
+		Args: &args,
+		Flag: buildFlags(&args),
+	}
+}
+
+type plugin struct {
+	stub stub.Stub
+	mask stub.EventMask
+}
+
+var (
+	globalSocket string
+	logWriter    *syslog.Writer
+
+	_ = stub.RunPodInterface(&plugin{})
+)
+
+// sendDataOverHTTP sends the prefetch data to the specified endpoint over HTTP using a Unix socket.
+func sendDataOverHTTP(data string, endpoint, sock string) error {
+	url := fmt.Sprintf("http://unix%s", endpoint)
+
+	req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", sock)
+			},
+		},
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to send data, status code: %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+
+	return nil
+}
+
+func (p *plugin) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) error {
+	prefetchList, ok := pod.Annotations[nydusPrefetchAnnotation]
+	if !ok {
+		return nil
+	}
+
+	err := sendDataOverHTTP(prefetchList, endpointPrefetch, globalSocket)
+	if err != nil {
+		log.G(ctx).Errorf("failed to send data: %v", err)
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	flags := NewPluginFlags()
+
+	app := &cli.App{
+		Name:        "prefetch-nri-plugin",
+		Usage:       "NRI plugin for obtaining and transmitting prefetch files path",
+		Version:     version.Version,
+		Flags:       flags.Flag,
+		HideVersion: true,
+		Action: func(_ *cli.Context) error {
+			var (
+				opts []stub.Option
+				err  error
+			)
+
+			// FIXME(thaJeztah): ucontainerd's log does not set "PadLevelText: true"
+			_ = log.SetFormat(log.TextFormat)
+			ctx := log.WithLogger(context.Background(), log.L)
+
+			configFileName := "prefetchConfig.toml"
+			configDir := defaultPrefetchConfigDir
+			configFilePath := filepath.Join(configDir, configFileName)
+
+			config, err := toml.LoadFile(configFilePath)
+			if err != nil {
+				log.G(ctx).Warnf("failed to read config file: %v", err)
+			}
+
+			configSocketAddrRaw := config.Get("file_prefetch.socket_address")
+			if configSocketAddrRaw != nil {
+				if configSocketAddr, ok := configSocketAddrRaw.(string); ok {
+					globalSocket = configSocketAddr
+				} else {
+					log.G(ctx).Warnf("failed to read config: 'file_prefetch.socket_address' is not a string")
+				}
+			} else {
+				globalSocket = flags.Args.SocketAddress
+			}
+
+			logWriter, err = syslog.New(syslog.LOG_INFO, "prefetch-nri-plugin")
+			if err == nil {
+				log.G(ctx).Logger.SetOutput(io.MultiWriter(os.Stdout, logWriter))
+			}
+
+			if flags.Args.PluginName != "" {
+				opts = append(opts, stub.WithPluginName(flags.Args.PluginName))
+			}
+			if flags.Args.PluginIdx != "" {
+				opts = append(opts, stub.WithPluginIdx(flags.Args.PluginIdx))
+			}
+
+			p := &plugin{}
+
+			if p.mask, err = api.ParseEventMask(defaultEvents); err != nil {
+				log.G(ctx).Fatalf("failed to parse events: %v", err)
+			}
+
+			if p.stub, err = stub.New(p, opts...); err != nil {
+				log.G(ctx).Fatalf("failed to create plugin stub: %v", err)
+			}
+
+			err = p.stub.Run(context.Background())
+			if err != nil {
+				return errors.Wrap(err, "plugin exited")
+			}
+			return nil
+		},
+	}
+	if err := app.Run(os.Args); err != nil {
+		if errdefs.IsConnectionClosed(err) {
+			log.L.Info("prefetch NRI plugin exited")
+		} else {
+			log.L.WithError(err).Fatal("failed to start prefetch NRI plugin")
+		}
+	}
+}

config/config.go

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2020. Ant Group. All rights reserved.
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -7,115 +8,378 @@
 package config
 
 import (
-	"path/filepath"
-	"time"
+	"os"
 
-	"github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/logging"
+	"dario.cat/mergo"
+	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
-	exec "golang.org/x/sys/execabs"
+
+	"github.com/containerd/nydus-snapshotter/internal/constant"
+	"github.com/containerd/nydus-snapshotter/internal/flags"
+	"github.com/containerd/nydus-snapshotter/pkg/cgroup"
+	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/file"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/parser"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/sysinfo"
 )
 
-const (
-	DefaultDaemonMode  string = "multiple"
-	DaemonModeMultiple string = "multiple"
-	DaemonModeShared   string = "shared"
-	DaemonModeNone     string = "none"
-	DaemonModePrefetch string = "prefetch"
-	DefaultLogLevel    string = "info"
-	defaultGCPeriod           = 24 * time.Hour
-
-	defaultNydusDaemonConfigPath string = "/etc/nydus/config.json"
-	nydusdBinaryName             string = "nydusd"
-	nydusImageBinaryName         string = "nydus-image"
-)
-
-const (
-	FsDriverFusedev string = "fusedev"
-	FsDriverFscache string = "fscache"
-)
-
-type Config struct {
-	Address              string        `toml:"-"`
-	ConvertVpcRegistry   bool          `toml:"-"`
-	DaemonCfgPath        string        `toml:"daemon_cfg_path"`
-	DaemonCfg            DaemonConfig  `toml:"-"`
-	PublicKeyFile        string        `toml:"-"`
-	RootDir              string        `toml:"-"`
-	CacheDir             string        `toml:"cache_dir"`
-	GCPeriod             time.Duration `toml:"gc_period"`
-	ValidateSignature    bool          `toml:"validate_signature"`
-	NydusdBinaryPath     string        `toml:"nydusd_binary_path"`
-	NydusImageBinaryPath string        `toml:"nydus_image_binary"`
-	DaemonMode           string        `toml:"daemon_mode"`
-	FsDriver             string        `toml:"daemon_backend"`
-	SyncRemove           bool          `toml:"sync_remove"`
-	EnableMetrics        bool          `toml:"enable_metrics"`
-	MetricsFile          string        `toml:"metrics_file"`
-	EnableStargz         bool          `toml:"enable_stargz"`
-	LogLevel             string        `toml:"-"`
-	LogDir               string        `toml:"log_dir"`
-	LogToStdout          bool          `toml:"log_to_stdout"`
-	DisableCacheManager  bool          `toml:"disable_cache_manager"`
-	EnableNydusOverlayFS bool          `toml:"enable_nydus_overlayfs"`
-	NydusdThreadNum      int           `toml:"nydusd_thread_num"`
-	CleanupOnClose       bool          `toml:"cleanup_on_close"`
-	KubeconfigPath       string        `toml:"kubeconfig_path"`
+func init() {
+	recoverPolicyParser = map[string]DaemonRecoverPolicy{
+		RecoverPolicyNone.String():     RecoverPolicyNone,
+		RecoverPolicyRestart.String():  RecoverPolicyRestart,
+		RecoverPolicyFailover.String(): RecoverPolicyFailover}
 }
 
-func (c *Config) FillupWithDefaults() error {
-	if c.LogLevel == "" {
-		c.LogLevel = DefaultLogLevel
-	}
-	if c.DaemonCfgPath == "" {
-		c.DaemonCfgPath = defaultNydusDaemonConfigPath
-	}
+// Define a policy how to fork nydusd daemon and attach file system instances to serve.
+type DaemonMode string
 
-	if c.DaemonMode == "" {
-		c.DaemonMode = DefaultDaemonMode
-	}
+const (
+	// Spawn a dedicated nydusd for each RAFS instance.
+	DaemonModeMultiple DaemonMode = DaemonMode(constant.DaemonModeMultiple)
+	// Spawn a dedicated nydusd for each RAFS instance.
+	DaemonModeDedicated DaemonMode = DaemonMode(constant.DaemonModeDedicated)
+	// Share a global nydusd to serve all RAFS instances.
+	DaemonModeShared DaemonMode = DaemonMode(constant.DaemonModeShared)
+	// Do not spawn nydusd for RAFS instances.
+	//
+	// For tarfs and rund, there's no need to create nydusd to serve RAFS instances,
+	// the snapshotter just returns mount slices with additional information for runC/runD
+	// to manage those snapshots.
+	DaemonModeNone    DaemonMode = DaemonMode(constant.DaemonModeNone)
+	DaemonModeInvalid DaemonMode = DaemonMode(constant.DaemonModeInvalid)
+)
 
-	if c.GCPeriod == 0 {
-		c.GCPeriod = defaultGCPeriod
+func parseDaemonMode(m string) (DaemonMode, error) {
+	switch m {
+	case string(DaemonModeMultiple):
+		return DaemonModeDedicated, nil
+	case string(DaemonModeDedicated):
+		return DaemonModeDedicated, nil
+	case string(DaemonModeShared):
+		return DaemonModeShared, nil
+	case string(DaemonModeNone):
+		return DaemonModeNone, nil
+	default:
+		return DaemonModeInvalid, errors.Errorf("invalid daemon mode %q", m)
 	}
-
-	if len(c.CacheDir) == 0 {
-		c.CacheDir = filepath.Join(c.RootDir, "cache")
-	}
-
-	if len(c.LogDir) == 0 {
-		c.LogDir = filepath.Join(c.RootDir, logging.DefaultLogDirName)
-	}
-	var daemonCfg DaemonConfig
-	if err := LoadConfig(c.DaemonCfgPath, &daemonCfg); err != nil {
-		return errors.Wrapf(err, "failed to load config file %q", c.DaemonCfgPath)
-	}
-	c.DaemonCfg = daemonCfg
-	return c.SetupNydusBinaryPaths()
 }
 
-func (c *Config) SetupNydusBinaryPaths() error {
-	// when using DaemonMode = none, nydusd and nydus-image binaries are not required
-	if c.DaemonMode == DaemonModeNone {
-		return nil
+type DaemonRecoverPolicy int
+
+const (
+	RecoverPolicyInvalid DaemonRecoverPolicy = iota
+	RecoverPolicyNone
+	RecoverPolicyRestart
+	RecoverPolicyFailover
+)
+
+func (p DaemonRecoverPolicy) String() string {
+	switch p {
+	case RecoverPolicyNone:
+		return "none"
+	case RecoverPolicyRestart:
+		return "restart"
+	case RecoverPolicyFailover:
+		return "failover"
+	case RecoverPolicyInvalid:
+		fallthrough
+	default:
+		return ""
+	}
+}
+
+var recoverPolicyParser map[string]DaemonRecoverPolicy
+
+func ParseRecoverPolicy(p string) (DaemonRecoverPolicy, error) {
+	policy, ok := recoverPolicyParser[p]
+	if !ok {
+		return RecoverPolicyInvalid, errors.Errorf("invalid recover policy %q", p)
 	}
 
-	// resolve nydusd path
-	if c.NydusdBinaryPath == "" {
-		path, err := exec.LookPath(nydusdBinaryName)
-		if err != nil {
-			return err
-		}
-		c.NydusdBinaryPath = path
+	return policy, nil
+}
+
+const (
+	FsDriverBlockdev string = constant.FsDriverBlockdev
+	FsDriverFusedev  string = constant.FsDriverFusedev
+	FsDriverFscache  string = constant.FsDriverFscache
+	FsDriverNodev    string = constant.FsDriverNodev
+	FsDriverProxy    string = constant.FsDriverProxy
+)
+
+type Experimental struct {
+	EnableStargz         bool        `toml:"enable_stargz"`
+	EnableReferrerDetect bool        `toml:"enable_referrer_detect"`
+	TarfsConfig          TarfsConfig `toml:"tarfs"`
+	EnableBackendSource  bool        `toml:"enable_backend_source"`
+}
+
+type TarfsConfig struct {
+	EnableTarfs       bool   `toml:"enable_tarfs"`
+	MountTarfsOnHost  bool   `toml:"mount_tarfs_on_host"`
+	TarfsHint         bool   `toml:"tarfs_hint"`
+	MaxConcurrentProc int    `toml:"max_concurrent_proc"`
+	ExportMode        string `toml:"export_mode"`
+}
+
+type CgroupConfig struct {
+	Enable      bool   `toml:"enable"`
+	MemoryLimit string `toml:"memory_limit"`
+}
+
+// Configure how to start and recover nydusd daemons
+type DaemonConfig struct {
+	NydusdPath       string `toml:"nydusd_path"`
+	NydusdConfigPath string `toml:"nydusd_config"`
+	NydusImagePath   string `toml:"nydusimage_path"`
+	RecoverPolicy    string `toml:"recover_policy"`
+	FsDriver         string `toml:"fs_driver"`
+	ThreadsNumber    int    `toml:"threads_number"`
+	LogRotationSize  int    `toml:"log_rotation_size"`
+}
+
+type LoggingConfig struct {
+	LogToStdout         bool   `toml:"log_to_stdout"`
+	LogLevel            string `toml:"level"`
+	LogDir              string `toml:"dir"`
+	RotateLogMaxSize    int    `toml:"log_rotation_max_size"`
+	RotateLogMaxBackups int    `toml:"log_rotation_max_backups"`
+	RotateLogMaxAge     int    `toml:"log_rotation_max_age"`
+	RotateLogLocalTime  bool   `toml:"log_rotation_local_time"`
+	RotateLogCompress   bool   `toml:"log_rotation_compress"`
+}
+
+// Nydus image layers additional process
+type ImageConfig struct {
+	PublicKeyFile     string `toml:"public_key_file"`
+	ValidateSignature bool   `toml:"validate_signature"`
+}
+
+// Configure containerd snapshots interfaces and how to process the snapshots
+// requests from containerd
+type SnapshotConfig struct {
+	EnableNydusOverlayFS bool   `toml:"enable_nydus_overlayfs"`
+	NydusOverlayFSPath   string `toml:"nydus_overlayfs_path"`
+	EnableKataVolume     bool   `toml:"enable_kata_volume"`
+	SyncRemove           bool   `toml:"sync_remove"`
+}
+
+// Configure cache manager that manages the cache files lifecycle
+type CacheManagerConfig struct {
+	Disable bool `toml:"disable"`
+	// Trigger GC gc_period after the specified period.
+	// Example format: 24h, 120min
+	GCPeriod string `toml:"gc_period"`
+	CacheDir string `toml:"cache_dir"`
+}
+
+// Configure how nydus-snapshotter receive auth information
+type AuthConfig struct {
+	// based on kubeconfig or ServiceAccount
+	EnableKubeconfigKeychain bool   `toml:"enable_kubeconfig_keychain"`
+	KubeconfigPath           string `toml:"kubeconfig_path"`
+	// CRI proxy mode
+	EnableCRIKeychain   bool   `toml:"enable_cri_keychain"`
+	ImageServiceAddress string `toml:"image_service_address"`
+}
+
+// Configure remote storage like container registry
+type RemoteConfig struct {
+	AuthConfig         AuthConfig    `toml:"auth"`
+	ConvertVpcRegistry bool          `toml:"convert_vpc_registry"`
+	SkipSSLVerify      bool          `toml:"skip_ssl_verify"`
+	MirrorsConfig      MirrorsConfig `toml:"mirrors_config"`
+}
+
+type MirrorsConfig struct {
+	Dir string `toml:"dir"`
+}
+
+type MetricsConfig struct {
+	Address string `toml:"address"`
+}
+
+type DebugConfig struct {
+	ProfileDuration int64  `toml:"daemon_cpu_profile_duration_secs"`
+	PprofAddress    string `toml:"pprof_address"`
+}
+
+type SystemControllerConfig struct {
+	Enable      bool        `toml:"enable"`
+	Address     string      `toml:"address"`
+	DebugConfig DebugConfig `toml:"debug"`
+}
+
+type SnapshotterConfig struct {
+	// Configuration format version
+	Version int `toml:"version"`
+	// Snapshotter's root work directory
+	Root       string `toml:"root"`
+	Address    string `toml:"address"`
+	DaemonMode string `toml:"daemon_mode"`
+	// Clean up all the resources when snapshotter is closed
+	CleanupOnClose bool `toml:"cleanup_on_close"`
+
+	SystemControllerConfig SystemControllerConfig `toml:"system"`
+	MetricsConfig          MetricsConfig          `toml:"metrics"`
+	DaemonConfig           DaemonConfig           `toml:"daemon"`
+	SnapshotsConfig        SnapshotConfig         `toml:"snapshot"`
+	RemoteConfig           RemoteConfig           `toml:"remote"`
+	ImageConfig            ImageConfig            `toml:"image"`
+	CacheManagerConfig     CacheManagerConfig     `toml:"cache_manager"`
+	LoggingConfig          LoggingConfig          `toml:"log"`
+	CgroupConfig           CgroupConfig           `toml:"cgroup"`
+	Experimental           Experimental           `toml:"experimental"`
+}
+
+func LoadSnapshotterConfig(path string) (*SnapshotterConfig, error) {
+	var config SnapshotterConfig
+	// get nydus-snapshotter configuration from specified path of toml file
+	if path == "" {
+		return nil, errors.New("snapshotter configuration path cannot be empty")
+	}
+	tree, err := toml.LoadFile(path)
+	if err != nil {
+		return nil, errors.Wrapf(err, "load toml configuration from file %q", path)
 	}
 
-	// resolve nydus-image path
-	if c.NydusImageBinaryPath == "" {
-		path, err := exec.LookPath(nydusImageBinaryName)
-		if err != nil {
-			return err
-		}
-		c.NydusImageBinaryPath = path
+	if err = tree.Unmarshal(&config); err != nil {
+		return nil, errors.Wrap(err, "unmarshal snapshotter configuration")
+	}
+	if config.Version != 1 {
+		return nil, errors.Errorf("unsupported configuration version %d", config.Version)
+	}
+	return &config, nil
+}
+
+func MergeConfig(to, from *SnapshotterConfig) error {
+	err := mergo.Merge(to, from)
+	if err != nil {
+		return err
 	}
 
 	return nil
 }
+
+func ValidateConfig(c *SnapshotterConfig) error {
+	if c == nil {
+		return errors.Wrapf(errdefs.ErrInvalidArgument, "configuration is none")
+	}
+
+	if c.ImageConfig.ValidateSignature {
+		if c.ImageConfig.PublicKeyFile == "" {
+			return errors.New("public key file for signature validation is not provided")
+		} else if _, err := os.Stat(c.ImageConfig.PublicKeyFile); err != nil {
+			return errors.Wrapf(err, "check publicKey file %q", c.ImageConfig.PublicKeyFile)
+		}
+	}
+
+	if len(c.Root) == 0 {
+		return errors.New("empty root directory")
+	}
+
+	if c.DaemonConfig.FsDriver != FsDriverFscache && c.DaemonConfig.FsDriver != FsDriverFusedev &&
+		c.DaemonConfig.FsDriver != FsDriverBlockdev && c.DaemonConfig.FsDriver != FsDriverNodev &&
+		c.DaemonConfig.FsDriver != FsDriverProxy {
+		return errors.Errorf("invalid filesystem driver %q", c.DaemonConfig.FsDriver)
+	}
+	if _, err := ParseRecoverPolicy(c.DaemonConfig.RecoverPolicy); err != nil {
+		return err
+	}
+	if c.DaemonConfig.ThreadsNumber > 1024 {
+		return errors.Errorf("nydusd worker thread number %d is too big, max 1024", c.DaemonConfig.ThreadsNumber)
+	}
+
+	if c.RemoteConfig.AuthConfig.EnableCRIKeychain && c.RemoteConfig.AuthConfig.EnableKubeconfigKeychain {
+		return errors.Wrapf(errdefs.ErrInvalidArgument,
+			"\"enable_cri_keychain\" and \"enable_kubeconfig_keychain\" can't be set at the same time")
+	}
+
+	if c.RemoteConfig.MirrorsConfig.Dir != "" {
+		dirExisted, err := file.IsDirExisted(c.RemoteConfig.MirrorsConfig.Dir)
+		if err != nil {
+			return err
+		}
+		if !dirExisted {
+			return errors.Errorf("mirrors config directory %s does not exist", c.RemoteConfig.MirrorsConfig.Dir)
+		}
+	}
+
+	return nil
+}
+
+// Parse command line arguments and fill the nydus-snapshotter configuration
+// Always let options from CLI override those from configuration file.
+func ParseParameters(args *flags.Args, cfg *SnapshotterConfig) error {
+	// --- essential configuration
+	if args.Address != "" {
+		cfg.Address = args.Address
+	}
+	if args.RootDir != "" {
+		cfg.Root = args.RootDir
+	}
+
+	// Give --shared-daemon higher priority
+	if args.DaemonMode != "" {
+		cfg.DaemonMode = args.DaemonMode
+	}
+
+	// --- image processor configuration
+	// empty
+
+	// --- daemon configuration
+	daemonConfig := &cfg.DaemonConfig
+	if args.NydusdConfigPath != "" {
+		daemonConfig.NydusdConfigPath = args.NydusdConfigPath
+	}
+	if args.NydusdPath != "" {
+		daemonConfig.NydusdPath = args.NydusdPath
+	}
+	if args.NydusImagePath != "" {
+		daemonConfig.NydusImagePath = args.NydusImagePath
+	}
+	if args.FsDriver != "" {
+		daemonConfig.FsDriver = args.FsDriver
+	}
+
+	// --- cache manager configuration
+	// empty
+
+	// --- logging configuration
+	logConfig := &cfg.LoggingConfig
+	if args.LogLevel != "" {
+		logConfig.LogLevel = args.LogLevel
+	}
+	if args.LogToStdoutCount > 0 {
+		logConfig.LogToStdout = args.LogToStdout
+	}
+
+	// --- remote storage configuration
+	// empty
+
+	// --- snapshot configuration
+	if args.NydusOverlayFSPath != "" {
+		cfg.SnapshotsConfig.NydusOverlayFSPath = args.NydusOverlayFSPath
+	}
+
+	// --- metrics configuration
+	// empty
+
+	return nil
+}
+
+func ParseCgroupConfig(config CgroupConfig) (cgroup.Config, error) {
+	totalMemory, err := sysinfo.GetTotalMemoryBytes()
+	if err != nil {
+		return cgroup.Config{}, errors.Wrap(err, "Failed  to get total memory bytes")
+	}
+
+	memoryLimitInBytes, err := parser.MemoryConfigToBytes(config.MemoryLimit, totalMemory)
+	if err != nil {
+		return cgroup.Config{}, err
+	}
+
+	return cgroup.Config{
+		MemoryLimitInBytes: memoryLimitInBytes,
+	}, nil
+}

config/config_test.go

@@ -0,0 +1,256 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package config
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/containerd/nydus-snapshotter/internal/constant"
+	"github.com/containerd/nydus-snapshotter/internal/flags"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadSnapshotterTOMLConfig(t *testing.T) {
+	A := assert.New(t)
+
+	cfg, err := LoadSnapshotterConfig("../misc/snapshotter/config.toml")
+	A.NoError(err)
+
+	exampleConfig := SnapshotterConfig{
+		Version:    1,
+		Root:       "/var/lib/containerd/io.containerd.snapshotter.v1.nydus",
+		Address:    "/run/containerd-nydus/containerd-nydus-grpc.sock",
+		DaemonMode: "dedicated",
+		Experimental: Experimental{
+			EnableStargz:         false,
+			EnableReferrerDetect: false,
+		},
+		CleanupOnClose: false,
+		SystemControllerConfig: SystemControllerConfig{
+			Enable:  true,
+			Address: "/run/containerd-nydus/system.sock",
+			DebugConfig: DebugConfig{
+				ProfileDuration: 5,
+				PprofAddress:    "",
+			},
+		},
+		DaemonConfig: DaemonConfig{
+			NydusdPath:       "/usr/local/bin/nydusd",
+			NydusImagePath:   "/usr/local/bin/nydus-image",
+			FsDriver:         "fusedev",
+			RecoverPolicy:    "restart",
+			NydusdConfigPath: "/etc/nydus/nydusd-config.fusedev.json",
+			ThreadsNumber:    4,
+			LogRotationSize:  100,
+		},
+		SnapshotsConfig: SnapshotConfig{
+			EnableNydusOverlayFS: false,
+			NydusOverlayFSPath:   "nydus-overlayfs",
+			SyncRemove:           false,
+		},
+		RemoteConfig: RemoteConfig{
+			ConvertVpcRegistry: false,
+			AuthConfig: AuthConfig{
+				EnableKubeconfigKeychain: false,
+				KubeconfigPath:           "",
+			},
+			MirrorsConfig: MirrorsConfig{
+				Dir: "",
+			},
+		},
+		ImageConfig: ImageConfig{
+			PublicKeyFile:     "",
+			ValidateSignature: false,
+		},
+		CacheManagerConfig: CacheManagerConfig{
+			Disable:  false,
+			GCPeriod: "24h",
+			CacheDir: "",
+		},
+		LoggingConfig: LoggingConfig{
+			LogLevel:            "info",
+			RotateLogCompress:   true,
+			RotateLogLocalTime:  true,
+			RotateLogMaxAge:     7,
+			RotateLogMaxBackups: 5,
+			RotateLogMaxSize:    100,
+			LogToStdout:         false,
+		},
+		MetricsConfig: MetricsConfig{
+			Address: ":9110",
+		},
+		CgroupConfig: CgroupConfig{
+			Enable:      true,
+			MemoryLimit: "",
+		},
+	}
+
+	A.EqualValues(cfg, &exampleConfig)
+
+	args := flags.Args{}
+	args.RootDir = "/var/lib/containerd/nydus"
+	exampleConfig.Root = "/var/lib/containerd/nydus"
+
+	err = ParseParameters(&args, cfg)
+	A.NoError(err)
+	A.EqualValues(cfg, &exampleConfig)
+
+	A.EqualValues(cfg.LoggingConfig.LogToStdout, false)
+
+	args.LogToStdout = true
+	args.LogToStdoutCount = 1
+	err = ParseParameters(&args, cfg)
+	A.NoError(err)
+	A.EqualValues(cfg.LoggingConfig.LogToStdout, true)
+
+	err = ProcessConfigurations(cfg)
+	A.NoError(err)
+
+	A.Equal(GetCacheGCPeriod(), time.Hour*24)
+}
+
+func TestSnapshotterConfig(t *testing.T) {
+	A := assert.New(t)
+
+	var cfg SnapshotterConfig
+	var args flags.Args
+
+	// The log_to_stdout is false in toml file without --log-to-stdout flag.
+	// Expected false.
+	cfg.LoggingConfig.LogToStdout = false
+	args.LogToStdoutCount = 0
+	err := ParseParameters(&args, &cfg)
+	A.NoError(err)
+	A.EqualValues(cfg.LoggingConfig.LogToStdout, false)
+
+	// The log_to_stdout is true in toml file without --log-to-stdout flag.
+	// Expected true.
+	// This case is failed.
+	cfg.LoggingConfig.LogToStdout = true
+	args.LogToStdoutCount = 0
+	err = ParseParameters(&args, &cfg)
+	A.NoError(err)
+	A.EqualValues(cfg.LoggingConfig.LogToStdout, true)
+
+	// The log_to_stdout is false in toml file with --log-to-stdout=true.
+	// Expected true (command flag has higher priority).
+	args.LogToStdout = true
+	args.LogToStdoutCount = 1
+	cfg.LoggingConfig.LogToStdout = false
+	err = ParseParameters(&args, &cfg)
+	A.NoError(err)
+	A.EqualValues(cfg.LoggingConfig.LogToStdout, true)
+
+	// The log_to_stdout is true in toml file with --log-to-stdout=true.
+	// Expected true (command flag has higher priority).
+	args.LogToStdout = true
+	args.LogToStdoutCount = 1
+	cfg.LoggingConfig.LogToStdout = true
+	err = ParseParameters(&args, &cfg)
+	A.NoError(err)
+	A.EqualValues(cfg.LoggingConfig.LogToStdout, true)
+
+	// The log_to_stdout is false in toml file with --log-to-stdout=false.
+	// Expected false (command flag has higher priority).
+	args.LogToStdout = false
+	args.LogToStdoutCount = 1
+	cfg.LoggingConfig.LogToStdout = false
+	err = ParseParameters(&args, &cfg)
+	A.NoError(err)
+	A.EqualValues(cfg.LoggingConfig.LogToStdout, false)
+
+	// The log_to_stdout is true in toml file with --log-to-stdout=false.
+	// Expected false (command flag has higher priority).
+	args.LogToStdout = false
+	args.LogToStdoutCount = 1
+	cfg.LoggingConfig.LogToStdout = true
+	err = ParseParameters(&args, &cfg)
+	A.NoError(err)
+	A.EqualValues(cfg.LoggingConfig.LogToStdout, false)
+}
+
+func TestMergeConfig(t *testing.T) {
+	A := assert.New(t)
+	var defaultSnapshotterConfig SnapshotterConfig
+	var snapshotterConfig1 SnapshotterConfig
+
+	err := defaultSnapshotterConfig.FillUpWithDefaults()
+	A.NoError(err)
+
+	err = MergeConfig(&snapshotterConfig1, &defaultSnapshotterConfig)
+	A.NoError(err)
+	A.Equal(snapshotterConfig1.Root, constant.DefaultRootDir)
+	A.Equal(snapshotterConfig1.LoggingConfig.LogDir, "")
+	A.Equal(snapshotterConfig1.CacheManagerConfig.CacheDir, "")
+
+	A.Equal(snapshotterConfig1.DaemonMode, constant.DefaultDaemonMode)
+	A.Equal(snapshotterConfig1.SystemControllerConfig.Address, constant.DefaultSystemControllerAddress)
+	A.Equal(snapshotterConfig1.LoggingConfig.LogLevel, constant.DefaultLogLevel)
+	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxSize, constant.DefaultRotateLogMaxSize)
+	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxBackups, constant.DefaultRotateLogMaxBackups)
+	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxAge, constant.DefaultRotateLogMaxAge)
+	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogCompress, constant.DefaultRotateLogCompress)
+
+	A.Equal(snapshotterConfig1.DaemonConfig.NydusdConfigPath, constant.DefaultNydusDaemonConfigPath)
+	A.Equal(snapshotterConfig1.DaemonConfig.RecoverPolicy, RecoverPolicyRestart.String())
+	A.Equal(snapshotterConfig1.CacheManagerConfig.GCPeriod, constant.DefaultGCPeriod)
+
+	var snapshotterConfig2 SnapshotterConfig
+	snapshotterConfig2.Root = "/snapshotter/root"
+
+	err = MergeConfig(&snapshotterConfig2, &defaultSnapshotterConfig)
+	A.NoError(err)
+	A.Equal(snapshotterConfig2.Root, "/snapshotter/root")
+	A.Equal(snapshotterConfig2.LoggingConfig.LogDir, "")
+	A.Equal(snapshotterConfig2.CacheManagerConfig.CacheDir, "")
+}
+
+func TestProcessConfigurations(t *testing.T) {
+	A := assert.New(t)
+	var defaultSnapshotterConfig SnapshotterConfig
+	var snapshotterConfig1 SnapshotterConfig
+
+	err := defaultSnapshotterConfig.FillUpWithDefaults()
+	A.NoError(err)
+	err = MergeConfig(&snapshotterConfig1, &defaultSnapshotterConfig)
+	A.NoError(err)
+	err = ValidateConfig(&snapshotterConfig1)
+	A.NoError(err)
+
+	err = ProcessConfigurations(&snapshotterConfig1)
+	A.NoError(err)
+
+	A.Equal(snapshotterConfig1.LoggingConfig.LogDir, filepath.Join(snapshotterConfig1.Root, "logs"))
+	A.Equal(snapshotterConfig1.CacheManagerConfig.CacheDir, filepath.Join(snapshotterConfig1.Root, "cache"))
+
+	var snapshotterConfig2 SnapshotterConfig
+	snapshotterConfig2.Root = "/snapshotter/root"
+
+	err = MergeConfig(&snapshotterConfig2, &defaultSnapshotterConfig)
+	A.NoError(err)
+	err = ValidateConfig(&snapshotterConfig2)
+	A.NoError(err)
+
+	err = ProcessConfigurations(&snapshotterConfig2)
+	A.NoError(err)
+
+	A.Equal(snapshotterConfig2.LoggingConfig.LogDir, filepath.Join(snapshotterConfig2.Root, "logs"))
+	A.Equal(snapshotterConfig2.CacheManagerConfig.CacheDir, filepath.Join(snapshotterConfig2.Root, "cache"))
+
+	var snapshotterConfig3 SnapshotterConfig
+	snapshotterConfig3.Root = "./snapshotter/root"
+
+	err = MergeConfig(&snapshotterConfig3, &defaultSnapshotterConfig)
+	A.NoError(err)
+	err = ValidateConfig(&snapshotterConfig3)
+	A.NoError(err)
+
+	err = ProcessConfigurations(&snapshotterConfig3)
+	A.NoError(err)
+}

config/daemonconfig.go

@@ -1,184 +0,0 @@
-/*
- * Copyright (c) 2020. Ant Group. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package config
-
-import (
-	"encoding/json"
-	"io/ioutil"
-
-	"github.com/pkg/errors"
-
-	"github.com/containerd/nydus-snapshotter/pkg/auth"
-	"github.com/containerd/nydus-snapshotter/pkg/utils/erofs"
-	"github.com/containerd/nydus-snapshotter/pkg/utils/registry"
-)
-
-const (
-	backendTypeLocalfs  = "localfs"
-	backendTypeOss      = "oss"
-	backendTypeRegistry = "registry"
-)
-
-type FSPrefetch struct {
-	Enable        bool `json:"enable"`
-	PrefetchAll   bool `json:"prefetch_all"`
-	ThreadsCount  int  `json:"threads_count"`
-	MergingSize   int  `json:"merging_size"`
-	BandwidthRate int  `json:"bandwidth_rate"`
-}
-
-type FscacheDaemonConfig struct {
-	// These fields is only for fscache daemon.
-	Type     string `json:"type"`
-	ID       string `json:"id"`
-	DomainID string `json:"domain_id"`
-	Config   struct {
-		ID            string        `json:"id"`
-		BackendType   string        `json:"backend_type"`
-		BackendConfig BackendConfig `json:"backend_config"`
-		CacheType     string        `json:"cache_type"`
-		CacheConfig   struct {
-			WorkDir string `json:"work_dir"`
-		} `json:"cache_config"`
-		MetadataPath string `json:"metadata_path"`
-	} `json:"config"`
-	FSPrefetch `json:"fs_prefetch,omitempty"`
-}
-
-type DaemonConfig struct {
-	Device         DeviceConfig `json:"device"`
-	Mode           string       `json:"mode"`
-	DigestValidate bool         `json:"digest_validate"`
-	IOStatsFiles   bool         `json:"iostats_files,omitempty"`
-	EnableXattr    bool         `json:"enable_xattr,omitempty"`
-
-	FSPrefetch `json:"fs_prefetch,omitempty"`
-	FscacheDaemonConfig
-}
-
-type BackendConfig struct {
-	// Localfs backend configs
-	BlobFile     string `json:"blob_file,omitempty"`
-	Dir          string `json:"dir,omitempty"`
-	ReadAhead    bool   `json:"readahead"`
-	ReadAheadSec int    `json:"readahead_sec,omitempty"`
-
-	// Registry backend configs
-	Host               string `json:"host,omitempty"`
-	Repo               string `json:"repo,omitempty"`
-	Auth               string `json:"auth,omitempty"`
-	RegistryToken      string `json:"registry_token,omitempty"`
-	BlobURLScheme      string `json:"blob_url_scheme,omitempty"`
-	BlobRedirectedHost string `json:"blob_redirected_host,omitempty"`
-
-	// OSS backend configs
-	EndPoint        string `json:"endpoint,omitempty"`
-	AccessKeyID     string `json:"access_key_id,omitempty"`
-	AccessKeySecret string `json:"access_key_secret,omitempty"`
-	BucketName      string `json:"bucket_name,omitempty"`
-	ObjectPrefix    string `json:"object_prefix,omitempty"`
-
-	// Shared by registry and oss backend
-	Scheme     string `json:"scheme,omitempty"`
-	SkipVerify bool   `json:"skip_verify,omitempty"`
-
-	// Below configs are common configs shared by all backends
-	Proxy struct {
-		URL           string `json:"url,omitempty"`
-		Fallback      bool   `json:"fallback"`
-		PingURL       string `json:"ping_url,omitempty"`
-		CheckInterval int    `json:"check_interval,omitempty"`
-	} `json:"proxy,omitempty"`
-	Timeout        int `json:"timeout,omitempty"`
-	ConnectTimeout int `json:"connect_timeout,omitempty"`
-	RetryLimit     int `json:"retry_limit,omitempty"`
-}
-
-type DeviceConfig struct {
-	Backend struct {
-		BackendType string        `json:"type"`
-		Config      BackendConfig `json:"config"`
-	} `json:"backend"`
-	Cache struct {
-		CacheType  string `json:"type"`
-		Compressed bool   `json:"compressed,omitempty"`
-		Config     struct {
-			WorkDir           string `json:"work_dir"`
-			DisableIndexedMap bool   `json:"disable_indexed_map"`
-		} `json:"config"`
-	} `json:"cache"`
-}
-
-func LoadConfig(configFile string, cfg *DaemonConfig) error {
-	b, err := ioutil.ReadFile(configFile)
-	if err != nil {
-		return err
-	}
-	if err := json.Unmarshal(b, cfg); err != nil {
-		return err
-	}
-	return nil
-}
-
-func SaveConfig(c interface{}, configFile string) error {
-	b, err := json.Marshal(c)
-	if err != nil {
-		return nil
-	}
-	return ioutil.WriteFile(configFile, b, 0755)
-}
-
-func NewDaemonConfig(fsDriver string, cfg DaemonConfig, imageID, snapshotID string, vpcRegistry bool, labels map[string]string) (DaemonConfig, error) {
-	image, err := registry.ParseImage(imageID)
-	if err != nil {
-		return DaemonConfig{}, errors.Wrapf(err, "failed to parse image %s", imageID)
-	}
-
-	backend := cfg.Device.Backend.BackendType
-	if fsDriver == FsDriverFscache {
-		backend = cfg.Config.BackendType
-	}
-
-	switch backend {
-	case backendTypeRegistry:
-		registryHost := image.Host
-		if vpcRegistry {
-			registryHost = registry.ConvertToVPCHost(registryHost)
-		} else if registryHost == "docker.io" {
-			// For docker.io images, we should use index.docker.io
-			registryHost = "index.docker.io"
-		}
-		keyChain := auth.GetRegistryKeyChain(registryHost, labels)
-		// If no auth is provided, don't touch auth from provided nydusd configuration file.
-		// We don't validate the original nydusd auth from configuration file since it can be empty
-		// when repository is public.
-		backendConfig := &cfg.Device.Backend.Config
-		if fsDriver == FsDriverFscache {
-			backendConfig = &cfg.Config.BackendConfig
-			fscacheID := erofs.FscacheID(snapshotID)
-			cfg.ID = fscacheID
-			cfg.DomainID = fscacheID
-			cfg.Config.ID = fscacheID
-		}
-		if keyChain != nil {
-			if keyChain.TokenBase() {
-				backendConfig.RegistryToken = keyChain.Password
-			} else {
-				backendConfig.Auth = keyChain.ToBase64()
-			}
-		}
-		backendConfig.Host = registryHost
-		backendConfig.Repo = image.Repo
-	// Localfs and OSS backends don't need any update, just use the provided config in template
-	case backendTypeLocalfs:
-	case backendTypeOss:
-	default:
-		return DaemonConfig{}, errors.Errorf("unknown backend type %s", backend)
-	}
-
-	return cfg, nil
-}

config/daemonconfig/daemonconfig.go

@@ -0,0 +1,236 @@
+/*
+ * Copyright (c) 2020. Ant Group. All rights reserved.
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package daemonconfig
+
+import (
+	"encoding/json"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/containerd/nydus-snapshotter/config"
+	"github.com/containerd/nydus-snapshotter/pkg/auth"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/registry"
+)
+
+type StorageBackendType = string
+
+const (
+	backendTypeLocalfs  StorageBackendType = "localfs"
+	backendTypeOss      StorageBackendType = "oss"
+	backendTypeRegistry StorageBackendType = "registry"
+)
+
+type DaemonConfig interface {
+	// Provide stuffs relevant to accessing registry apart from auth
+	Supplement(host, repo, snapshotID string, params map[string]string)
+	// Provide auth
+	FillAuth(kc *auth.PassKeyChain)
+	StorageBackend() (StorageBackendType, *BackendConfig)
+	UpdateMirrors(mirrorsConfigDir, registryHost string) error
+	DumpString() (string, error)
+	DumpFile(path string) error
+}
+
+// Daemon configurations factory
+func NewDaemonConfig(fsDriver, path string) (DaemonConfig, error) {
+	switch fsDriver {
+	case config.FsDriverFscache:
+		cfg, err := LoadFscacheConfig(path)
+		if err != nil {
+			return nil, err
+		}
+		return cfg, nil
+	case config.FsDriverFusedev:
+		cfg, err := LoadFuseConfig(path)
+		if err != nil {
+			return nil, err
+		}
+		return cfg, nil
+	default:
+		return nil, errors.Errorf("unsupported, fs driver %q", fsDriver)
+	}
+}
+
+type MirrorConfig struct {
+	Host                string            `json:"host,omitempty"`
+	Headers             map[string]string `json:"headers,omitempty"`
+	HealthCheckInterval int               `json:"health_check_interval,omitempty"`
+	FailureLimit        uint8             `json:"failure_limit,omitempty"`
+	PingURL             string            `json:"ping_url,omitempty"`
+}
+
+type BackendConfig struct {
+	// Localfs backend configs
+	BlobFile     string `json:"blob_file,omitempty"`
+	Dir          string `json:"dir,omitempty"`
+	ReadAhead    bool   `json:"readahead"`
+	ReadAheadSec int    `json:"readahead_sec,omitempty"`
+
+	// Registry backend configs
+	Host               string         `json:"host,omitempty"`
+	Repo               string         `json:"repo,omitempty"`
+	Auth               string         `json:"auth,omitempty" secret:"true"`
+	RegistryToken      string         `json:"registry_token,omitempty" secret:"true"`
+	BlobURLScheme      string         `json:"blob_url_scheme,omitempty"`
+	BlobRedirectedHost string         `json:"blob_redirected_host,omitempty"`
+	Mirrors            []MirrorConfig `json:"mirrors,omitempty"`
+
+	// OSS backend configs
+	EndPoint        string `json:"endpoint,omitempty"`
+	AccessKeyID     string `json:"access_key_id,omitempty" secret:"true"`
+	AccessKeySecret string `json:"access_key_secret,omitempty" secret:"true"`
+	BucketName      string `json:"bucket_name,omitempty"`
+	ObjectPrefix    string `json:"object_prefix,omitempty"`
+
+	// Shared by registry and oss backend
+	Scheme     string `json:"scheme,omitempty"`
+	SkipVerify bool   `json:"skip_verify,omitempty"`
+
+	// Below configs are common configs shared by all backends
+	Proxy struct {
+		URL           string `json:"url,omitempty"`
+		Fallback      bool   `json:"fallback"`
+		PingURL       string `json:"ping_url,omitempty"`
+		CheckInterval int    `json:"check_interval,omitempty"`
+		UseHTTP       bool   `json:"use_http,omitempty"`
+	} `json:"proxy,omitempty"`
+	Timeout        int `json:"timeout,omitempty"`
+	ConnectTimeout int `json:"connect_timeout,omitempty"`
+	RetryLimit     int `json:"retry_limit,omitempty"`
+}
+
+type DeviceConfig struct {
+	Backend struct {
+		BackendType string        `json:"type"`
+		Config      BackendConfig `json:"config"`
+	} `json:"backend"`
+	Cache struct {
+		CacheType  string `json:"type"`
+		Compressed bool   `json:"compressed,omitempty"`
+		Config     struct {
+			WorkDir           string `json:"work_dir"`
+			DisableIndexedMap bool   `json:"disable_indexed_map"`
+		} `json:"config"`
+	} `json:"cache"`
+}
+
+// For nydusd as FUSE daemon. Serialize Daemon info and persist to a json file
+// We don't have to persist configuration file for fscache since its configuration
+// is passed through HTTP API.
+func DumpConfigFile(c interface{}, path string) error {
+	if config.IsBackendSourceEnabled() {
+		c = serializeWithSecretFilter(c)
+	}
+	b, err := json.Marshal(c)
+	if err != nil {
+		return errors.Wrapf(err, "marshal config")
+	}
+
+	return os.WriteFile(path, b, 0600)
+}
+
+func DumpConfigString(c interface{}) (string, error) {
+	b, err := json.Marshal(c)
+	return string(b), err
+}
+
+// Achieve a daemon configuration from template or snapshotter's configuration
+func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string,
+	vpcRegistry bool, labels map[string]string, params map[string]string) error {
+
+	image, err := registry.ParseImage(imageID)
+	if err != nil {
+		return errors.Wrapf(err, "parse image %s", imageID)
+	}
+
+	backendType, _ := c.StorageBackend()
+
+	switch backendType {
+	case backendTypeRegistry:
+		registryHost := image.Host
+		if vpcRegistry {
+			registryHost = registry.ConvertToVPCHost(registryHost)
+		} else if registryHost == "docker.io" {
+			// For docker.io images, we should use index.docker.io
+			registryHost = "index.docker.io"
+		}
+
+		if err := c.UpdateMirrors(config.GetMirrorsConfigDir(), registryHost); err != nil {
+			return errors.Wrap(err, "update mirrors config")
+		}
+
+		// If no auth is provided, don't touch auth from provided nydusd configuration file.
+		// We don't validate the original nydusd auth from configuration file since it can be empty
+		// when repository is public.
+		keyChain := auth.GetRegistryKeyChain(registryHost, imageID, labels)
+		c.Supplement(registryHost, image.Repo, snapshotID, params)
+		c.FillAuth(keyChain)
+
+	// Localfs and OSS backends don't need any update,
+	// just use the provided config in template
+	case backendTypeLocalfs:
+	case backendTypeOss:
+	default:
+		return errors.Errorf("unknown backend type %s", backendType)
+	}
+
+	return nil
+}
+
+func serializeWithSecretFilter(obj interface{}) map[string]interface{} {
+	result := make(map[string]interface{})
+	value := reflect.ValueOf(obj)
+	typeOfObj := reflect.TypeOf(obj)
+
+	if value.Kind() == reflect.Ptr {
+		value = value.Elem()
+		typeOfObj = typeOfObj.Elem()
+	}
+
+	for i := 0; i < value.NumField(); i++ {
+		field := value.Field(i)
+		fieldType := typeOfObj.Field(i)
+		secretTag := fieldType.Tag.Get("secret")
+		jsonTags := strings.Split(fieldType.Tag.Get("json"), ",")
+		omitemptyTag := false
+
+		for _, tag := range jsonTags {
+			if tag == "omitempty" {
+				omitemptyTag = true
+				break
+			}
+		}
+
+		if secretTag == "true" {
+			continue
+		}
+
+		if field.Kind() == reflect.Ptr && field.IsNil() {
+			continue
+		}
+
+		if omitemptyTag && reflect.DeepEqual(reflect.Zero(field.Type()).Interface(), field.Interface()) {
+			continue
+		}
+
+		//nolint:exhaustive
+		switch fieldType.Type.Kind() {
+		case reflect.Struct:
+			result[jsonTags[0]] = serializeWithSecretFilter(field.Interface())
+		case reflect.Ptr:
+			result[jsonTags[0]] = serializeWithSecretFilter(field.Elem().Interface())
+		default:
+			result[jsonTags[0]] = field.Interface()
+		}
+	}
+
+	return result
+}

config/daemonconfig/daemonconfig_test.go

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2020. Ant Group. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package daemonconfig
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadConfig(t *testing.T) {
+	buf := []byte(`{
+  "device": {
+    "backend": {
+      "type": "registry",
+      "config": {
+        "skip_verify": true,
+        "host": "acr-nydus-registry-vpc.cn-hangzhou.cr.aliyuncs.com",
+        "repo": "test/myserver",
+        "auth": "",
+        "blob_url_scheme": "http",
+        "proxy": {
+          "url": "http://p2p-proxy:65001",
+          "fallback": true,
+          "ping_url": "http://p2p-proxy:40901/server/ping",
+          "check_interval": 5
+        },
+        "timeout": 5,
+        "connect_timeout": 5,
+        "retry_limit": 0
+      }
+    },
+    "cache": {
+      "type": "blobcache",
+      "config": {
+        "work_dir": "/cache"
+      }
+    }
+  },
+  "mode": "direct",
+  "digest_validate": true,
+  "iostats_files": true,
+  "enable_xattr": true,
+  "amplify_io": 1048576,
+  "fs_prefetch": {
+    "enable": true,
+    "threads_count": 10,
+    "merging_size": 131072
+  }
+}`)
+	var cfg FuseDaemonConfig
+	err := json.Unmarshal(buf, &cfg)
+	require.Nil(t, err)
+	require.Equal(t, cfg.FSPrefetch.Enable, true)
+	require.Equal(t, cfg.FSPrefetch.MergingSize, 131072)
+	require.Equal(t, cfg.FSPrefetch.ThreadsCount, 10)
+	require.Equal(t, cfg.Device.Backend.Config.BlobURLScheme, "http")
+	require.Equal(t, cfg.Device.Backend.Config.SkipVerify, true)
+	require.Equal(t, cfg.Device.Backend.Config.Proxy.CheckInterval, 5)
+}
+
+func TestAmplifyIo(t *testing.T) {
+	// Test non-zero value
+	input1 := []byte(`{"amplify_io": 1048576}`)
+	var cfg1 FuseDaemonConfig
+	err1 := json.Unmarshal(input1, &cfg1)
+	require.Nil(t, err1)
+	require.Equal(t, *cfg1.AmplifyIo, 1048576)
+	output1, _ := json.Marshal(cfg1)
+	require.Contains(t, string(output1), `"amplify_io":1048576`)
+
+	// Test zero value
+	input2 := []byte(`{"amplify_io": 0}`)
+	var cfg2 FuseDaemonConfig
+	err2 := json.Unmarshal(input2, &cfg2)
+	require.Nil(t, err2)
+	require.Equal(t, *cfg2.AmplifyIo, 0)
+	output2, _ := json.Marshal(cfg2)
+	require.Contains(t, string(output2), `"amplify_io":0`)
+
+	// Test nil value
+	input3 := []byte(`{}`)
+	var cfg3 FuseDaemonConfig
+	err3 := json.Unmarshal(input3, &cfg3)
+	require.Nil(t, err3)
+	require.Nil(t, cfg3.AmplifyIo)
+	output3, _ := json.Marshal(cfg3)
+	require.NotContains(t, string(output3), `amplify_io`)
+}
+
+func TestSerializeWithSecretFilter(t *testing.T) {
+	buf := []byte(`{
+  "device": {
+    "backend": {
+      "type": "registry",
+      "config": {
+        "skip_verify": true,
+        "host": "acr-nydus-registry-vpc.cn-hangzhou.cr.aliyuncs.com",
+        "repo": "test/myserver",
+        "auth": "token_token",
+        "blob_url_scheme": "http",
+        "proxy": {
+          "url": "http://p2p-proxy:65001",
+          "fallback": true,
+          "ping_url": "http://p2p-proxy:40901/server/ping",
+          "check_interval": 5
+        },
+        "timeout": 5,
+        "connect_timeout": 5,
+        "retry_limit": 0
+      }
+    },
+    "cache": {
+      "type": "blobcache",
+      "config": {
+        "work_dir": "/cache"
+      }
+    }
+  },
+  "mode": "direct",
+  "digest_validate": true,
+  "iostats_files": true,
+  "enable_xattr": true,
+  "fs_prefetch": {
+    "enable": true,
+    "threads_count": 10,
+    "merging_size": 131072
+  }
+}`)
+	var cfg FuseDaemonConfig
+	_ = json.Unmarshal(buf, &cfg)
+	filter := serializeWithSecretFilter(&cfg)
+	jsonData, err := json.Marshal(filter)
+	require.Nil(t, err)
+	var newCfg FuseDaemonConfig
+	err = json.Unmarshal(jsonData, &newCfg)
+	require.Nil(t, err)
+	require.Equal(t, newCfg.FSPrefetch, cfg.FSPrefetch)
+	require.Equal(t, newCfg.Device.Cache.CacheType, cfg.Device.Cache.CacheType)
+	require.Equal(t, newCfg.Device.Cache.Config, cfg.Device.Cache.Config)
+	require.Equal(t, newCfg.Mode, cfg.Mode)
+	require.Equal(t, newCfg.DigestValidate, cfg.DigestValidate)
+	require.Equal(t, newCfg.IOStatsFiles, cfg.IOStatsFiles)
+	require.Equal(t, newCfg.Device.Backend.Config.Host, cfg.Device.Backend.Config.Host)
+	require.Equal(t, newCfg.Device.Backend.Config.Repo, cfg.Device.Backend.Config.Repo)
+	require.Equal(t, newCfg.Device.Backend.Config.Proxy, cfg.Device.Backend.Config.Proxy)
+	require.Equal(t, newCfg.Device.Backend.Config.BlobURLScheme, cfg.Device.Backend.Config.BlobURLScheme)
+	require.Equal(t, newCfg.Device.Backend.Config.Auth, "")
+	require.NotEqual(t, newCfg.Device.Backend.Config.Auth, cfg.Device.Backend.Config.Auth)
+}

config/daemonconfig/fscache.go

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package daemonconfig
+
+import (
+	"encoding/json"
+	"os"
+	"path"
+
+	"github.com/containerd/log"
+	"github.com/containerd/nydus-snapshotter/pkg/auth"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/erofs"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	WorkDir   string = "workdir"
+	Bootstrap string = "bootstrap"
+)
+
+type BlobPrefetchConfig struct {
+	Enable        bool `json:"enable"`
+	ThreadsCount  int  `json:"threads_count"`
+	MergingSize   int  `json:"merging_size"`
+	BandwidthRate int  `json:"bandwidth_rate"`
+}
+
+type FscacheDaemonConfig struct {
+	// These fields is only for fscache daemon.
+	Type string `json:"type"`
+	// Snapshotter fills
+	ID       string `json:"id"`
+	DomainID string `json:"domain_id"`
+	Config   *struct {
+		ID            string        `json:"id"`
+		BackendType   string        `json:"backend_type"`
+		BackendConfig BackendConfig `json:"backend_config"`
+		CacheType     string        `json:"cache_type"`
+		// Snapshotter fills
+		CacheConfig struct {
+			WorkDir string `json:"work_dir"`
+		} `json:"cache_config"`
+		BlobPrefetchConfig BlobPrefetchConfig `json:"prefetch_config"`
+		MetadataPath       string             `json:"metadata_path"`
+	} `json:"config"`
+}
+
+// Load Fscache configuration template file
+func LoadFscacheConfig(p string) (*FscacheDaemonConfig, error) {
+	var cfg FscacheDaemonConfig
+	b, err := os.ReadFile(p)
+	if err != nil {
+		return nil, errors.Wrapf(err, "read fscache configuration file %s", p)
+	}
+	if err = json.Unmarshal(b, &cfg); err != nil {
+		return nil, errors.Wrapf(err, "unmarshal")
+	}
+
+	if cfg.Config == nil {
+		return nil, errors.New("invalid fscache configuration")
+	}
+
+	return &cfg, nil
+}
+
+func (c *FscacheDaemonConfig) UpdateMirrors(mirrorsConfigDir, registryHost string) error {
+	mirrors, err := LoadMirrorsConfig(mirrorsConfigDir, registryHost)
+	if err != nil {
+		return err
+	}
+	if len(mirrors) > 0 {
+		c.Config.BackendConfig.Mirrors = mirrors
+	}
+	return nil
+}
+
+func (c *FscacheDaemonConfig) StorageBackend() (string, *BackendConfig) {
+	return c.Config.BackendType, &c.Config.BackendConfig
+}
+
+// Each fscache/erofs has a configuration with different fscache ID built from snapshot ID.
+func (c *FscacheDaemonConfig) Supplement(host, repo, snapshotID string, params map[string]string) {
+	c.Config.BackendConfig.Host = host
+	c.Config.BackendConfig.Repo = repo
+
+	fscacheID := erofs.FscacheID(snapshotID)
+	c.ID = fscacheID
+
+	if c.DomainID != "" {
+		log.L.Warnf("Linux Kernel Shared Domain feature in use. make sure your kernel version >= 6.1")
+	} else {
+		c.DomainID = fscacheID
+	}
+
+	c.Config.ID = fscacheID
+
+	if WorkDir, ok := params[WorkDir]; ok {
+		c.Config.CacheConfig.WorkDir = WorkDir
+	}
+
+	if bootstrap, ok := params[Bootstrap]; ok {
+		c.Config.MetadataPath = bootstrap
+	}
+}
+
+func (c *FscacheDaemonConfig) FillAuth(kc *auth.PassKeyChain) {
+	if kc != nil {
+		if kc.TokenBase() {
+			c.Config.BackendConfig.RegistryToken = kc.Password
+		} else {
+			c.Config.BackendConfig.Auth = kc.ToBase64()
+		}
+	}
+}
+
+func (c *FscacheDaemonConfig) DumpString() (string, error) {
+	return DumpConfigString(c)
+}
+
+func (c *FscacheDaemonConfig) DumpFile(f string) error {
+	if err := os.MkdirAll(path.Dir(f), 0755); err != nil {
+		return err
+	}
+
+	return DumpConfigFile(c, f)
+}

config/daemonconfig/fuse.go

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package daemonconfig
+
+import (
+	"encoding/json"
+	"os"
+	"path"
+
+	"github.com/pkg/errors"
+
+	"github.com/containerd/nydus-snapshotter/pkg/auth"
+)
+
+const CacheDir string = "cachedir"
+
+// Used when nydusd works as a FUSE daemon or vhost-user-fs backend
+type FuseDaemonConfig struct {
+	Device          *DeviceConfig `json:"device"`
+	Mode            string        `json:"mode"`
+	DigestValidate  bool          `json:"digest_validate"`
+	IOStatsFiles    bool          `json:"iostats_files,omitempty"`
+	EnableXattr     bool          `json:"enable_xattr,omitempty"`
+	AccessPattern   bool          `json:"access_pattern,omitempty"`
+	LatestReadFiles bool          `json:"latest_read_files,omitempty"`
+	AmplifyIo       *int          `json:"amplify_io,omitempty"`
+	FSPrefetch      `json:"fs_prefetch,omitempty"`
+	// (experimental) The nydus daemon could cache more data to increase hit ratio when enabled the warmup feature.
+	Warmup uint64 `json:"warmup,omitempty"`
+}
+
+// Control how to perform prefetch from file system layer
+type FSPrefetch struct {
+	Enable        bool `json:"enable"`
+	PrefetchAll   bool `json:"prefetch_all"`
+	ThreadsCount  int  `json:"threads_count"`
+	MergingSize   int  `json:"merging_size"`
+	BandwidthRate int  `json:"bandwidth_rate"`
+}
+
+// Load fuse daemon configuration from template file
+func LoadFuseConfig(p string) (*FuseDaemonConfig, error) {
+	b, err := os.ReadFile(p)
+	if err != nil {
+		return nil, errors.Wrapf(err, "read FUSE configuration file %s", p)
+	}
+	var cfg FuseDaemonConfig
+	if err := json.Unmarshal(b, &cfg); err != nil {
+		return nil, errors.Wrapf(err, "unmarshal %s", p)
+	}
+
+	if cfg.Device == nil {
+		return nil, errors.New("invalid fuse daemon configuration")
+	}
+
+	return &cfg, nil
+}
+
+func (c *FuseDaemonConfig) Supplement(host, repo, _ string, params map[string]string) {
+	c.Device.Backend.Config.Host = host
+	c.Device.Backend.Config.Repo = repo
+	c.Device.Cache.Config.WorkDir = params[CacheDir]
+}
+
+func (c *FuseDaemonConfig) FillAuth(kc *auth.PassKeyChain) {
+	if kc != nil {
+		if kc.TokenBase() {
+			c.Device.Backend.Config.RegistryToken = kc.Password
+		} else {
+			c.Device.Backend.Config.Auth = kc.ToBase64()
+		}
+	}
+}
+
+func (c *FuseDaemonConfig) UpdateMirrors(mirrorsConfigDir, registryHost string) error {
+	mirrors, err := LoadMirrorsConfig(mirrorsConfigDir, registryHost)
+	if err != nil {
+		return err
+	}
+	if len(mirrors) > 0 {
+		c.Device.Backend.Config.Mirrors = mirrors
+	}
+	return nil
+}
+
+func (c *FuseDaemonConfig) StorageBackend() (string, *BackendConfig) {
+	return c.Device.Backend.BackendType, &c.Device.Backend.Config
+}
+
+func (c *FuseDaemonConfig) DumpString() (string, error) {
+	return DumpConfigString(c)
+}
+
+func (c *FuseDaemonConfig) DumpFile(f string) error {
+	if err := os.MkdirAll(path.Dir(f), 0755); err != nil {
+		return err
+	}
+	return DumpConfigFile(c, f)
+}

config/daemonconfig/mirrors.go

@@ -0,0 +1,259 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package daemonconfig
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/containerd/log"
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+)
+
+// Copied from containerd, for compatibility with containerd's toml configuration file.
+type HostFileConfig struct {
+	Capabilities []string               `toml:"capabilities"`
+	CACert       interface{}            `toml:"ca"`
+	Client       interface{}            `toml:"client"`
+	SkipVerify   *bool                  `toml:"skip_verify"`
+	Header       map[string]interface{} `toml:"header"`
+	OverridePath bool                   `toml:"override_path"`
+
+	// The following configuration items are specific to nydus.
+	HealthCheckInterval int    `toml:"health_check_interval,omitempty"`
+	FailureLimit        uint8  `toml:"failure_limit,omitempty"`
+	PingURL             string `toml:"ping_url,omitempty"`
+}
+
+type hostConfig struct {
+	Scheme string
+	Host   string
+	Header http.Header
+
+	HealthCheckInterval int
+	FailureLimit        uint8
+	PingURL             string
+}
+
+func makeStringSlice(slice []interface{}, cb func(string) string) ([]string, error) {
+	out := make([]string, len(slice))
+	for i, value := range slice {
+		str, ok := value.(string)
+		if !ok {
+			return nil, fmt.Errorf("unable to cast %v to string", value)
+		}
+
+		if cb != nil {
+			out[i] = cb(str)
+		} else {
+			out[i] = str
+		}
+	}
+	return out, nil
+}
+
+func parseMirrorsConfig(hosts []hostConfig) []MirrorConfig {
+	var parsedMirrors = make([]MirrorConfig, len(hosts))
+
+	for i, host := range hosts {
+		parsedMirrors[i].Host = fmt.Sprintf("%s://%s", host.Scheme, host.Host)
+		parsedMirrors[i].HealthCheckInterval = host.HealthCheckInterval
+		parsedMirrors[i].FailureLimit = host.FailureLimit
+		parsedMirrors[i].PingURL = host.PingURL
+
+		if len(host.Header) > 0 {
+			mirrorHeader := make(map[string]string, len(host.Header))
+			for key, value := range host.Header {
+				if len(value) > 1 {
+					log.L.Warnf("some values of the header[%q] are omitted: %#v", key, value[1:])
+				}
+				mirrorHeader[key] = host.Header.Get(key)
+			}
+			parsedMirrors[i].Headers = mirrorHeader
+		}
+	}
+
+	return parsedMirrors
+}
+
+// hostDirectory converts ":port" to "_port_" in directory names
+func hostDirectory(host string) string {
+	idx := strings.LastIndex(host, ":")
+	if idx > 0 {
+		return host[:idx] + "_" + host[idx+1:] + "_"
+	}
+	return host
+}
+
+func hostPaths(root, host string) []string {
+	var hosts []string
+	ch := hostDirectory(host)
+	if ch != host {
+		hosts = append(hosts, filepath.Join(root, ch))
+	}
+	return append(hosts,
+		filepath.Join(root, host),
+		filepath.Join(root, "_default"),
+	)
+}
+
+func hostDirFromRoot(root, host string) (string, error) {
+	for _, p := range hostPaths(root, host) {
+		if _, err := os.Stat(p); err == nil {
+			return p, nil
+		} else if !os.IsNotExist(err) {
+			return "", err
+		}
+	}
+	return "", nil
+}
+
+// getSortedHosts returns the list of hosts as they defined in the file.
+func getSortedHosts(root *toml.Tree) ([]string, error) {
+	iter, ok := root.Get("host").(*toml.Tree)
+	if !ok {
+		return nil, errors.New("invalid `host` tree")
+	}
+
+	list := append([]string{}, iter.Keys()...)
+
+	// go-toml stores TOML sections in the map object, so no order guaranteed.
+	// We retrieve line number for each key and sort the keys by position.
+	sort.Slice(list, func(i, j int) bool {
+		h1 := iter.GetPath([]string{list[i]}).(*toml.Tree)
+		h2 := iter.GetPath([]string{list[j]}).(*toml.Tree)
+		return h1.Position().Line < h2.Position().Line
+	})
+
+	return list, nil
+}
+
+// parseHostConfig returns the parsed host configuration, make sure the server is not null.
+func parseHostConfig(server string, config HostFileConfig) (hostConfig, error) {
+	var (
+		result = hostConfig{}
+		err    error
+	)
+
+	if !strings.HasPrefix(server, "http") {
+		server = "https://" + server
+	}
+	u, err := url.Parse(server)
+	if err != nil {
+		return hostConfig{}, fmt.Errorf("unable to parse server %v: %w", server, err)
+	}
+	result.Scheme = u.Scheme
+	result.Host = u.Host
+
+	if config.Header != nil {
+		header := http.Header{}
+		for key, ty := range config.Header {
+			switch value := ty.(type) {
+			case string:
+				header[key] = []string{value}
+			case []interface{}:
+				header[key], err = makeStringSlice(value, nil)
+				if err != nil {
+					return hostConfig{}, err
+				}
+			default:
+				return hostConfig{}, fmt.Errorf("invalid type %v for header %q", ty, key)
+			}
+		}
+		result.Header = header
+	}
+
+	result.HealthCheckInterval = config.HealthCheckInterval
+	result.FailureLimit = config.FailureLimit
+	result.PingURL = config.PingURL
+
+	return result, nil
+}
+
+func parseHostsFile(b []byte) ([]hostConfig, error) {
+	tree, err := toml.LoadBytes(b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse TOML: %w", err)
+	}
+	c := struct {
+		// HostConfigs store the per-host configuration
+		HostConfigs map[string]HostFileConfig `toml:"host"`
+	}{}
+
+	orderedHosts, err := getSortedHosts(tree)
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		hosts []hostConfig
+	)
+
+	if err := tree.Unmarshal(&c); err != nil {
+		return nil, err
+	}
+
+	// Parse hosts array
+	for _, host := range orderedHosts {
+		if host != "" {
+			config := c.HostConfigs[host]
+			parsed, err := parseHostConfig(host, config)
+			if err != nil {
+				return nil, err
+			}
+			hosts = append(hosts, parsed)
+		}
+	}
+
+	return hosts, nil
+}
+
+func loadHostDir(hostsDir string) ([]hostConfig, error) {
+	b, err := os.ReadFile(filepath.Join(hostsDir, "hosts.toml"))
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+		return []hostConfig{}, nil
+	}
+
+	hosts, err := parseHostsFile(b)
+	if err != nil {
+		return nil, err
+	}
+
+	return hosts, nil
+}
+
+func LoadMirrorsConfig(mirrorsConfigDir, registryHost string) ([]MirrorConfig, error) {
+	var mirrors []MirrorConfig
+
+	if mirrorsConfigDir == "" {
+		return mirrors, nil
+	}
+	hostDir, err := hostDirFromRoot(mirrorsConfigDir, registryHost)
+	if err != nil {
+		return nil, err
+	}
+	if hostDir == "" {
+		return mirrors, nil
+	}
+
+	hostConfig, err := loadHostDir(hostDir)
+	if err != nil {
+		return nil, err
+	}
+	mirrors = append(mirrors, parseMirrorsConfig(hostConfig)...)
+
+	return mirrors, nil
+}

config/daemonconfig/mirrors_test.go

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package daemonconfig
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadMirrorConfig(t *testing.T) {
+	tmpDir := t.TempDir()
+	defer os.RemoveAll(tmpDir)
+
+	registryHost := "registry.docker.io"
+
+	mirrorsConfigDir := filepath.Join(tmpDir, "certs.d")
+	registryHostConfigDir := filepath.Join(mirrorsConfigDir, registryHost)
+	defaultHostConfigDir := filepath.Join(mirrorsConfigDir, "_default")
+
+	mirrors, err := LoadMirrorsConfig("", registryHost)
+	require.NoError(t, err)
+	require.Nil(t, mirrors)
+
+	mirrors, err = LoadMirrorsConfig(mirrorsConfigDir, registryHost)
+	require.NoError(t, err)
+	require.Nil(t, mirrors)
+
+	err = os.MkdirAll(defaultHostConfigDir, os.ModePerm)
+	assert.NoError(t, err)
+
+	mirrors, err = LoadMirrorsConfig(mirrorsConfigDir, registryHost)
+	require.NoError(t, err)
+	require.Nil(t, mirrors)
+
+	buf1 := []byte(`server = "https://default-docker.hub.com"
+	[host]
+	  [host."http://default-p2p-mirror1:65001"]
+		[host."http://default-p2p-mirror1:65001".header]
+		  X-Dragonfly-Registry = ["https://default-docker.hub.com"]
+	`)
+	err = os.WriteFile(filepath.Join(defaultHostConfigDir, "hosts.toml"), buf1, 0600)
+	assert.NoError(t, err)
+	mirrors, err = LoadMirrorsConfig(mirrorsConfigDir, registryHost)
+	require.NoError(t, err)
+	require.Equal(t, len(mirrors), 1)
+	require.Equal(t, mirrors[0].Host, "http://default-p2p-mirror1:65001")
+	require.Equal(t, mirrors[0].Headers["X-Dragonfly-Registry"], "https://default-docker.hub.com")
+
+	err = os.MkdirAll(registryHostConfigDir, os.ModePerm)
+	assert.NoError(t, err)
+
+	buf2 := []byte(`server = "https://docker.hub.com"
+	[host]
+	  [host."http://p2p-mirror1:65001"]
+		[host."http://p2p-mirror1:65001".header]
+		  X-Dragonfly-Registry = ["https://docker.hub.com"]
+	`)
+	err = os.WriteFile(filepath.Join(registryHostConfigDir, "hosts.toml"), buf2, 0600)
+	assert.NoError(t, err)
+	mirrors, err = LoadMirrorsConfig(mirrorsConfigDir, registryHost)
+	require.NoError(t, err)
+	require.Equal(t, len(mirrors), 1)
+	require.Equal(t, mirrors[0].Host, "http://p2p-mirror1:65001")
+	require.Equal(t, mirrors[0].Headers["X-Dragonfly-Registry"], "https://docker.hub.com")
+
+	buf3 := []byte(`
+		[host."http://p2p-mirror2:65001"]
+		[host."http://p2p-mirror2:65001".header]
+			X-Dragonfly-Registry = ["https://docker.hub.com"]
+	`)
+	err = os.WriteFile(filepath.Join(registryHostConfigDir, "hosts.toml"), buf3, 0600)
+	assert.NoError(t, err)
+	mirrors, err = LoadMirrorsConfig(mirrorsConfigDir, registryHost)
+	require.NoError(t, err)
+	require.Equal(t, len(mirrors), 1)
+	require.Equal(t, mirrors[0].Host, "http://p2p-mirror2:65001")
+	require.Equal(t, mirrors[0].Headers["X-Dragonfly-Registry"], "https://docker.hub.com")
+}

config/daemonconfig_test.go

@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2020. Ant Group. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package config
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestLoadConfig(t *testing.T) {
-	buf := []byte(`{
-  "device": {
-    "backend": {
-      "type": "registry",
-      "config": {
-        "scheme": "https",
-        "skip_verify": true,
-        "host": "acr-nydus-registry-vpc.cn-hangzhou.cr.aliyuncs.com",
-        "repo": "test/myserver",
-        "auth": "",
-        "blob_url_scheme": "http",
-        "proxy": {
-          "url": "http://p2p-proxy:65001",
-          "fallback": true,
-          "ping_url": "http://p2p-proxy:40901/server/ping",
-          "check_interval": 5
-        },
-        "timeout": 5,
-        "connect_timeout": 5,
-        "retry_limit": 0
-      }
-    },
-    "cache": {
-      "type": "blobcache",
-      "config": {
-        "work_dir": "/cache"
-      }
-    }
-  },
-  "mode": "direct",
-  "digest_validate": true,
-  "iostats_files": true,
-  "enable_xattr": true,
-  "fs_prefetch": {
-    "enable": true,
-    "threads_count": 10,
-    "merging_size": 131072
-  }
-}`)
-	var cfg DaemonConfig
-	err := json.Unmarshal(buf, &cfg)
-	require.Nil(t, err)
-	require.Equal(t, cfg.FSPrefetch.Enable, true)
-	require.Equal(t, cfg.FSPrefetch.MergingSize, 131072)
-	require.Equal(t, cfg.FSPrefetch.ThreadsCount, 10)
-	require.Equal(t, cfg.Device.Backend.Config.BlobURLScheme, "http")
-	require.Equal(t, cfg.Device.Backend.Config.SkipVerify, true)
-	require.Equal(t, cfg.Device.Backend.Config.Proxy.CheckInterval, 5)
-}

config/default.go

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package config
+
+import (
+	"os/exec"
+
+	"github.com/containerd/nydus-snapshotter/internal/constant"
+)
+
+func (c *SnapshotterConfig) FillUpWithDefaults() error {
+	c.Version = 1
+	c.Root = constant.DefaultRootDir
+	c.Address = constant.DefaultAddress
+
+	// essential configuration
+	if c.DaemonMode == "" {
+		c.DaemonMode = constant.DefaultDaemonMode
+	}
+
+	// system controller configuration
+	c.SystemControllerConfig.Address = constant.DefaultSystemControllerAddress
+
+	// logging configuration
+	logConfig := &c.LoggingConfig
+	if logConfig.LogLevel == "" {
+		logConfig.LogLevel = constant.DefaultLogLevel
+	}
+	logConfig.RotateLogMaxSize = constant.DefaultRotateLogMaxSize
+	logConfig.RotateLogMaxBackups = constant.DefaultRotateLogMaxBackups
+	logConfig.RotateLogMaxAge = constant.DefaultRotateLogMaxAge
+	logConfig.RotateLogLocalTime = constant.DefaultRotateLogLocalTime
+	logConfig.RotateLogCompress = constant.DefaultRotateLogCompress
+
+	// daemon configuration
+	daemonConfig := &c.DaemonConfig
+	if daemonConfig.NydusdConfigPath == "" {
+		daemonConfig.NydusdConfigPath = constant.DefaultNydusDaemonConfigPath
+	}
+	daemonConfig.RecoverPolicy = RecoverPolicyRestart.String()
+	daemonConfig.FsDriver = constant.DefaultFsDriver
+	daemonConfig.LogRotationSize = constant.DefaultDaemonRotateLogMaxSize
+
+	// cache configuration
+	cacheConfig := &c.CacheManagerConfig
+	if cacheConfig.GCPeriod == "" {
+		cacheConfig.GCPeriod = constant.DefaultGCPeriod
+	}
+
+	return c.SetupNydusBinaryPaths()
+}
+
+func (c *SnapshotterConfig) SetupNydusBinaryPaths() error {
+	// resolve nydusd path
+	if path, err := exec.LookPath(constant.NydusdBinaryName); err == nil {
+		c.DaemonConfig.NydusdPath = path
+	}
+
+	// resolve nydus-image path
+	if path, err := exec.LookPath(constant.NydusImageBinaryName); err == nil {
+		c.DaemonConfig.NydusImagePath = path
+	}
+
+	return nil
+}

config/global.go

@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// Expose configurations across nydus-snapshotter, the configurations is parsed
+// and extracted from nydus-snapshotter toml based configuration file or command line
+
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/containerd/log"
+	"github.com/pkg/errors"
+
+	"github.com/containerd/nydus-snapshotter/internal/logging"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/mount"
+)
+
+var (
+	globalConfig GlobalConfig
+)
+
+// Global cached configuration information to help:
+// - access configuration information without passing a configuration object
+// - avoid frequent generation of information from configuration information
+type GlobalConfig struct {
+	origin           *SnapshotterConfig
+	SnapshotsDir     string
+	DaemonMode       DaemonMode
+	SocketRoot       string
+	ConfigRoot       string
+	RootMountpoint   string
+	DaemonThreadsNum int
+	CacheGCPeriod    time.Duration
+	MirrorsConfig    MirrorsConfig
+}
+
+func IsFusedevSharedModeEnabled() bool {
+	return globalConfig.DaemonMode == DaemonModeShared
+}
+
+func GetDaemonMode() DaemonMode {
+	return globalConfig.DaemonMode
+}
+
+func GetSnapshotsRootDir() string {
+	return globalConfig.SnapshotsDir
+}
+
+func GetRootMountpoint() string {
+	return globalConfig.RootMountpoint
+}
+
+func GetSocketRoot() string {
+	return globalConfig.SocketRoot
+}
+
+func GetConfigRoot() string {
+	return globalConfig.ConfigRoot
+}
+
+func GetMirrorsConfigDir() string {
+	return globalConfig.MirrorsConfig.Dir
+}
+
+func GetFsDriver() string {
+	return globalConfig.origin.DaemonConfig.FsDriver
+}
+
+func GetCacheGCPeriod() time.Duration {
+	return globalConfig.CacheGCPeriod
+}
+
+func GetLogDir() string {
+	return globalConfig.origin.LoggingConfig.LogDir
+}
+
+func GetLogLevel() string {
+	return globalConfig.origin.LoggingConfig.LogLevel
+}
+
+func GetDaemonLogRotationSize() int {
+	return globalConfig.origin.DaemonConfig.LogRotationSize
+}
+
+func GetDaemonThreadsNumber() int {
+	return globalConfig.origin.DaemonConfig.ThreadsNumber
+}
+
+func GetLogToStdout() bool {
+	return globalConfig.origin.LoggingConfig.LogToStdout
+}
+
+func IsBackendSourceEnabled() bool {
+	return globalConfig.origin.Experimental.EnableBackendSource && globalConfig.origin.SystemControllerConfig.Enable
+}
+
+func IsSystemControllerEnabled() bool {
+	return globalConfig.origin.SystemControllerConfig.Enable
+}
+
+func SystemControllerAddress() string {
+	return globalConfig.origin.SystemControllerConfig.Address
+}
+
+func SystemControllerPprofAddress() string {
+	return globalConfig.origin.SystemControllerConfig.DebugConfig.PprofAddress
+}
+
+func GetDaemonProfileCPUDuration() int64 {
+	return globalConfig.origin.SystemControllerConfig.DebugConfig.ProfileDuration
+}
+
+func GetSkipSSLVerify() bool {
+	return globalConfig.origin.RemoteConfig.SkipSSLVerify
+}
+
+const (
+	TarfsLayerVerityOnly      string = "layer_verity_only"
+	TarfsImageVerityOnly      string = "image_verity_only"
+	TarfsLayerBlockDevice     string = "layer_block"
+	TarfsImageBlockDevice     string = "image_block"
+	TarfsLayerBlockWithVerity string = "layer_block_with_verity"
+	TarfsImageBlockWithVerity string = "image_block_with_verity"
+)
+
+func GetTarfsMountOnHost() bool {
+	return globalConfig.origin.Experimental.TarfsConfig.MountTarfsOnHost
+}
+
+func GetTarfsExportEnabled() bool {
+	switch globalConfig.origin.Experimental.TarfsConfig.ExportMode {
+	case TarfsLayerVerityOnly, TarfsLayerBlockDevice, TarfsLayerBlockWithVerity:
+		return true
+	case TarfsImageVerityOnly, TarfsImageBlockDevice, TarfsImageBlockWithVerity:
+		return true
+	default:
+		return false
+	}
+}
+
+// Returns (wholeImage, generateBlockImage, withVerityInfo)
+// wholeImage: generate tarfs for the whole image instead of of a specific layer.
+// generateBlockImage: generate a block image file.
+// withVerityInfo: generate disk verity information.
+func GetTarfsExportFlags() (bool, bool, bool) {
+	switch globalConfig.origin.Experimental.TarfsConfig.ExportMode {
+	case "layer_verity_only":
+		return false, false, true
+	case "image_verity_only":
+		return true, false, true
+	case "layer_block":
+		return false, true, false
+	case "image_block":
+		return true, true, false
+	case "layer_block_with_verity":
+		return false, true, true
+	case "image_block_with_verity":
+		return true, true, true
+	default:
+		return false, false, false
+	}
+}
+
+func ProcessConfigurations(c *SnapshotterConfig) error {
+	if c.LoggingConfig.LogDir == "" {
+		c.LoggingConfig.LogDir = filepath.Join(c.Root, logging.DefaultLogDirName)
+	}
+	if c.CacheManagerConfig.CacheDir == "" {
+		c.CacheManagerConfig.CacheDir = filepath.Join(c.Root, "cache")
+	}
+
+	globalConfig.origin = c
+
+	globalConfig.SnapshotsDir = filepath.Join(c.Root, "snapshots")
+	globalConfig.ConfigRoot = filepath.Join(c.Root, "config")
+	globalConfig.SocketRoot = filepath.Join(c.Root, "socket")
+	globalConfig.RootMountpoint = filepath.Join(c.Root, "mnt")
+
+	globalConfig.MirrorsConfig = c.RemoteConfig.MirrorsConfig
+
+	if c.CacheManagerConfig.GCPeriod != "" {
+		d, err := time.ParseDuration(c.CacheManagerConfig.GCPeriod)
+		if err != nil {
+			return errors.Errorf("invalid GC period '%s'", c.CacheManagerConfig.GCPeriod)
+		}
+		globalConfig.CacheGCPeriod = d
+	}
+
+	m, err := parseDaemonMode(c.DaemonMode)
+	if err != nil {
+		return err
+	}
+
+	if c.DaemonConfig.FsDriver == FsDriverFscache && m != DaemonModeShared {
+		log.L.Infof("fscache driver only supports 'shared' mode, override daemon mode from '%s' to 'shared'", m)
+		m = DaemonModeShared
+	}
+
+	globalConfig.DaemonMode = m
+
+	return nil
+}
+
+func SetUpEnvironment(c *SnapshotterConfig) error {
+	if err := os.MkdirAll(c.Root, 0700); err != nil {
+		return errors.Wrapf(err, "create root dir %s", c.Root)
+	}
+
+	realPath, err := mount.NormalizePath(c.Root)
+	if err != nil {
+		return errors.Wrapf(err, "invalid root path")
+	}
+	c.Root = realPath
+	return nil
+}

docs/configure_nydus.md

@@ -1,37 +1,139 @@
-# How to Configure Nydus
+# Configure Nydus-snapshotter
 
-Nydus-snapshotter receives a json file as nydusd configuration through CLI option `--config-path`.
-An example configuration looks like below.
-You can find registry auth from local docker configuration.
+Nydus-snapshotter can receive a toml file as its configurations to start providing image service through CLI parameter `--config`. An example configuration file can be found [here](../misc/snapshotter/config.toml). Besides nydus-snapshotter's configuration, `nydusd`'s configuration has to be provided to nydus-snapshotter too. Nydusd is started by nydus-snapshotter and it is configured by the provided json configuration file. A minimal configuration file can be found [here](../misc/snapshotter/nydusd-config.fusedev.json)
 
-```json
-{
-  "device": {
-    "backend": {
-      "type": "registry",
-      "config": {
-        "scheme": "https",
-        "auth": "<registry auth token>",
-        "timeout": 5,
-        "connect_timeout": 5,
-        "retry_limit": 0
-      }
-    },
-    "cache": {
-      "type": "blobcache",
-      "config": {
-        "work_dir": "/tmp/cache"
-      }
-    }
-  },
-  "mode": "direct",
-  "digest_validate": false,
-  "iostats_files": false,
-  "enable_xattr": true,
-  "fs_prefetch": {
-    "enable": true,
-    "threads_count": 6,
-    "merging_size": 131072
-  }
-}
+## Authentication
+
+As [containerd#3731](https://github.com/containerd/containerd/issues/3731) discussed, containerd doesn't share credentials with third snapshotters now. Like [stargz snapshotter](https://github.com/containerd/stargz-snapshotter/blob/main/docs/overview.md#authentication), nydus-snapshotter supports 3 main ways to access registries with custom configurations. You can use configuration file to enable them.
+
+The snapshotter will try to get image pull keychain in the following order if such way is enabled:
+
+1. intercept CRI request and extract private registry auth
+2. docker config (default enabled)
+3. k8s docker config secret
+
+### dockerconfig-based authentication
+
+By default, the snapshotter tries to get credentials from `$DOCKER_CONFIG` or `~/.docker/config.json`.
+Following example enables nydus-snapshotter to access to private registries using `docker login` command.
+
+```console
+# docker login
+(Enter username and password)
+# crictl pull --creds USERNAME[:PASSWORD] docker.io/<your-repository>/ubuntu:22.04
+(Here the credential is only used by containerd)
 ```
+
+### CRI-based authentication
+
+Following configuration enables nydus-snapshotter to pull private images via CRI requests.
+
+```toml
+
+[remote.auth]
+# Fetch the private registry auth as CRI image service proxy
+enable_cri_keychain = true
+image_service_address = "/run/containerd/containerd.sock"
+```
+
+The snapshotter works as a proxy of CRI Image Service and exposes CRI Image Service API on the snapshotter's unix socket (i.e. `/run/containerd/containerd-nydus-grpc.sock`). The snapshotter acquires registry creds by scanning requests.
+The `image_service_address` is the original image service socket. It can be omitted and the default value will be the containerd's default socket path `/run/containerd/containerd.sock`.
+
+You **must** specify `--image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock` option to kubelet when using Kubernetes. Or specify `image-endpoint: "unix:////run/containerd-nydus/containerd-nydus-grpc.sock"` in `crictl.yaml` when using `crictl`.
+
+### kubeconfig-based authentication
+
+This is another way to enable lazy pulling of private images on Kubernetes, Nydus snapshotter will start a goroutine to listen on secrets (type = `kubernetes.io/dockerconfigjson`) for private registries.
+
+#### use kubeconfig file
+
+Following configuration enables nydus-snapshotter to access to private registries using kubernetes secrets in the cluster using kubeconfig files.
+
+```toml
+[remote.auth]
+enable_kubeconfig_keychain = true
+kubeconfig_path = "/path/to/.kubeconfig"
+```
+
+If no `kubeconfig_path` is specified, snapshotter searches kubeconfig files from `$KUBECONFIG` or `~/.kube/config`.
+
+Please note that kubeconfig-based authentication requires additional privilege (i.e. kubeconfig to list/watch secrets) to the node.
+And this doesn't work if `kubelet` retrieve credentials from somewhere not API server (e.g. [credential provider](https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider/)).
+
+#### use ServiceAccount
+
+If your Nydus snapshotter runs in a Kubernetes cluster and you don't want to use kubeconfig, you can also choose to use ServiceAccount to configure the corresponding permissions for Nydus snapshotter.
+
+```yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nydus-snapshotter-sa
+  namespace: nydus-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role-binding
+roleRef:
+  kind: ClusterRole
+  name: nydus-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: nydus-snapshotter-sa
+  namespace: nydus-system
+```
+
+Then you can set the desired ServiceAccount with the required permissions fro your Nydus snapshotter Pod:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+spec:
+  containers:
+    name: nydus-snapshotter
+  serviceAccountName: nydus-snapshotter-sa
+  ...
+```
+
+#### create secrets
+
+If you have logged into a private registry, you can create a secret from the config file:
+
+```bash
+$ kubectl create --namespace nydus-system secret generic regcred \
+    --from-file=.dockerconfigjson=$HOME/.docker/config.json \
+    --type=kubernetes.io/dockerconfigjson
+```
+
+The Nydus snapshotter will get the new secret and parse the authorization. If your new Pod uses a private registry, then this authentication information will be used to pull the image from the private registry.
+
+## Metrics
+
+Nydusd records metrics in its own format. The metrics are exported via a HTTP server on top of unix domain socket. Nydus-snapshotter fetches the metrics and convert them in to Prometheus format which is exported via a network address. Nydus-snapshotter by default does not fetch metrics from nydusd. You can enable the nydusd metrics download by assigning a network address to `metrics.address` in nydus-snapshotter's toml [configuration file](../misc/snapshotter/config.toml).
+
+Once this entry is enabled, not only nydusd metrics, but also some information about the nydus-snapshotter 
+runtime and snapshot related events are exported in Prometheus format as well.
+
+## Diagnose
+
+A system controller can be ran insides nydus-snapshotter.
+By setting `system.enable` to `true`,  nydus-snapshotter will start a simple HTTP server on unix domain socket `system.address` path and exports some internal working status to users. The address defaults to `/var/run/containerd-nydus/system.sock`

docs/optimize_nydus_image.md

@@ -0,0 +1,135 @@
+# Optimize a nydus image
+
+To improve the prefetch hit rate, we can specify a prefetch table when converting an OCI image to a nydus image. The prefetch table is workload related and we should generate the list of accessed files in order when the workload is running. Nydus-snapshotter has implemented an optimizer with Containerd NRI plugin to optimize the nydus image. The optimizer is image format independent and requires NRI 2.0, which is available in containerd (>=v1.7.0). The optimizer subscribes container events to watch what files are opened and read, etc. during container application is starting up. This enables nydus image build tools to optimize the nydus image making it put the necessary and prioritized files into a special region of nydus image with extra image metadata. So nydus runtime can pull this files into local disk in top priority, thus to boost the performance further.
+
+## Requirements
+
+- [NRI 2.0](https://github.com/containerd/nri): Which has been integrated into containerd since [v1.7.0](https://github.com/containerd/containerd/tree/v1.7.0-beta.1).
+
+## Workflow
+
+![optimizer workload](./diagram/optmizer_workflow.svg)
+
+1. Run the optimizer as a NRI plugin to run optimizer server when the StartContainer event occurs.
+2. Optimizer server joins container mount namespace and starts fanotify server.
+3. Optimizer server detects fanotify event and sends the accessed file descriptor to client.
+4. Optimizer client converts file descriptor to path.
+5. Generate list of accessed files on local disk.
+6. Convert OCI images with accessed files list to nydus image.
+
+## Generate an accessed files list
+
+To install the optimizer and optimizer-server, invoke below command:
+
+```console
+make optimizer && make install-optimizer
+```
+
+This command installs the optimizer's default toml configuration file in `/etc/nri/conf.d/02-optimizer-nri-plugin.conf`. Here is an example:
+
+```toml
+# The directory to persist accessed files list for container.
+persist_dir = "/opt/nri/optimizer/results"
+# Whether to make the csv file human readable.
+readable = false
+# The path of optimizer server binary.
+server_path = "/usr/local/bin/optimizer-server"
+# The timeout to kill optimizer server, 0 to disable it.
+timeout = 0
+# Whether to overwrite the existed persistent files.
+overwrite = false
+# The events that containerd subscribes to.
+# Do not change this element.
+events = [ "StartContainer", "StopContainer" ]
+```
+
+Modify containerd's toml configuration file to enable NRI.
+
+```console
+sudo tee -a /etc/containerd/config.toml <<- EOF
+[plugins."io.containerd.nri.v1.nri"]
+  config_file = "/etc/nri/nri.conf"
+  disable = false
+  plugin_path = "/opt/nri/plugins"
+  socket_path = "/var/run/nri.sock"
+EOF
+
+```
+
+Containerd will load all NRI plugins in the `plugin_path` directory on startup. If you want to start a NRI plugin manually, please add the following configuration to allow other NRI plugins to connect via `socket_path`.
+
+```console
+sudo tee /etc/nri/nri.conf <<- EOF
+disableConnections: false
+EOF
+
+```
+
+Restart the containerd service.
+
+```console
+sudo systemctl restart containerd
+
+```
+
+Now, just run a container workload in sandbox and you will get the list of accessed files in `persist_dir`.
+Note that NRI plugin can only be called from containerd/CRI. So start a container using `crictl` as below.
+
+```console
+sudo tee nginx.yaml <<- EOF
+metadata:
+  name: nginx
+image:
+  image: nginx:latest
+log_path: nginx.0.log
+linux: {}
+EOF
+
+sudo tee pod.yaml <<- EOF
+metadata:
+  name: nginx-sandbox
+  namespace: default
+  attempt: 1
+  uid: hdishd83djaidwnduwk28bcsb
+log_directory: /tmp
+linux: {}
+EOF
+
+crictl run nginx.yaml pod.yaml
+
+```
+
+The result file for the nginx image is `/opt/nri/optimizer/results/library/nginx:latest`.
+
+## Build Nydus Image with Optimizer's Suggestions
+
+Nydus provides a [nydusify](https://github.com/dragonflyoss/nydus/blob/master/docs/nydusify.md) CLI tool to convert OCI images from the source registry or local file system to nydus format and push them to the target registry.
+
+We can install the `nydusify` cli tool from the nydus package.
+
+```console
+VERSION=v2.3.0
+
+wget https://github.com/dragonflyoss/nydus/releases/download/$VERSION/nydus-static-$VERSION-linux-amd64.tgz
+tar -zxvf nydus-static-$VERSION-linux-amd64.tgz
+sudo install -D -m 755 nydus-static/nydusify /usr/local/bin/nydusify
+sudo install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image
+```
+
+A simple example converts the OCI image nginx to nydus format in RAFS V6.
+
+```console
+sudo nydusify convert --source nginx --target sctb512/nginx:nydusv6 --fs-version 6
+```
+
+With the `--prefetch-patterns` argument, we can specify the list of files to be written in the front of the nydus image and be prefetched in order when starting a container.
+
+```console
+sudo nydusify convert --source nginx --target sctb512/nginx:optimized-nydusv6 --fs-version 6 --prefetch-patterns < /opt/nri/optimizer/results/library/nginx:latest
+```
+
+On a host with nydus-snapshotter installed and configured properly, start a container with an optimized nydus image.
+
+```console
+sudo nerdctl --snapshotter nydus run --rm --net host -it sctb512/nginx:optimized-nydusv6
+```

docs/run_nydus_in_kubernetes.md

@@ -0,0 +1,137 @@
+# Run Dragonfly & Nydus in Kubernetes
+
+We recommend using the Dragonfly P2P data distribution system to further improve the runtime performance of Nydus images.
+
+If you want to deploy Dragonfly and Nydus at the same time through Helm, please refer to the **[Quick Start](https://github.com/dragonflyoss/helm-charts/blob/main/INSTALL.md)**.
+
+# Run Nydus snapshotter in Kubernetes
+
+This document will introduce how to run Nydus snapshotter in Kubernetes cluster, you can use helm to deploy Nydus snapshotter container.
+
+**NOTE:** This document is mainly to allow everyone to quickly deploy and experience Nydus snapshotter in the Kubernetes cluster. You cannot use it as a deployment and configuration solution for the production environment.
+
+## Setup Kubernetes using kind
+
+[kind](https://kind.sigs.k8s.io/) is a tool for running local Kubernetes clusters using Docker container “nodes”.
+kind was primarily designed for testing Kubernetes itself, but may be used for local development or CI.
+
+First, we need to prepare a configuration file(`kind-config.yaml`) for kind to specify the devices and files we need to mount to the kind node.
+
+```yaml
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  ipFamily: dual
+nodes:
+  - role: control-plane
+    image: kindest/node:v1.30.2
+    extraMounts:
+      - hostPath: ./containerd-config.toml
+        containerPath: /etc/containerd/config.toml
+      - hostPath: /dev/fuse
+        containerPath: /dev/fuse
+```
+
+Next, we also need a config for containerd(`containerd-config.toml`).
+
+**NOTE:** It may be necessary to explain here why `disable_snapshot_annotations` and `discard_unpacked_layers` need to be configured in containerd.
+- `disable_snapshot_annotations`: This variable disables to pass additional annotations (image related information) to snapshotters in containerd (default value is `true`). In nydus snapshotter, we need these annotations to pull images. Therefore, we need to set it to `false`.
+- `discard_unpacked_layers`: This variable allows GC to remove layers from the content store after successfully unpacking these layers to the snapshotter in containerd (default value is `true`). In nydus snapshotter, we need to preserve layers for demand pulling and sharing even after they are unpacked. Therefore, we need to set it to `false`.
+
+```toml
+version = 2
+[debug]
+  level = "debug"
+
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  discard_unpacked_layers = false
+  disable_snapshot_annotations = false
+  snapshotter = "overlayfs"
+  default_runtime_name = "runc"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+[plugins."io.containerd.grpc.v1.cri"]
+  sandbox_image = "registry.k8s.io/pause:3.6"
+```
+
+With these two configuration files, we can create a kind cluster.
+
+```bash
+$ kind create cluster --config=kind-config.yaml
+```
+
+If everything is fine, kind should have prepared the configuration for us, and we can run the `kubectl` command directly on the host machine, such as:
+
+```bash
+$ kubectl get nodes
+NAME                 STATUS   ROLES                  AGE   VERSION
+kind-control-plane   Ready    control-plane,master   19m   v1.23.4
+```
+
+## Use helm to install Nydus snapshotter
+
+Before proceeding, you need to make sure [`helm` is installed](https://helm.sh/docs/intro/quickstart/#install-helm).
+
+First, you need to create a `config-nydus.yaml` for helm to install Nydus-snapshotter.
+
+```yaml
+name: nydus-snapshotter
+pullPolicy: Always
+hostNetwork: true
+dragonfly:
+  enable: false
+
+containerRuntime:
+  containerd:
+    enable: true
+```
+
+Then clone the Nydus snapshotter helm chart.
+
+```bash
+$ git clone https://github.com/dragonflyoss/helm-charts.git
+```
+
+Last run helm to create Nydus snapshotter.
+
+```bash
+$ cd helm-charts
+$ helm install --wait --timeout 10m --dependency-update \
+    --create-namespace --namespace nydus-system \
+    -f config-nydus.yaml \
+    nydus-snapshotter charts/nydus-snapshotter
+```
+
+## Run Nydus containers
+
+We can then create a Pod(`nydus-pod.yaml`) config file that runs Nydus image.
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nydus-pod
+spec:
+  containers:
+    - name: nginx
+      image: ghcr.io/dragonflyoss/image-service/nginx:nydus-latest
+      imagePullPolicy: Always
+      command: ["sh", "-c"]
+      args:
+        - tail -f /dev/null
+```
+
+Use `kubectl` to create the Pod.
+
+```bash
+$ kubectl create -f nydus-pod.yaml
+$ kubectl get pods -w
+```
+
+The `-w` options will block the console and wait for the changes of the status of the pod. If everything is normal, you can see that the pod will become `Running` after a while.
+
+```bash
+NAME       READY   STATUS              RESTARTS   AGE
+nydus-pod  1/1     Running             0          51s
+```

docs/setup_snapshotter_by_daemonset.md

@@ -0,0 +1,165 @@
+# Setup Nydus Snapshotter by DaemonSet
+
+This document will guide you through the simple steps of setting up and cleaning up the nydus snapshotter in a kubernetes cluster that runs on the host.
+
+## Steps for Setting up Nydus Snapshotter 
+
+To begin, let's clone the Nydus Snapshotter repository.
+
+```bash
+git clone https://github.com/containerd/nydus-snapshotter
+cd nydus-snapshotter
+```
+
+We can build the docker image locally. (optional)
+```bash
+$ export NYDUS_VER=$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name)
+$ make  # build snapshotter binaries
+$ cp bin/* misc/snapshotter/
+$ pushd misc/snapshotter/
+$ docker build --build-arg NYDUS_VER="${NYDUS_VER}" -t ghcr.io/containerd/nydus-snapshotter:latest .
+$ popd
+```
+**NOTE:** By default, the nydus snapshotter would use the latest release nydus version. If you want to use a specific version, you can set `NYDUS_VER` on your side.
+
+Next, we can configure access control for nydus snapshotter.
+```bash
+kubectl apply -f misc/snapshotter/nydus-snapshotter-rbac.yaml
+```
+
+Afterward, we can deploy a DaemonSet for nydus snapshotter, according to the kubernetes flavour you're using.
+
+```bash
+# Vanilla kubernetes
+kubectl apply -f misc/snapshotter/base/nydus-snapshotter.yaml
+```
+
+```bash
+# k3s
+kubectl apply -k misc/snapshotter/overlays/k3s/
+```
+
+```bash
+# rke2
+kubectl apply -k misc/snapshotter/overlays/rke2/
+```
+
+Then, we can confirm that nydus snapshotter is running through the DaemonSet.
+```bash
+$ kubectl get pods -n nydus-system 
+NAME                      READY   STATUS    RESTARTS   AGE
+nydus-snapshotter-26rf7   1/1     Running   0          18s
+```
+
+Finally, we can view the logs in the pod.
+```bash
+$ kubectl logs nydus-snapshotter-26rf7 -n nydus-system
+install nydus snapshotter artifacts
+there is no proxy plugin!
+Created symlink /etc/systemd/system/multi-user.target.wants/nydus-snapshotter.service → /etc/systemd/system/nydus-snapshotter.service.
+```
+
+And we can see the nydus snapshotter service on the host.
+```bash
+$ systemctl status nydus-snapshotter
+● nydus-snapshotter.service - nydus snapshotter
+     Loaded: loaded (/etc/systemd/system/nydus-snapshotter.service; enabled; vendor preset: enabled)
+    Drop-In: /etc/systemd/system/nydus-snapshotter.service.d
+             └─proxy.conf
+     Active: active (running) since Wed 2024-01-17 16:14:22 UTC; 56s ago
+   Main PID: 1100169 (containerd-nydu)
+      Tasks: 11 (limit: 96376)
+     Memory: 8.6M
+        CPU: 35ms
+     CGroup: /system.slice/nydus-snapshotter.service
+             └─1100169 /opt/nydus/bin/containerd-nydus-grpc --config /etc/nydus/config.toml
+
+Jan 17 16:14:22 worker systemd[1]: Started nydus snapshotter.
+Jan 17 16:14:22 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:22.998798369Z" level=info msg="Start nydus-snapshotter. Version: v0.7.0-308-g106a6cb, PID: 1100169, FsDriver: fusedev, DaemonMode: dedicated"
+Jan 17 16:14:23 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:23.000186538Z" level=info msg="Run daemons monitor..."
+```
+
+**NOTE:** By default, the nydus snapshotter operates as a systemd service. If you prefer to run nydus snapshotter as a standalone process, you can set `ENABLE_SYSTEMD_SERVICE` to `false` in `nydus-snapshotter.yaml`.
+
+## Steps for Cleaning up Nydus Snapshotter 
+
+We use `preStop`` hook in the DaemonSet to uninstall nydus snapshotter and roll back the containerd configuration.
+
+```bash
+# Vanilla kubernetes
+$ kubectl delete -f misc/snapshotter/base/nydus-snapshotter.yaml 
+```
+
+```bash
+# k3s
+$ kubectl delete -k misc/snapshotter/overlays/k3s/ 
+```
+
+```bash
+# rke2
+$ kubectl delete -k misc/snapshotter/overlays/rke2/
+```
+
+```bash
+$ kubectl delete -f misc/snapshotter/nydus-snapshotter-rbac.yaml 
+$ systemd restart containerd.service
+```
+
+## Customized Setup
+
+As we know, nydus snapshotter supports four filesystem drivers (fs_driver): `fusedev`, `fscache`, `blockdev`, `proxy`. Within the container image, we have included configurations for these snapshotter drivers, as well as the corresponding nydusd configurations. By default, the fusedev driver is enabled in the nydus snapshotter, using the snapshotter configuration [`config-fusedev.toml`](../misc/snapshotter/config-fusedev.toml) and the nydusd configuration [`nydusd-config.fusedev.json`](../misc/snapshotter/nydusd-config.fusedev.json).
+
+### Other filesystem driver with related default configuration
+
+If we want to setup the nydus snapshotter with the default configuration for different fs_driver (such as `proxy`), we can modify the values in the `Configmap` in `nydus-snapshotter.yaml`:
+```yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
+  namespace: nydus-snapshotter
+data:
+  FS_DRIVER: "proxy"
+  NYDUSD_DAEMON_MODE: "none"
+```
+
+Then we can run the nydus snapshotter enabling `proxy` `fs_driver` with the snapshotter configuration [`config-proxy.toml`](../misc/snapshotter/config-proxy.toml).
+
+**NOTE:** The fs_driver (`blockdev` and `proxy`) do not need nydusd, so they do not need nydusd config.
+
+### Same filesystem with different snapshotter configuration and different nydusd configuration
+
+If we want to setup the nydus snapshotter for the same fs_driver (such as `fusedev`) with different snapshotter configuration and different nydusd configuration, we can enable `ENABLE_CONFIG_FROM_VOLUME` and add the snapshotter configuration [`config.toml`](../misc/snapshotter/config.toml) and nydusd configuration [`nydusd-config.json`](../misc/snapshotter/nydusd-config.fusedev.json) in the `Configmap` in `nydus-snapshotter.yaml`:
+
+```yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
+  namespace: nydus-snapshotter
+data:
+  ENABLE_CONFIG_FROM_VOLUME: "true"
+
+  config.toml: |-
+    # The snapshotter config content copied here
+
+  nydusd-config.json: |-
+    # The nydusd config content copied here
+```
+
+**NOTE:** We need to set `nydusd_config` to `/etc/nydus/nydusd-config.json` in the `config.toml`, so that snapshotter can find the nydusd configuration from configmap.
+
+### Customized Options
+
+| Options                             | Type   | Default                               | Comment                                                                                                                                         |
+| ----------------------------------- | ------ | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| FS_DRIVER                           | string | "fusedev"                             | the filesystem driver of snapshotter                                                                                                            |
+| ENABLE_CONFIG_FROM_VOLUME           | bool   | false                                 | enabling to use the configurations from volume                                                                                                  |
+| ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER | bool   | false                                 | enabling to skip to set `plugins."io.containerd.grpc.v1.cri".containerd` to `nydus` for runtime specific snapshotter feature in containerd 1.7+ |
+| ENABLE_SYSTEMD_SERVICE              | bool   | true                                  | enabling to run nydus snapshotter as a systemd service                                                                                          |

docs/tarfs.md

@@ -0,0 +1,175 @@
+# Nydus Tarfs Mode
+
+`Nydus Tarfs Mode` or `Tarfs` is a working mode for Nydus Image, which uses tar files as Nydus data blobs instead of generating native Nydus data blobs.
+
+### Enable Tarfs
+`Nydus Tarfs Mode` is still an experiment feature, please edit the snapshotter configuration file to enable the feature:
+```
+[experimental.tarfs]
+enable_tarfs = true
+```
+
+### Generate Raw Disk Image for Each Layer of a Container Image
+`Tarfs` supports generating a raw disk image for each layer of a container image, which can be directly mounted as EROFS filesystem through loopdev. Please edit the snapshotter configuration file to enable this submode:
+```
+[experimental.tarfs]
+enable_tarfs = true
+export_mode = "layer_block"
+```
+
+This is an example to generate and verify raw disk image for each layer of a container image:
+```
+$ containerd-nydus-grpc --config /etc/nydus/config.toml &
+$ nerdctl run --snapshotter nydus --rm nginx
+
+# Show mounted rootfs a container
+$ mount
+/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
+
+# Show loop devices used to mount layers and bootstrap for a container image
+$ losetup 
+NAME SIZELIMIT OFFSET AUTOCLEAR RO BACK-FILE                                                                       DIO LOG-SEC
+/dev/loop11 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa 0     512
+/dev/loop12 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 0     512
+/dev/loop13 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd 0     512
+/dev/loop14 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4 0     512
+/dev/loop15 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 0     512
+/dev/loop16 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75 0     512
+/dev/loop17 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/fs/image/image.boot                         0     512
+
+# Files without suffix are tar files, files with suffix `layer.disk` are raw disk image for container image layers
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
+total 376800
+-rw-r--r-- 1 root root      3584 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+-rw-r--r-- 1 root root    527872 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk
+-rw-r--r-- 1 root root  77814784 Aug 30 23:18 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5
+-rw-r--r-- 1 root root  78863360 Aug 30 23:18 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5.layer.disk
+-rw-r--r-- 1 root root      4608 Aug 30 23:18 96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd
+-rw-r--r-- 1 root root    528896 Aug 30 23:18 96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd.layer.disk
+-rw-r--r-- 1 root root      2560 Aug 30 23:18 a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4
+-rw-r--r-- 1 root root    526848 Aug 30 23:18 a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4.layer.disk
+-rw-r--r-- 1 root root      7168 Aug 30 23:18 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75
+-rw-r--r-- 1 root root    531456 Aug 30 23:18 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.layer.disk
+-rw-r--r-- 1 root root      5120 Aug 30 23:18 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665
+-rw-r--r-- 1 root root    529408 Aug 30 23:18 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665.layer.disk
+-rw-r--r-- 1 root root 112968704 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa
+-rw-r--r-- 1 root root 113492992 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa.layer.disk
+$ file /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3: POSIX tar archive
+
+# Mount the raw disk image for a container image layer
+$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk 
+$ mount -t erofs /dev/loop100 ./mnt/
+$ mount
+tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=1544836k,nr_inodes=386209,mode=700,inode64)
+/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
+/dev/loop100 on /root/ws/nydus-snapshotter.git/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
+
+```
+
+### Generate Raw Disk Image for a Container Image
+`Tarfs` supports generating a raw disk image a container image, which can be directly mounted as EROFS filesystem through loopdev. Please edit the snapshotter configuration file to enable this submode:
+```
+[experimental.tarfs]
+enable_tarfs = true
+export_mode = "image_block"
+```
+
+This is an example to generate and verify raw disk image for a container image:
+```
+$ containerd-nydus-grpc --config /etc/nydus/config.toml &
+$ nerdctl run --snapshotter nydus --rm nginx
+
+# Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
+total 376320
+-rw-r--r-- 1 root root      3584 Aug 30 23:35 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+-rw-r--r-- 1 root root  77814784 Aug 30 23:35 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5
+-rw-r--r-- 1 root root      4608 Aug 30 23:35 96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd
+-rw-r--r-- 1 root root      2560 Aug 30 23:35 a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4
+-rw-r--r-- 1 root root      7168 Aug 30 23:35 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75
+-rw-r--r-- 1 root root 194518016 Aug 30 23:36 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+-rw-r--r-- 1 root root      5120 Aug 30 23:35 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665
+-rw-r--r-- 1 root root 112968704 Aug 30 23:36 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa
+
+```
+
+### Generate Raw Disk Image with dm-verity Information
+`Tarfs` supports generating raw disk images with dm-verity information, to enable runtime data integrity validation. Please change `export_mode` in snapshotter configuration file to `layer_block_with_verity` or `image_block_with_verity`.
+
+```
+[experimental.tarfs]
+enable_tarfs = true
+export_mode = "image_block_with_verity"
+```
+
+This is an example to generate and verify raw disk image for a container image with dm-verity information:
+```
+$ containerd-nydus-grpc --config /etc/nydus/config.toml &
+$ nerdctl run --snapshotter nydus --rm nginx
+
+# Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
+total 388296
+-rw-r--r-- 1 root root      3584 Aug 30 23:45 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+-rw-r--r-- 1 root root  77814784 Aug 30 23:46 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5
+-rw-r--r-- 1 root root      4608 Aug 30 23:45 96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd
+-rw-r--r-- 1 root root      2560 Aug 30 23:45 a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4
+-rw-r--r-- 1 root root      7168 Aug 30 23:45 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75
+-rw-r--r-- 1 root root 206782464 Aug 30 23:46 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+-rw-r--r-- 1 root root      5120 Aug 30 23:45 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665
+-rw-r--r-- 1 root root 112968704 Aug 30 23:46 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa
+
+$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+$  veritysetup open --no-superblock --format=1 -s "" --hash=sha256 --data-block-size=512 --hash-block-size=4096 --data-blocks 379918 --hash-offset 194519040 /dev/loop100 image1 /dev/loop100 8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd
+$ lsblk
+loop100   7:100  0 197.2M  0 loop  
+└─dm-0  252:0    0 185.5M  1 crypt 
+
+$ veritysetup status dm-0
+/dev/mapper/dm-0 is active and is in use.
+  type:        VERITY
+  status:      verified
+  hash type:   1
+  data block:  512
+  hash block:  4096
+  hash name:   sha256
+  salt:        -
+  data device: /dev/loop100
+  data loop:   /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+  size:        379918 sectors
+  mode:        readonly
+  hash device: /dev/loop100
+  hash loop:   /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+  hash offset: 379920 sectors
+  root hash:   8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd
+
+$ mount -t erofs /dev/dm-0 ./mnt/
+mount: /root/ws/nydus-snapshotter.git/mnt: WARNING: source write-protected, mounted read-only.
+$ ls -l mnt/
+total 14
+lrwxrwxrwx  1 root root    7 Aug 14 08:00 bin -> usr/bin
+drwxr-xr-x  2 root root   27 Jul 15 00:00 boot
+drwxr-xr-x  2 root root   27 Aug 14 08:00 dev
+drwxr-xr-x  2 root root  184 Aug 16 17:50 docker-entrypoint.d
+-rwxrwxr-x  1 root root 1620 Aug 16 17:50 docker-entrypoint.sh
+drwxr-xr-x 34 root root 1524 Aug 16 17:50 etc
+drwxr-xr-x  2 root root   27 Jul 15 00:00 home
+lrwxrwxrwx  1 root root    7 Aug 14 08:00 lib -> usr/lib
+lrwxrwxrwx  1 root root    9 Aug 14 08:00 lib32 -> usr/lib32
+lrwxrwxrwx  1 root root    9 Aug 14 08:00 lib64 -> usr/lib64
+lrwxrwxrwx  1 root root   10 Aug 14 08:00 libx32 -> usr/libx32
+drwxr-xr-x  2 root root   27 Aug 14 08:00 media
+drwxr-xr-x  2 root root   27 Aug 14 08:00 mnt
+drwxr-xr-x  2 root root   27 Aug 14 08:00 opt
+drwxr-xr-x  2 root root   27 Jul 15 00:00 proc
+drwx------  2 root root   66 Aug 14 08:00 root
+drwxr-xr-x  3 root root   43 Aug 14 08:00 run
+lrwxrwxrwx  1 root root    8 Aug 14 08:00 sbin -> usr/sbin
+drwxr-xr-x  2 root root   27 Aug 14 08:00 srv
+drwxr-xr-x  2 root root   27 Jul 15 00:00 sys
+drwxrwxrwt  2 root root   27 Aug 16 17:50 tmp
+drwxr-xr-x 14 root root  229 Aug 14 08:00 usr
+drwxr-xr-x 11 root root  204 Aug 14 08:00 var
+
+```

export/snapshotter/snapshotter.go

@@ -1,8 +1,10 @@
 package snapshotter
 
 import (
-	"github.com/containerd/containerd/platforms"
-	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/v2/plugins"
+	"github.com/containerd/platforms"
+	"github.com/containerd/plugin"
+	"github.com/containerd/plugin/registry"
 	"github.com/pkg/errors"
 
 	"github.com/containerd/nydus-snapshotter/config"
@@ -10,23 +12,25 @@ import (
 )
 
 func init() {
-	plugin.Register(&plugin.Registration{
-		Type:   plugin.SnapshotPlugin,
+	registry.Register(&plugin.Registration{
+		Type:   plugins.SnapshotPlugin,
 		ID:     "nydus",
-		Config: &config.Config{},
+		Config: &config.SnapshotterConfig{},
 		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
 			ic.Meta.Platforms = append(ic.Meta.Platforms, platforms.DefaultSpec())
 
-			cfg, ok := ic.Config.(*config.Config)
+			cfg, ok := ic.Config.(*config.SnapshotterConfig)
 			if !ok {
 				return nil, errors.New("invalid nydus snapshotter configuration")
 			}
 
-			if cfg.RootDir == "" {
-				cfg.RootDir = ic.Root
+			root := ic.Properties[plugins.PropertyRootDir]
+			if root == "" {
+				cfg.Root = root
 			}
-			if err := cfg.FillupWithDefaults(); err != nil {
-				return nil, errors.New("failed to fillup nydus configuration with defaults")
+
+			if err := cfg.FillUpWithDefaults(); err != nil {
+				return nil, errors.New("failed to fill up nydus configuration with defaults")
 			}
 
 			rs, err := snapshot.NewSnapshotter(ic.Context, cfg)

go.mod

@@ -1,91 +1,186 @@
 module github.com/containerd/nydus-snapshotter
 
-go 1.17
+go 1.22.0
 
 require (
-	github.com/KarpelesLab/reflink v0.0.2
-	github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5
-	github.com/containerd/containerd v1.6.6
-	github.com/containerd/continuity v0.2.2
-	github.com/containerd/fifo v1.0.0
-	github.com/containerd/stargz-snapshotter/estargz v0.11.4
-	github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible
+	dario.cat/mergo v1.0.1
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6
+	github.com/KarpelesLab/reflink v1.0.1
+	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
+	github.com/aws/aws-sdk-go-v2 v1.30.3
+	github.com/aws/aws-sdk-go-v2/config v1.27.27
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.27
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.8
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2
+	github.com/containerd/cgroups/v3 v3.0.3
+	github.com/containerd/containerd/api v1.8.0
+	github.com/containerd/containerd/v2 v2.0.2
+	github.com/containerd/continuity v0.4.4
+	github.com/containerd/errdefs v1.0.0
+	github.com/containerd/fifo v1.1.0
+	github.com/containerd/log v0.1.0
+	github.com/containerd/nri v0.8.0
+	github.com/containerd/platforms v1.0.0-rc.1
+	github.com/containerd/plugin v1.0.0
+	github.com/containerd/stargz-snapshotter v0.15.2-0.20240709063920-1dac5ef89319
+	github.com/containerd/stargz-snapshotter/estargz v0.15.2-0.20240709063920-1dac5ef89319
+	github.com/containers/ocicrypt v1.2.0
+	github.com/distribution/reference v0.6.0
+	github.com/docker/cli v27.1.0+incompatible
+	github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
-	github.com/google/go-containerregistry v0.5.1
-	github.com/google/uuid v1.2.0
-	github.com/hashicorp/go-retryablehttp v0.7.1
+	github.com/google/go-containerregistry v0.20.1
+	github.com/gorilla/mux v1.8.1
+	github.com/hashicorp/go-retryablehttp v0.7.7
+	github.com/klauspost/compress v1.17.11
+	github.com/moby/locker v1.0.1
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
+	github.com/opencontainers/image-spec v1.1.0
+	github.com/opencontainers/runtime-spec v1.2.0
+	github.com/pelletier/go-toml v1.9.5
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.11.1
-	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.30.0
-	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
-	github.com/urfave/cli/v2 v2.3.0
-	go.etcd.io/bbolt v1.3.6
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
-	google.golang.org/grpc v1.43.0
-	k8s.io/api v0.22.5
-	k8s.io/apimachinery v0.22.5
-	k8s.io/client-go v0.22.5
+	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/client_model v0.6.1
+	github.com/rs/xid v1.5.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.10.0
+	github.com/urfave/cli/v2 v2.27.5
+	go.etcd.io/bbolt v1.3.11
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	golang.org/x/net v0.30.0
+	golang.org/x/sync v0.8.0
+	golang.org/x/sys v0.26.0
+	google.golang.org/grpc v1.67.1
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
+	gotest.tools v2.2.0+incompatible
+	k8s.io/api v0.31.2
+	k8s.io/apimachinery v0.31.2
+	k8s.io/client-go v0.31.2
+	k8s.io/cri-api v0.31.2
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 )
 
 require (
-	github.com/Microsoft/go-winio v0.5.1 // indirect
-	github.com/Microsoft/hcsshim v0.9.3 // indirect
-	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
+	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.12.9 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
+	github.com/aws/smithy-go v1.20.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/containerd/cgroups v1.0.3 // indirect
-	github.com/containerd/ttrpc v1.1.0 // indirect
-	github.com/containerd/typeurl v1.0.2 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cilium/ebpf v0.11.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
+	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/distribution v2.8.1+incompatible // indirect
-	github.com/docker/docker v20.10.17+incompatible // indirect
-	github.com/docker/docker-credential-helpers v0.6.3 // indirect
-	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
-	github.com/go-logr/logr v1.2.2 // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.1 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/moby/sys/mountinfo v0.5.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/miekg/pkcs11 v1.1.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opencontainers/selinux v1.11.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/russross/blackfriday/v2 v2.0.1 // indirect
-	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/vbatts/tar-split v0.11.2 // indirect
-	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/vbatts/tar-split v0.11.5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.mozilla.org/pkcs7 v0.9.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
+	go.opentelemetry.io/otel v1.31.0 // indirect
+	go.opentelemetry.io/otel/metric v1.31.0 // indirect
+	go.opentelemetry.io/otel/trace v1.31.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/oauth2 v0.22.0 // indirect
+	golang.org/x/term v0.25.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/klog/v2 v2.30.0 // indirect
-	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+	tags.cncf.io/container-device-interface v0.8.0 // indirect
+	tags.cncf.io/container-device-interface/specs-go v0.8.0 // indirect
 )
 
-replace (
-	cloud.google.com/go => cloud.google.com/go v0.81.0
-	github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.2
+retract (
+	v0.11.3
+	v0.11.2
+	v0.11.1
+	v0.11.0
+	v0.3.0 // retagged: cd604c1b597558ea045a79c4f80a8c780909801b -> 85653575c7dafb6b06548478ee1dc61ac5905d00
+)
+
+exclude (
+	// These dependencies were updated to "master" in some modules we depend on,
+	// but have no code-changes since their last release. Unfortunately, this also
+	// causes a ripple effect, forcing all users of the containerd module to also
+	// update these dependencies to an unrelease / un-tagged version.
+	//
+	// Both these dependencies will unlikely do a new release in the near future,
+	// so exclude these versions so that we can downgrade to the current release.
+	//
+	// For additional details, see this PR and links mentioned in that PR:
+	// https://github.com/kubernetes-sigs/kustomize/pull/5830#issuecomment-2569960859
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 )

integration/Dockerfile

@@ -0,0 +1,70 @@
+ARG CONTAINERD_VER=2.0.0-rc.3
+ARG CONTAINERD_PROJECT=/containerd
+ARG RUNC_VER=1.1.4
+ARG NYDUS_SNAPSHOTTER_PROJECT=/nydus-snapshotter
+ARG DOWNLOADS_MIRROR="https://github.com"
+ARG NYDUS_VER=2.3.0
+ARG NERDCTL_VER=1.7.6
+ARG DELVE_VER=1.23.0
+ARG GO_VER=1.22.5-bookworm
+
+FROM golang:$GO_VER AS golang-base
+ARG TARGETARCH
+ARG CONTAINERD_VER
+ARG CONTAINERD_PROJECT
+ARG RUNC_VER
+ARG NYDUS_SNAPSHOTTER_PROJECT
+ARG DOWNLOADS_MIRROR
+ARG NYDUS_VER
+ARG NERDCTL_VER
+ARG DELVE_VER
+
+# RUN echo '\
+# deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye main contrib non-free\n\
+# deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-updates main contrib non-free\n\
+# deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-backports main contrib non-free\n\
+# deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bullseye-security main contrib non-free\n' > /etc/apt/sources.list
+
+RUN apt-get update -qq && apt-get install -qq libbtrfs-dev libseccomp-dev sudo psmisc jq lsof net-tools
+
+RUN go install github.com/go-delve/delve/cmd/dlv@v"$DELVE_VER"
+
+# Install containerd
+RUN wget -q "$DOWNLOADS_MIRROR"/containerd/containerd/releases/download/v"$CONTAINERD_VER"/containerd-"$CONTAINERD_VER"-linux-"$TARGETARCH".tar.gz && \
+  tar xzf containerd-"$CONTAINERD_VER"-linux-"$TARGETARCH".tar.gz && \
+  install -D -m 755 bin/* /usr/local/bin/
+COPY misc/example/containerd-config.toml /etc/containerd/config.toml
+
+# Install runc
+RUN wget -q "$DOWNLOADS_MIRROR"/opencontainers/runc/releases/download/v"$RUNC_VER"/runc."$TARGETARCH" && \
+  install -D -m 755 runc."$TARGETARCH" /usr/local/bin/runc
+
+# Install nydusd nydus-image
+RUN  wget -q "$DOWNLOADS_MIRROR"/dragonflyoss/nydus/releases/download/v"$NYDUS_VER"/nydus-static-v"$NYDUS_VER"-linux-"$TARGETARCH".tgz && \
+  tar xzf nydus-static-v"$NYDUS_VER"-linux-"$TARGETARCH".tgz && \
+  install -D -m 755 nydus-static/nydusd /usr/local/bin/nydusd && \
+  install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image && \
+  install -D -m 755 nydus-static/nydusctl /usr/local/bin/nydusctl
+
+# Install nerdctl
+RUN wget -q "$DOWNLOADS_MIRROR"/containerd/nerdctl/releases/download/v"$NERDCTL_VER"/nerdctl-"$NERDCTL_VER"-linux-"$TARGETARCH".tar.gz && \
+  tar xzf nerdctl-"$NERDCTL_VER"-linux-"$TARGETARCH".tar.gz && \
+  install -D -m 755 nerdctl /usr/local/bin/nerdctl
+
+# Install fscache driver configuration file
+COPY misc/snapshotter/nydusd-config.fscache.json /etc/nydus/nydusd-config.fscache.json
+COPY misc/snapshotter/nydusd-config-localfs.json /etc/nydus/nydusd-config-localfs.json
+COPY misc/snapshotter/config.toml /etc/nydus/config.toml
+
+VOLUME [ "/var/lib" ]
+
+COPY integration/entrypoint.sh /
+
+WORKDIR /nydus-snapshotter
+
+ENV PATH="${PATH}:/usr/local/bin/"
+
+# Prevent git from complaining about ownership
+RUN git config --global --add safe.directory /nydus-snapshotter
+
+ENTRYPOINT [ "/bin/bash",  "-c",  "make install && /entrypoint.sh"]

integration/entrypoint.sh

@@ -0,0 +1,611 @@
+#!/bin/bash
+
+# Copyright (c) 2022. Nydus Developers. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -eEuo pipefail
+
+FSCACHE_NYDUSD_CONFIG=/etc/nydus/nydusd-config.fscache.json
+FUSE_NYDUSD_LOCALFS_CONFIG=/etc/nydus/nydusd-config-localfs.json
+SNAPSHOTTER_CONFIG=/etc/nydus/config.toml
+
+CONTAINERD_ROOT=/var/lib/containerd/
+CONTAINERD_STATUS=/run/containerd/
+REMOTE_SNAPSHOTTER_SOCKET=/run/containerd-nydus/containerd-nydus-grpc.sock
+REMOTE_SNAPSHOTTER_ROOT=/var/lib/containerd/io.containerd.snapshotter.v1.nydus
+CONTAINERD_SOCKET=/run/containerd/containerd.sock
+SNAPSHOTTER_SHARED_MNT=${REMOTE_SNAPSHOTTER_ROOT}/mnt
+SNAPSHOTTER_CACHE_DIR=${REMOTE_SNAPSHOTTER_ROOT}/cache
+
+JAVA_IMAGE=${JAVA_IMAGE:-ghcr.io/dragonflyoss/image-service/java:nydus-nightly-v6}
+WORDPRESS_IMAGE=${WORDPRESS_IMAGE:-ghcr.io/dragonflyoss/image-service/wordpress:nydus-nightly-v6}
+TOMCAT_IMAGE=${TOMCAT_IMAGE:-ghcr.io/dragonflyoss/image-service/tomcat:nydus-nightly-v5}
+STARGZ_IMAGE=${STARGZ_IMAGE:-ghcr.io/stargz-containers/wordpress:5.9.2-esgz}
+REDIS_OCI_IMAGE=${REDIS_OCI_IMAGE:-ghcr.io/stargz-containers/redis:6.2.6-org}
+WORDPRESS_OCI_IMAGE=${WORDPRESS_OCI_IMAGE:-ghcr.io/dragonflyoss/image-service/wordpress:latest}
+
+PLUGIN=nydus
+
+RETRYNUM=30
+RETRYINTERVAL=1
+TIMEOUTSEC=180
+
+GORACE_REPORT="$(pwd)/go_race_report"
+export GORACE="log_path=${GORACE_REPORT}"
+
+# trap "{ pause 1000; }" ERR
+
+function detect_go_race {
+    if [ -n "$(ls -A ${GORACE_REPORT}.* 2>/dev/null)" ]; then
+        echo "go race detected"
+        reports=$(ls -A ${GORACE_REPORT}.* 2>/dev/null)
+        for r in ${reports}; do
+            cat "$r"
+        done
+        exit 1
+    fi
+}
+
+function stop_all_containers {
+    containers=$(nerdctl ps -q | tr '\n' ' ')
+    if [[ ${containers} == "" ]]; then
+        return 0
+    else
+        echo "Killing containers ${containers}"
+        for C in ${containers}; do
+            nerdctl kill "${C}" || true
+            nerdctl stop "${C}" || true
+            nerdctl rm "${C}" || true
+        done
+        return 1
+    fi
+}
+
+function pause {
+    echo "I am going to wait for ${1} seconds only ..."
+    sleep "${1}"
+}
+
+function func_retry {
+    local SUCCESS=false
+    for i in $(seq ${RETRYNUM}); do
+        if "${*}"; then
+            SUCCESS=true
+            break
+        fi
+        echo "Fail(${i}). Retrying function..."
+        sleep ${RETRYINTERVAL}
+    done
+    if [ "${SUCCESS}" == "true" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function retry {
+    local SUCCESS=false
+    for i in $(seq ${RETRYNUM}); do
+        if eval "timeout ${TIMEOUTSEC} ${@}"; then
+            SUCCESS=true
+            break
+        fi
+        echo "Fail(${i}). Retrying..."
+        sleep ${RETRYINTERVAL}
+    done
+    if [ "${SUCCESS}" == "true" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function can_erofs_ondemand_read {
+    return 1
+    # grep 'CONFIG_EROFS_FS_ONDEMAND=[ym]' /usr/src/linux-headers-"$(uname -r)"/.config 1>/dev/null
+    # echo $?
+}
+
+function validate_mnt_number {
+    expected="${1}"
+    found=$(mount -t fuse | wc -l)
+    if [[ $found != "$expected" ]]; then
+        echo "expecting $expected mountpoints, but found $found"
+        return 1
+    else
+        return 0
+    fi
+}
+
+function set_config_option {
+    KEY="${1}"
+    VALUE="${2}"
+
+    sed -i "s/\($KEY *= *\).*/\1$VALUE/" "${SNAPSHOTTER_CONFIG}"
+}
+
+function set_recover_policy {
+    policy="${1}"
+
+    set_config_option "recover_policy" \"${policy}\"
+}
+
+function set_enable_referrer_detect {
+    set_config_option "enable_referrer_detect" "true"
+}
+
+function reboot_containerd {
+    killall "containerd" || true
+    killall "containerd-nydus-grpc" || true
+    # In case nydusd is using cache dir
+    killall "nydusd" || true
+
+    # Let snapshotter shutdown all its services.
+    sleep 2
+
+    # FIXME
+    echo "umount globally shared mountpoint"
+    umount_global_shared_mnt
+
+    rm -rf "${CONTAINERD_STATUS}"*
+    rm -rf "${CONTAINERD_ROOT}"*
+    if [ -f "${REMOTE_SNAPSHOTTER_SOCKET}" ]; then
+        rm "${REMOTE_SNAPSHOTTER_SOCKET}"
+    fi
+
+    local daemon_mode=${1}
+    local fs_driver=${2:-fusedev}
+    local recover_policy=${3:-none}
+
+    if [ -d "${REMOTE_SNAPSHOTTER_ROOT:?}/snapshotter/snapshots/" ]; then
+        umount -t fuse --all
+    fi
+
+    if [[ "${fs_driver}" == fusedev ]]; then
+        nydusd_config=/etc/nydus/nydusd-config.json
+    else
+        nydusd_config="$FSCACHE_NYDUSD_CONFIG"
+    fi
+
+    # Override nydus configuration, this configuration is usually set by each case
+    if [[ -n ${NYDUS_CONFIG_PATH:-} ]]; then
+        nydusd_config=${NYDUS_CONFIG_PATH}
+    fi
+
+    # rm -rf "${REMOTE_SNAPSHOTTER_ROOT:?}"/* || fuser -m "${REMOTE_SNAPSHOTTER_ROOT}/mnt" && false
+    rm -rf "${REMOTE_SNAPSHOTTER_ROOT:?}"/*
+
+    set_recover_policy "${recover_policy}"
+
+    containerd-nydus-grpc --log-to-stdout \
+        --daemon-mode "${daemon_mode}" --fs-driver "${fs_driver}" \
+        --config "${SNAPSHOTTER_CONFIG}" --nydusd-config "${nydusd_config}" &
+
+    retry ls "${REMOTE_SNAPSHOTTER_SOCKET}"
+    containerd --log-level info --config=/etc/containerd/config.toml &
+    retry ls "${CONTAINERD_SOCKET}"
+
+    # Makes sure containerd and containerd-nydus-grpc are up-and-running.
+    UNIQUE_SUFFIX=$(date +%s%N | shasum | base64 | fold -w 10 | head -1)
+    retry ctr snapshots --snapshotter="${PLUGIN}" prepare "connectiontest-dummy-${UNIQUE_SUFFIX}" ""
+}
+
+function restart_snapshotter {
+    killall -INT containerd-nydus-grpc
+    local daemon_mode=$1
+}
+
+function umount_global_shared_mnt {
+    umount "${SNAPSHOTTER_SHARED_MNT}" || true
+}
+
+function is_cache_cleared {
+    # With fscache driver, 2.1 nydusd don't have API to release the cache files.
+    # Thy locate at directory ${SNAPSHOTTER_CACHE_DIR}/cache
+    if [[ $(ls -A -p "${SNAPSHOTTER_CACHE_DIR}" | grep -v /) == "" ]]; then
+        true
+    else
+        echo "ERROR: Cache is not cleared"
+        false
+    fi
+}
+
+function nerdctl_prune_images {
+    # Wait for containers observation.
+    sleep 1
+    func_retry stop_all_containers
+    nerdctl container prune -f
+    nerdctl image prune --all -f
+    nerdctl images
+    is_cache_cleared
+}
+
+function start_single_container_multiple_daemons {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd multiple
+
+    nerdctl --snapshotter nydus run -d --net none "${JAVA_IMAGE}"
+
+    detect_go_race
+}
+
+function start_multiple_containers_multiple_daemons {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd multiple
+
+    nerdctl --snapshotter nydus run -d --net none "${JAVA_IMAGE}"
+    nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus run -d --net none "${TOMCAT_IMAGE}"
+
+    nerdctl_prune_images
+
+    nerdctl --snapshotter nydus run -d --net none "${TOMCAT_IMAGE}"
+    nerdctl --snapshotter nydus run -d --net none "${JAVA_IMAGE}"
+    nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_IMAGE}"
+
+    detect_go_race
+}
+
+function start_multiple_containers_shared_daemon {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd shared
+
+    nerdctl --snapshotter nydus run -d --net none "${JAVA_IMAGE}"
+    nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus run -d --net none "${TOMCAT_IMAGE}"
+
+    detect_go_race
+}
+
+function start_single_container_on_stargz {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd multiple
+
+    killall "containerd-nydus-grpc" || true
+    sleep 2
+
+    containerd-nydus-grpc --enable-stargz --daemon-mode multiple --fs-driver fusedev \
+        --recover-policy none --log-to-stdout --config-path /etc/nydus/nydusd-config.json &
+
+    nerdctl --snapshotter nydus run -d --net none "${STARGZ_IMAGE}"
+    detect_go_race
+}
+
+function start_container_on_oci {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd multiple
+
+    nerdctl --snapshotter nydus run -d --net none "${REDIS_OCI_IMAGE}"
+    nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_OCI_IMAGE}"
+    pause 2
+
+    func_retry stop_all_containers
+
+    # Deleteing with flag --async as a fuzzer
+    nerdctl image rm --async --force "${REDIS_OCI_IMAGE}"
+    nerdctl image rm --force "${WORDPRESS_OCI_IMAGE}"
+}
+
+function start_container_with_referrer_detect {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd multiple
+
+    set_enable_referrer_detect
+    nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_OCI_IMAGE}"
+
+    detect_go_race
+}
+
+function pull_remove_one_image {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd multiple
+
+    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
+    nerdctl --snapshotter nydus image rm "${JAVA_IMAGE}"
+
+    detect_go_race
+}
+
+function pull_remove_multiple_images {
+    local daemon_mode=$1
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd "${daemon_mode}"
+
+    # Because nydusd is not started right after image pull.
+    # Nydusd is started when preparing the writable active snapshot as the
+    # uppermost layer. So we must create a container to start nydusd.
+    # Then to test if snapshotter's nydusd daemons management works well
+
+    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${TOMCAT_IMAGE}"
+
+    nerdctl --snapshotter nydus create --rm --net none "${TOMCAT_IMAGE}"
+    nerdctl --snapshotter nydus create --rm --net none "${WORDPRESS_IMAGE}"
+
+    nerdctl --snapshotter nydus image rm --force "${JAVA_IMAGE}"
+    nerdctl --snapshotter nydus image rm --force "${WORDPRESS_IMAGE}"
+
+    # Deleteing with flag --async as a fuzzer
+    nerdctl --snapshotter nydus image rm --force --async "${TOMCAT_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${TOMCAT_IMAGE}"
+    nerdctl --snapshotter nydus create --net none "${TOMCAT_IMAGE}"
+
+    detect_go_race
+
+    # TODO: Validate running nydusd number
+}
+
+function start_multiple_containers_shared_daemon_fscache {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd shared fscache
+
+    nerdctl --snapshotter nydus run -d --net none "${JAVA_IMAGE}"
+    nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_IMAGE}"
+
+    detect_go_race
+}
+
+function kill_snapshotter_and_nydusd_recover {
+    local daemon_mode=$1
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd "${daemon_mode}"
+
+    nerdctl --snapshotter nydus image pull "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
+    c1=$(nerdctl --snapshotter nydus create --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus create --net none "${WORDPRESS_IMAGE}")
+
+    sleep 1
+
+    echo "killing nydusd"
+    killall -9 nydusd || true
+
+    echo "killing nydus-snapshotter"
+    killall -9 containerd-nydus-grpc || true
+
+    rm "${REMOTE_SNAPSHOTTER_SOCKET:?}"
+    containerd-nydus-grpc --config "${SNAPSHOTTER_CONFIG}" \
+        --daemon-mode "${daemon_mode}" --log-to-stdout --config-path /etc/nydus/nydusd-config.json &
+    retry ls "${REMOTE_SNAPSHOTTER_SOCKET}"
+
+    echo "start new containers"
+    nerdctl --snapshotter nydus start "$c1"
+    nerdctl --snapshotter nydus start "$c2"
+
+    detect_go_race
+}
+
+# No restart or failover recover policy. Just let snapshotter start a new nydusd when it refreshes.
+function fscache_kill_snapshotter_and_nydusd_recover {
+    local daemon_mode=$1
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd "${daemon_mode}" fscache
+
+    nerdctl --snapshotter nydus image pull "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
+    c1=$(nerdctl --snapshotter nydus create --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus create --net none "${WORDPRESS_IMAGE}")
+
+    sleep 1
+
+    echo "killing nydusd"
+    killall -9 nydusd || true
+    killall -9 containerd-nydus-grpc || true
+
+    sleep 1
+
+    rm "${REMOTE_SNAPSHOTTER_SOCKET:?}"
+    containerd-nydus-grpc --log-to-stdout --config "${SNAPSHOTTER_CONFIG}" \
+        --daemon-mode "${daemon_mode}" --fs-driver fscache --config-path /etc/nydus/nydusd-config.fscache.json &
+    retry ls "${REMOTE_SNAPSHOTTER_SOCKET}"
+
+    echo "start new containers"
+    nerdctl --snapshotter nydus start "$c1"
+    nerdctl --snapshotter nydus start "$c2"
+
+    # killall -9 nydusd
+    sleep 0.2
+    detect_go_race
+}
+
+function fscache_kill_nydusd_failover() {
+    local daemon_mode=shared
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd "${daemon_mode}" fscache failover
+
+    nerdctl --snapshotter nydus image pull "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
+    c1=$(nerdctl --snapshotter nydus create --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus create --net none "${WORDPRESS_IMAGE}")
+
+    killall -9 nydusd
+
+    echo "start new containers"
+    nerdctl --snapshotter nydus start "$c1"
+    nerdctl --snapshotter nydus start "$c2"
+
+    sleep 1
+
+    detect_go_race
+}
+
+function only_restart_snapshotter {
+    local daemon_mode=$1
+    echo "testing $FUNCNAME ${daemon_mode}"
+    nerdctl_prune_images
+    reboot_containerd "${daemon_mode}"
+
+    nerdctl --snapshotter nydus image pull "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
+    c1=$(nerdctl --snapshotter nydus create --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus create --net none "${WORDPRESS_IMAGE}")
+
+    echo "killing snapshotter"
+    killall -9 containerd-nydus-grpc || true
+
+    rm "${REMOTE_SNAPSHOTTER_SOCKET:?}"
+    containerd-nydus-grpc --config "${SNAPSHOTTER_CONFIG}" --daemon-mode \
+        "${daemon_mode}" --log-to-stdout --config-path /etc/nydus/nydusd-config.json &
+    retry ls "${REMOTE_SNAPSHOTTER_SOCKET}"
+
+    if [[ "${daemon_mode}" == "shared" ]]; then
+        validate_mnt_number 1
+    else
+        validate_mnt_number 2
+    fi
+
+    echo "start new containers"
+    nerdctl --snapshotter nydus start "$c1"
+    nerdctl --snapshotter nydus start "$c2"
+
+    detect_go_race
+}
+
+function kill_nydusd_recover_nydusd {
+    local daemon_mode=$1
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+
+    reboot_containerd "${daemon_mode}" fusedev restart
+
+    nerdctl --snapshotter nydus image pull "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
+    c1=$(nerdctl --snapshotter nydus create --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus create --net none "${WORDPRESS_IMAGE}")
+
+    pause 1
+
+    echo "killing nydusd"
+    killall -9 nydusd || true
+
+    echo "start new containers"
+    nerdctl --snapshotter nydus start "$c1"
+    nerdctl --snapshotter nydus start "$c2"
+
+    detect_go_race
+}
+
+function ctr_snapshot_usage {
+    local daemon_mode=$1
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+
+    reboot_containerd "${daemon_mode}" fusedev restart
+
+    nerdctl --snapshotter nydus image pull "${WORDPRESS_IMAGE}"
+    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
+    c1=$(nerdctl --snapshotter nydus create --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus create --net none "${WORDPRESS_IMAGE}")
+
+    pause 1
+
+    ctr snapshot --snapshotter nydus ls
+    ctr snapshot --snapshotter nydus usage
+
+    echo "start new containers"
+    nerdctl --snapshotter nydus start "$c1"
+    nerdctl --snapshotter nydus start "$c2"
+
+    ctr snapshot --snapshotter nydus ls
+    ctr snapshot --snapshotter nydus usage
+
+    detect_go_race
+}
+
+function kill_multiple_nydusd_recover_failover {
+    local daemon_mode=$1
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+
+    reboot_containerd "${daemon_mode}" fusedev failover
+
+    c1=$(nerdctl --snapshotter nydus run -d --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_IMAGE}")
+
+    pause 1
+
+    nerdctl kill "$c1" || true
+    nerdctl kill "$c2 " || true
+
+    echo "killing nydusd"
+    killall -9 nydusd || true
+
+    echo "start new containers"
+
+    c1=$(nerdctl --snapshotter nydus run -d --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_IMAGE}")
+
+    pause 1
+
+    nerdctl kill "$c1" || true
+    nerdctl kill "$c2 " || true
+
+    echo "killing nydusd again"
+    killall -9 nydusd || true
+
+    c1=$(nerdctl --snapshotter nydus run -d --net none "${JAVA_IMAGE}")
+    c2=$(nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_IMAGE}")
+
+    detect_go_race
+}
+
+# Refer to https://github.com/moby/moby/blob/088afc99e4bf8adb78e29733396182417d67ada2/hack/dind#L28-L38
+function enable_nesting_for_cgroup_v2() {
+    if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+        mkdir -p /sys/fs/cgroup/init
+        xargs -rn1 </sys/fs/cgroup/cgroup.procs >/sys/fs/cgroup/init/cgroup.procs || :
+        sed -e 's/ / +/g' -e 's/^/-/' </sys/fs/cgroup/cgroup.controllers \
+            >/sys/fs/cgroup/cgroup.subtree_control
+    fi
+}
+
+enable_nesting_for_cgroup_v2
+
+reboot_containerd multiple
+
+start_single_container_multiple_daemons
+start_multiple_containers_multiple_daemons
+start_multiple_containers_shared_daemon
+
+pull_remove_one_image
+
+pull_remove_multiple_images shared
+pull_remove_multiple_images multiple
+
+# start_single_container_on_stargz
+
+only_restart_snapshotter shared
+only_restart_snapshotter multiple
+
+kill_snapshotter_and_nydusd_recover shared
+kill_snapshotter_and_nydusd_recover multiple
+
+ctr_snapshot_usage multiple
+ctr_snapshot_usage shared
+
+if [[ $(can_erofs_ondemand_read) == 0 ]]; then
+    kill_multiple_nydusd_recover_failover multiple
+    kill_multiple_nydusd_recover_failover shared
+
+    start_multiple_containers_shared_daemon_fscache
+    fscache_kill_snapshotter_and_nydusd_recover shared
+    fscache_kill_nydusd_failover
+fi
+
+start_container_on_oci
+
+start_container_with_referrer_detect

internal/constant/values.go

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// constants of nydus snapshotter CLI config
+
+package constant
+
+const (
+	DaemonModeMultiple  string = "multiple"
+	DaemonModeDedicated string = "dedicated"
+	DaemonModeShared    string = "shared"
+	DaemonModeNone      string = "none"
+	DaemonModeInvalid   string = ""
+)
+
+const (
+	// Mount RAFS filesystem by using EROFS over block devices.
+	FsDriverBlockdev string = "blockdev"
+	// Mount RAFS filesystem by using FUSE subsystem
+	FsDriverFusedev string = "fusedev"
+	// Mount RAFS filesystem by using fscache/EROFS.
+	FsDriverFscache string = "fscache"
+	// Only prepare/supply meta/data blobs, do not mount RAFS filesystem.
+	FsDriverNodev string = "nodev"
+	// Relay layer content download operation to other agents.
+	FsDriverProxy string = "proxy"
+)
+
+const (
+	DefaultDaemonMode string = DaemonModeMultiple
+
+	DefaultFsDriver string = FsDriverFusedev
+
+	DefaultLogLevel string = "info"
+	DefaultGCPeriod string = "24h"
+
+	DefaultNydusDaemonConfigPath string = "/etc/nydus/nydusd-config.json"
+	NydusdBinaryName             string = "nydusd"
+	NydusImageBinaryName         string = "nydus-image"
+
+	DefaultRootDir                 = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+	DefaultAddress                 = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+	DefaultSystemControllerAddress = "/run/containerd-nydus/system.sock"
+
+	// Log rotation
+	DefaultDaemonRotateLogMaxSize = 100 // 100 megabytes
+	DefaultRotateLogMaxSize       = 200 // 200 megabytes
+	DefaultRotateLogMaxBackups    = 5
+	DefaultRotateLogMaxAge        = 0 // days
+	DefaultRotateLogLocalTime     = true
+	DefaultRotateLogCompress      = true
+)

internal/flags/flags.go

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2020. Ant Group. All rights reserved.
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package flags
+
+import (
+	"github.com/containerd/nydus-snapshotter/internal/constant"
+	"github.com/urfave/cli/v2"
+)
+
+type Args struct {
+	Address               string
+	NydusdConfigPath      string
+	SnapshotterConfigPath string
+	RootDir               string
+	NydusdPath            string
+	NydusImagePath        string
+	NydusOverlayFSPath    string
+	DaemonMode            string
+	FsDriver              string
+	LogLevel              string
+	LogToStdout           bool
+	LogToStdoutCount      int
+	PrintVersion          bool
+}
+
+type Flags struct {
+	Args *Args
+	F    []cli.Flag
+}
+
+func buildFlags(args *Args) []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "root",
+			Usage:       "directory to store snapshotter data and working states",
+			Destination: &args.RootDir,
+			DefaultText: constant.DefaultRootDir,
+		},
+		&cli.StringFlag{
+			Name:        "address",
+			Usage:       "remote snapshotter gRPC socket path",
+			Destination: &args.Address,
+			DefaultText: constant.DefaultAddress,
+		},
+		&cli.StringFlag{
+			Name:        "config",
+			Usage:       "path to nydus-snapshotter configuration (such as: config.toml)",
+			Destination: &args.SnapshotterConfigPath,
+		},
+		&cli.StringFlag{
+			Name:        "nydus-image",
+			Usage:       "path to `nydus-image` binary, default to search in $PATH (such as: /usr/local/bin/nydus-image)",
+			Destination: &args.NydusImagePath,
+		},
+		&cli.StringFlag{
+			Name:        "nydusd",
+			Usage:       "path to `nydusd` binary, default to search in $PATH (such as: /usr/local/bin/nydusd)",
+			Destination: &args.NydusdPath,
+		},
+		&cli.StringFlag{
+			Name:        "nydusd-config",
+			Aliases:     []string{"config-path"},
+			Usage:       "path to nydusd configuration (such as: nydusd-config.json or nydusd-config-v2.toml)",
+			Destination: &args.NydusdConfigPath,
+			DefaultText: constant.DefaultNydusDaemonConfigPath,
+		},
+		&cli.StringFlag{
+			Name:        "nydus-overlayfs-path",
+			Usage:       "path of nydus-overlayfs or name of binary from $PATH, defaults to 'nydus-overlayfs'",
+			Destination: &args.NydusOverlayFSPath,
+		},
+		&cli.StringFlag{
+			Name:        "daemon-mode",
+			Usage:       "nydusd daemon working mode, possible values: \"dedicated\", \"multiple\", \"shared\" or \"none\". \"multiple\" is an alias of \"dedicated\" and will be deprecated in v1.0",
+			Destination: &args.DaemonMode,
+			DefaultText: constant.DaemonModeMultiple,
+		},
+		&cli.StringFlag{
+			Name:        "fs-driver",
+			Usage:       "driver to mount RAFS filesystem, possible values: \"fusedev\", \"fscache\"",
+			Destination: &args.FsDriver,
+			DefaultText: constant.FsDriverFusedev,
+		},
+		&cli.StringFlag{
+			Name:        "log-level",
+			Usage:       "logging level, possible values: \"trace\", \"debug\", \"info\", \"warn\", \"error\"",
+			Destination: &args.LogLevel,
+			DefaultText: constant.DefaultLogLevel,
+		},
+		&cli.BoolFlag{
+			Name:        "log-to-stdout",
+			Usage:       "print log messages to standard output",
+			Destination: &args.LogToStdout,
+			Count:       &args.LogToStdoutCount,
+		},
+		&cli.BoolFlag{
+			Name:        "version",
+			Usage:       "print version and build information",
+			Destination: &args.PrintVersion,
+		},
+	}
+}
+
+func NewFlags() *Flags {
+	var args Args
+	return &Flags{
+		Args: &args,
+		F:    buildFlags(&args),
+	}
+}

internal/flags/flags_test.go

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package command
+package flags
 
 import (
 	"flag"
@@ -17,11 +17,12 @@ func TestNewFlags(t *testing.T) {
 	set := flag.NewFlagSet("test", 0)
 	flags := NewFlags()
 	for _, i := range flags.F {
-		i.Apply(set)
+		err := i.Apply(set)
+		assert.Nil(t, err)
 	}
-	err := set.Parse([]string{"--config-path", "/etc/testconfig", "--root", "/root"})
+	err := set.Parse([]string{"--config-path", "/etc/testconfig", "--root", "/root", "--log-level", "info"})
 	assert.Nil(t, err)
-	assert.Equal(t, flags.Args.ConfigPath, "/etc/testconfig")
+	assert.Equal(t, flags.Args.NydusdConfigPath, "/etc/testconfig")
 	assert.Equal(t, flags.Args.LogLevel, "info")
 	assert.Equal(t, flags.Args.RootDir, "/root")
 }

internal/logging/setup.go

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2020. Ant Group. All rights reserved.
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -11,9 +12,10 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	lumberjack "gopkg.in/natefinch/lumberjack.v2"
 )
 
 const (
@@ -21,7 +23,15 @@ const (
 	defaultLogFileName = "nydus-snapshotter.log"
 )
 
-func SetUp(logLevel string, logToStdout bool, logDir string, rootDir string) error {
+type RotateLogArgs struct {
+	RotateLogMaxSize    int
+	RotateLogMaxBackups int
+	RotateLogMaxAge     int
+	RotateLogLocalTime  bool
+	RotateLogCompress   bool
+}
+
+func SetUp(logLevel string, logToStdout bool, logDir string, logRotateArgs *RotateLogArgs) error {
 	lvl, err := logrus.ParseLevel(logLevel)
 	if err != nil {
 		return err
@@ -31,18 +41,24 @@ func SetUp(logLevel string, logToStdout bool, logDir string, rootDir string) err
 	if logToStdout {
 		logrus.SetOutput(os.Stdout)
 	} else {
-		if len(logDir) == 0 {
-			logDir = filepath.Join(rootDir, DefaultLogDirName)
+		if logRotateArgs == nil {
+			return errors.New("logRotateArgs is needed when logToStdout is false")
 		}
+
 		if err := os.MkdirAll(logDir, 0755); err != nil {
 			return errors.Wrapf(err, "create log dir %s", logDir)
 		}
 		logFile := filepath.Join(logDir, defaultLogFileName)
-		f, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0755)
-		if err != nil {
-			return errors.Wrapf(err, "open log file %s", logFile)
+
+		lumberjackLogger := &lumberjack.Logger{
+			Filename:   logFile,
+			MaxSize:    logRotateArgs.RotateLogMaxSize,
+			MaxBackups: logRotateArgs.RotateLogMaxBackups,
+			MaxAge:     logRotateArgs.RotateLogMaxAge,
+			Compress:   logRotateArgs.RotateLogCompress,
+			LocalTime:  logRotateArgs.RotateLogLocalTime,
 		}
-		logrus.SetOutput(f)
+		logrus.SetOutput(lumberjackLogger)
 	}
 
 	logrus.SetFormatter(&logrus.TextFormatter{

internal/logging/setup_test.go

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package logging
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/containerd/log"
+	"github.com/sirupsen/logrus"
+	"gotest.tools/assert"
+)
+
+const (
+	TestLogDirName  = "test-rotate-logs"
+	TestRootDirName = "test-root"
+)
+
+func GetRotateLogFileNumbers(testLogDir string, suffix string) int {
+	i := 0
+	err := filepath.Walk(testLogDir, func(fname string, fi os.FileInfo, _ error) error {
+		if !fi.IsDir() && strings.HasSuffix(fname, suffix) {
+			i++
+		}
+		return nil
+	})
+	if err != nil {
+		log.L.Fatal("walk path")
+	}
+	return i
+}
+
+func TestSetUp(t *testing.T) {
+	// Try to clean previously created test directory.
+	os.RemoveAll(TestLogDirName)
+
+	logRotateArgs := &RotateLogArgs{
+		RotateLogMaxSize:    1, // 1MB
+		RotateLogMaxBackups: 5,
+		RotateLogMaxAge:     0,
+		RotateLogLocalTime:  true,
+		RotateLogCompress:   true,
+	}
+	logLevel := logrus.InfoLevel.String()
+
+	err := SetUp(logLevel, true, TestLogDirName, nil)
+	assert.NilError(t, err, nil)
+
+	err = SetUp(logLevel, false, TestLogDirName, nil)
+	assert.ErrorContains(t, err, "logRotateArgs is needed when logToStdout is false")
+
+	err = SetUp(logLevel, false, TestLogDirName, logRotateArgs)
+	assert.NilError(t, err)
+	for i := 0; i < 100000; i++ { // total 9.1MB
+		log.L.Infof("test log, now: %s", time.Now().Format("2006-01-02 15:04:05"))
+	}
+	assert.Equal(t, GetRotateLogFileNumbers(TestLogDirName, "log.gz"), logRotateArgs.RotateLogMaxBackups)
+
+	os.RemoveAll(TestLogDirName)
+}

misc/example/README.md

@@ -1,4 +1,4 @@
-## Example to setup and test nydus-snapshoter
+## Example to setup and test nydus-snapshotter
 
 The directory holds a few example config files to setup a nydus POC environment that saves its all state files in `/var/lib/containerd-test`, as specified in `containerd-test-config.toml`.
 For now it contains following files:
@@ -26,8 +26,8 @@ sudo tar xzf cni-plugins-linux-amd64-$CNI_VERSION.tgz -C /opt/cni/bin/
 wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz
 tar xzf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin/
 # install nydus-overlayfs
-NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//')
-wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
+NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//')
+wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
 tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz
 sudo cp nydus-static/nydus-overlayfs /usr/local/sbin/
 ```

misc/example/containerd-config-v1.toml

@@ -1,28 +0,0 @@
-root = "/var/lib/containerd"
-state = "/run/containerd"
-oom_score = 0
-
-[grpc]
-  address = "/run/containerd/containerd.sock"
-
-[debug]
-  level = "debug"
-
-[plugins]
-  [plugins.cgroups]
-    no_prometheus = false
-  [plugins.cri]
-    [plugins.cri.containerd]
-      snapshotter = "nydus"
-      disable_snapshot_annotations = false
-    [plugins.cri.containerd.runtimes]
-      [plugins.cri.containerd.runtimes.kata]
-         runtime_type = "io.containerd.kata.v2"
-	 privileged_without_host_devices = true
-    [plugins.cri.cni]
-      bin_dir = "/opt/cni/bin"
-      conf_dir = "/etc/cni/net.d"
-[proxy_plugins]
-  [proxy_plugins.nydus]
-     type = "snapshot"
-     address = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus/containerd-nydus-grpc.sock"

misc/example/containerd-config.toml

@@ -3,9 +3,6 @@ root = "/var/lib/containerd"
 state = "/run/containerd"
 oom_score = 0
 
-[grpc]
-  address = "/run/containerd/containerd.sock"
-
 [debug]
   level = "debug"
 
@@ -24,5 +21,6 @@ oom_score = 0
 
 [proxy_plugins]
   [proxy_plugins.nydus]
-     type = "snapshot"
-     address = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus/containerd-nydus-grpc.sock"
+      type = "snapshot"
+      address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+

misc/example/containerd-test-config.toml

@@ -24,5 +24,5 @@ oom_score = 0
 
 [proxy_plugins]
   [proxy_plugins.nydus]
-     type = "snapshot"
-     address = "/var/lib/containerd-test/io.containerd.snapshotter.v1.nydus/containerd-nydus-grpc.sock"
+      type = "snapshot"
+      address = "/var/lib/containerd-test/io.containerd.snapshotter.v1.nydus/containerd-nydus-grpc.sock"

misc/example/optimizer-nri-plugin.conf

@@ -0,0 +1,13 @@
+# The directory to persist accessed files list for container.
+persist_dir = "/opt/nri/optimizer/results"
+# Whether to make the csv file human readable.
+readable = false
+# The path of optimizer server binary.
+server_path = "/usr/local/bin/optimizer-server"
+# The timeout to kill optimizer server, 0 to disable it.
+timeout = 0
+# Whether to overwrite the existed persistent files.
+overwrite = false
+# The events that containerd subscribes to.
+# Do not change this element.
+events = [ "StartContainer", "StopContainer" ]

misc/nri-prefetch/prefetchConfig.toml

@@ -0,0 +1,3 @@
+[file_prefetch]
+# This is used to configure the socket address for the file prefetch.
+socket_address = "/run/containerd-nydus/system.sock"

misc/optimizer/containerd-config.toml

@@ -0,0 +1,26 @@
+#   Copyright 2018-2022 Docker Inc.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+version = 2
+
+[proxy_plugins]
+  [proxy_plugins.nydus]
+    type = "snapshot"
+    address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+
+[plugins."io.containerd.nri.v1.nri"]
+  config_file = "/etc/nri/nri.conf"
+  disable = false
+  plugin_path = "/opt/nri/plugins"
+  socket_path = "/var/run/nri.sock"

misc/optimizer/crictl.yaml

@@ -0,0 +1,5 @@
+runtime-endpoint: unix:///run/containerd/containerd.sock
+image-endpoint: unix:////run/containerd/containerd.sock
+timeout: 2
+debug: false
+pull-image-on-create: false

misc/optimizer/nginx.yaml

@@ -0,0 +1,18 @@
+metadata:
+  name: nginx
+
+image:
+  image: nginx:1.23.3
+
+mounts:
+  - host_path: script
+    container_path: /script
+
+command:
+  - /script/entrypoint.sh
+
+args:
+ - /script/file_list.txt
+
+log_path: nginx.0.log
+linux: {}

misc/optimizer/sandbox.yaml

@@ -0,0 +1,7 @@
+metadata:
+  name: nginx-sandbox
+  namespace: default
+  attempt: 1
+  uid: hdishd83djaidwnduwk28bcsb
+log_directory: /tmp
+linux: {}

misc/optimizer/script/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+path=$1
+
+default_path=file_list_path.txt
+if [[ $# -eq 0 ]]; then
+    path=${default_path}
+fi
+
+files=($(cat ${path} | tr "\n" " "))
+files_number=${#files[@]}
+echo "file number: $files_number"
+
+for file in "${files[@]}"; do
+    file_size=$(stat -c%s "${file}")
+    echo "file: ${file} size: ${file_size}"
+    cat ${file} >/dev/null
+done
+
+echo "Read file list done."

misc/optimizer/script/file_list.txt

@@ -0,0 +1,12 @@
+/lib/x86_64-linux-gnu/ld-2.31.so
+/lib/x86_64-linux-gnu/libc-2.31.so
+/lib/x86_64-linux-gnu/libtinfo.so.6.2
+/lib/x86_64-linux-gnu/libdl-2.31.so
+/lib/x86_64-linux-gnu/libnss_files-2.31.so
+/lib/x86_64-linux-gnu/libselinux.so.1
+/usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.10.1
+/lib/x86_64-linux-gnu/libpthread-2.31.so
+/docker-entrypoint.sh
+/docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
+/docker-entrypoint.d/20-envsubst-on-templates.sh
+/docker-entrypoint.d/30-tune-worker-processes.sh

misc/snapshotter/Dockerfile

@@ -1,22 +1,46 @@
-FROM ubuntu:20.04 AS sourcer
+FROM alpine:3.17.0 AS base
 
-RUN apt update; apt install --no-install-recommends -y curl wget ca-certificates
-RUN export NYDUS_VERSION=$(curl --silent "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")'); \
-    wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VERSION/nydus-static-$NYDUS_VERSION-linux-amd64.tgz; \
-    tar xzf nydus-static-$NYDUS_VERSION-linux-amd64.tgz
-RUN mv nydus-static/* /; mv nydusd-fusedev nydusd
+FROM base AS sourcer
+ARG TARGETARCH
+ARG NYDUS_VER=v2.3.0
 
-FROM ubuntu:20.04
+RUN apk add -q --no-cache curl && \
+    apk add -q --no-cache --upgrade grep && \
+    curl -fsSL -O https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz && \
+    echo $NYDUS_VER > /.nydus_version && \
+    tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz && \
+    rm nydus-static-$NYDUS_VER-linux-amd64.tgz && \
+    mv nydus-static/* / \
+    && rm -rf /nydus-overlayfs
+
+FROM base AS kubectl-sourcer
+ARG TARGETARCH
+
+RUN apk add -q --no-cache curl && \
+    curl -fsSL -o /usr/bin/kubectl https://dl.k8s.io/release/"$(curl -L -s https://dl.k8s.io/release/stable.txt)"/bin/linux/"$TARGETARCH"/kubectl && \
+    chmod +x /usr/bin/kubectl
+
+FROM base
+ARG DESTINATION=/opt/nydus-artifacts
+ARG CONFIG_DESTINATION=${DESTINATION}/etc/nydus
+ARG BINARY_DESTINATION=${DESTINATION}/usr/local/bin
+ARG SCRIPT_DESTINATION=${DESTINATION}/opt/nydus
 
 WORKDIR /root/
-RUN mkdir -p /usr/local/bin/ /etc/nydus/ /var/lib/nydus/cache/ /tmp/blobs/
-COPY --from=sourcer /nydusd /nydus-image /usr/local/bin/
-COPY containerd-nydus-grpc /usr/local/bin/
-COPY nydusd-config.fusedev.json /etc/nydus/config.json
-COPY nydusd-config-localfs.json /etc/nydus/localfs.json
-RUN apt update && \
-    apt install --no-install-recommends -y ca-certificates && \
-    rm -rf /var/cache/apt/* /var/lib/apt/lists/*
-COPY entrypoint.sh /
+RUN apk add -q --no-cache libc6-compat bash
 
-ENTRYPOINT ["/entrypoint.sh"]
+VOLUME /var/lib/containerd/io.containerd.snapshotter.v1.nydus
+VOLUME /run/containerd-nydus
+
+COPY --from=sourcer /.nydus_version /.nydus_version
+COPY --from=kubectl-sourcer /usr/bin/kubectl /usr/bin/kubectl
+
+RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache /tmp/blobs/
+COPY --from=sourcer /nydus* ${BINARY_DESTINATION}/
+COPY --chmod=755 containerd-nydus-grpc nydus-overlayfs ${BINARY_DESTINATION}/
+COPY --chmod=755 snapshotter.sh ${SCRIPT_DESTINATION}/snapshotter.sh
+COPY nydusd-config.fusedev.json ${CONFIG_DESTINATION}/nydusd-fusedev.json
+COPY nydusd-config-localfs.json ${CONFIG_DESTINATION}/nydusd-localfs.json
+COPY nydusd-config.fscache.json ${CONFIG_DESTINATION}/nydusd-fscache.json
+COPY config.toml ${CONFIG_DESTINATION}/config.toml
+COPY nydus-snapshotter.service ${DESTINATION}/etc/systemd/system/nydus-snapshotter.service

misc/snapshotter/base/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+- nydus-snapshotter.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+

misc/snapshotter/base/nydus-snapshotter.yaml

@@ -0,0 +1,140 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
+  namespace: nydus-system
+data:
+  FS_DRIVER: "fusedev"
+  ENABLE_CONFIG_FROM_VOLUME: "false"
+  ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false"
+  ENABLE_SYSTEMD_SERVICE: "true"
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+  labels:
+    app: nydus-snapshotter
+spec:
+  selector:
+    matchLabels:
+      app: nydus-snapshotter
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        app: nydus-snapshotter
+
+    spec:
+      serviceAccountName: nydus-snapshotter-sa
+      hostNetwork: true
+      hostPID: true
+      containers:
+      - name: nydus-snapshotter
+        image: "ghcr.io/containerd/nydus-snapshotter:latest"
+        imagePullPolicy: Always
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: FS_DRIVER
+            valueFrom:
+              configMapKeyRef:
+                name: nydus-snapshotter-configs
+                key: FS_DRIVER
+                optional: true
+          - name: ENABLE_CONFIG_FROM_VOLUME
+            valueFrom:
+              configMapKeyRef:
+                name: nydus-snapshotter-configs
+                key: ENABLE_CONFIG_FROM_VOLUME
+                optional: true 
+          - name: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER
+            valueFrom:
+              configMapKeyRef:
+                name: nydus-snapshotter-configs
+                key: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER
+                optional: true
+          - name: ENABLE_SYSTEMD_SERVICE
+            valueFrom:
+              configMapKeyRef:
+                name: nydus-snapshotter-configs
+                key: ENABLE_SYSTEMD_SERVICE
+                optional: true                    
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - "bash"
+              - "-c"
+              - |
+                /opt/nydus-artifacts/opt/nydus/snapshotter.sh cleanup
+        command: 
+         - bash
+         - -c
+         - |-
+           /opt/nydus-artifacts/opt/nydus/snapshotter.sh deploy
+        volumeMounts:
+        - name: config-volume
+          mountPath: "/etc/nydus-snapshotter"
+        - name: nydus-lib
+          mountPath: "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+          mountPropagation: Bidirectional
+        - name: nydus-run
+          mountPath: "/run/containerd-nydus"
+          mountPropagation: Bidirectional
+        - name: nydus-opt
+          mountPath: "/opt/nydus"
+          mountPropagation: Bidirectional
+        - name: nydus-etc
+          mountPath: "/etc/nydus"
+          mountPropagation: Bidirectional     
+        - name: containerd-conf
+          mountPath: "/etc/containerd/"
+        - name: local-bin
+          mountPath: "/usr/local/bin/"
+        - name: etc-systemd-system
+          mountPath: "/etc/systemd/system/"
+        securityContext:
+          privileged: true
+
+      volumes:
+      - name: config-volume
+        configMap:
+          name: nydus-snapshotter-configs
+          optional: true
+      - name: nydus-run
+        hostPath:
+          path: /run/containerd-nydus
+          type: DirectoryOrCreate
+      - name: nydus-lib
+        hostPath:
+          path: /var/lib/containerd/io.containerd.snapshotter.v1.nydus
+          type: DirectoryOrCreate
+      - name: nydus-opt
+        hostPath:
+          path: /opt/nydus
+          type: DirectoryOrCreate
+      - name: nydus-etc
+        hostPath:
+          path: /etc/nydus
+          type: DirectoryOrCreate
+      - name: containerd-conf
+        hostPath:
+          path: /etc/containerd/
+      - name: local-bin
+        hostPath:
+          path: /usr/local/bin/
+      - name: etc-systemd-system
+        hostPath:
+          path: /etc/systemd/system/
+      

misc/snapshotter/config-blockdev.toml

@@ -0,0 +1,39 @@
+version = 1
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+# No nydusd daemon needed
+daemon_mode = "none"
+
+[daemon]
+# Use `blockdev` for tarfs
+fs_driver = "blockdev"
+# Path to nydus-image binary
+nydusimage_path = "/usr/local/bin/nydus-image"
+
+[remote]
+skip_ssl_verify = true
+
+[snapshot]
+# Insert Kata volume information to `Mount.Options`
+enable_kata_volume = true
+
+[experimental.tarfs]
+# Whether to enable nydus tarfs mode. Tarfs is supported by:
+# - The EROFS filesystem driver since Linux 6.4
+# - Nydus Image Service release v2.3
+enable_tarfs = true
+
+# Mount rafs on host by loopdev and EROFS
+mount_tarfs_on_host = false
+
+# Mode to export tarfs images:
+# - "none" or "": do not export tarfs
+# - "layer_verity_only": only generate disk verity information for a layer blob
+# - "image_verity_only": only generate disk verity information for all blobs of an image
+# - "layer_block": generate a raw block disk image with tarfs for a layer
+# - "image_block": generate a raw block disk image with tarfs for an image
+# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
+# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
+export_mode = "image_block_with_verity"

misc/snapshotter/config-proxy.toml

@@ -0,0 +1,15 @@
+version = 1
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+# No nydusd daemon needed
+daemon_mode = "none"
+
+[daemon]
+# Enable proxy mode
+fs_driver = "proxy"
+
+[snapshot]
+# Insert Kata volume information to `Mount.Options`
+enable_kata_volume = true

misc/snapshotter/config.toml

@@ -0,0 +1,137 @@
+version = 1
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+# The nydus daemon mode can be one of the following options: multiple, dedicated, shared, or none. 
+# If `daemon_mode` option is not specified, the default value is multiple.
+daemon_mode = "dedicated"
+# Whether snapshotter should try to clean up resources when it is closed
+cleanup_on_close = false
+
+[system]
+# Snapshotter's debug and trace HTTP server interface
+enable = true
+# Unix domain socket path where system controller is listening on
+address = "/run/containerd-nydus/system.sock"
+
+[system.debug]
+# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started.
+# This option specifies the profile duration when nydusd is downloading and uncomproessing data.
+daemon_cpu_profile_duration_secs = 5
+# Enable by assigning an address, empty indicates pprof server is disabled
+pprof_address = ""
+
+[daemon]
+# Specify a configuration file for nydusd
+nydusd_config = "/etc/nydus/nydusd-config.fusedev.json"
+nydusd_path = "/usr/local/bin/nydusd"
+nydusimage_path = "/usr/local/bin/nydus-image"
+# The fs driver can be one of the following options: fusedev, fscache, blockdev, proxy, or nodev. 
+# If `fs_driver` option is not specified, the default value is fusedev.
+fs_driver = "fusedev"
+# How to process when daemon dies: "none", "restart" or "failover"
+recover_policy = "restart"
+# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024].
+# Setting to 0 will use the default configuration of nydusd.
+threads_number = 4
+# Log rotation size for nydusd, in unit MB(megabytes). (default 100MB)
+log_rotation_size = 100
+
+[cgroup]
+# Whether to use separate cgroup for nydusd.
+enable = true
+# The memory limit for nydusd cgroup, which contains all nydusd processes.
+# Percentage is supported as well, please ensure it is end with "%".
+# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%".
+memory_limit = ""
+
+[log]
+# Print logs to stdout rather than logging files
+log_to_stdout = false
+# Snapshotter's log level
+level = "info"
+log_rotation_compress = true
+log_rotation_local_time = true
+# Max number of days to retain logs
+log_rotation_max_age = 7
+log_rotation_max_backups = 5
+# In unit MB(megabytes)
+log_rotation_max_size = 100
+
+[metrics]
+# Enable by assigning an address, empty indicates metrics server is disabled
+address = ":9110"
+
+[remote]
+convert_vpc_registry = false
+
+[remote.mirrors_config]
+# Snapshotter will overwrite daemon's mirrors configuration
+# if the values loaded from this driectory are not null before starting a daemon.
+# Set to "" or an empty directory to disable it.
+#dir = "/etc/nydus/certs.d"
+
+[remote.auth]
+# Fetch the private registry auth by listening to K8s API server
+enable_kubeconfig_keychain = false
+# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`)
+kubeconfig_path = ""
+# Fetch the private registry auth as CRI image service proxy
+enable_cri_keychain = false
+# the target image service when using image proxy
+#image_service_address = "/run/containerd/containerd.sock"
+
+[snapshot]
+# Let containerd use nydus-overlayfs mount helper
+enable_nydus_overlayfs = false
+# Path to the nydus-overlayfs binary or name of the binary in $PATH
+nydus_overlayfs_path = "nydus-overlayfs"
+# Insert Kata Virtual Volume option to `Mount.Options`
+enable_kata_volume = false
+# Whether to remove resources when a snapshot is removed
+sync_remove = false
+
+[cache_manager]
+# Disable or enable recyclebin
+disable = false
+# How long to keep deleted files in recyclebin
+gc_period = "24h"
+# Directory to host cached files
+cache_dir = ""
+
+[image]
+public_key_file = ""
+validate_signature = false
+
+# The configuraions for features that are not production ready
+[experimental]
+# Whether to enable stargz support
+enable_stargz = false
+# Whether to enable referrers support
+# The option enables trying to fetch the Nydus image associated with the OCI image and run it.
+# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers
+enable_referrer_detect = false
+# Whether to enable authentication support
+# The option enables nydus snapshot to provide backend information to nydusd.
+enable_backend_source = false
+[experimental.tarfs]
+# Whether to enable nydus tarfs mode. Tarfs is supported by:
+# - The EROFS filesystem driver since Linux 6.4
+# - Nydus Image Service release v2.3
+enable_tarfs = false
+# Mount rafs on host by loopdev and EROFS
+mount_tarfs_on_host = false
+# Only enable nydus tarfs mode for images with `tarfs hint` label when true
+tarfs_hint = false
+# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default
+max_concurrent_proc = 0
+# Mode to export tarfs images:
+# - "none" or "": do not export tarfs
+# - "layer_verity_only": only generate disk verity information for a layer blob
+# - "image_verity_only": only generate disk verity information for all blobs of an image
+# - "layer_block": generate a raw block disk image with tarfs for a layer
+# - "image_block": generate a raw block disk image with tarfs for an image
+# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
+# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
+export_mode = ""

misc/snapshotter/entrypoint.sh

@@ -1,20 +0,0 @@
-#! /bin/bash
-
-CONTAINERD_ROOT="${CONTAINERD_ROOT:-/var/lib/containerd/}"
-
-set -eu
-BACKEND_TYPE="${BACKEND_TYPE:-config}"
-if [ "$#" -eq 0 ]; then
-	containerd-nydus-grpc \
-	    --log-level trace \
-	    --nydusd-path /usr/local/bin/nydusd \
-	    --config-path /etc/nydus/${BACKEND_TYPE}.json \
-	    --root ${CONTAINERD_ROOT}/io.containerd.snapshotter.v1.nydus \
-	    --address ${CONTAINERD_ROOT}/io.containerd.snapshotter.v1.nydus/containerd-nydus-grpc.sock \
-	    --enable-nydus-overlayfs \
-	    --daemon-mode shared \
-			--enable-stargz \
-	    --log-to-stdout
-fi
-
-exec $@

misc/snapshotter/nydus-snapshotter-rbac.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nydus-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nydus-snapshotter-sa
+  namespace: nydus-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nydus-snapshotter-role
+subjects:
+- kind: ServiceAccount
+  name: nydus-snapshotter-sa
+  namespace: nydus-system

misc/snapshotter/nydus-snapshotter.fscache.service

@@ -6,7 +6,7 @@ Before=containerd.service
 [Service]
 Type=simple
 Environment=HOME=/root
-ExecStart=/usr/local/bin/containerd-nydus-grpc --fs-driver fscache --daemon-mode shared --config-path /etc/nydus/config.json
+ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config.toml --fs-driver fscache --nydusd-config /etc/nydus/nydusd-config.fscache.json
 Restart=always
 RestartSec=1
 KillMode=process

misc/snapshotter/nydus-snapshotter.fusedev.service

@@ -6,7 +6,7 @@ Before=containerd.service
 [Service]
 Type=simple
 Environment=HOME=/root
-ExecStart=/usr/local/bin/containerd-nydus-grpc --config-path /etc/nydus/config.json
+ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config.toml
 Restart=always
 RestartSec=1
 KillMode=process

misc/snapshotter/nydus-snapshotter.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=nydus snapshotter
+After=network.target
+Before=containerd.service
+
+[Service]
+Type=simple
+Environment=HOME=/root
+ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config-proxy.toml
+Restart=always
+RestartSec=1
+KillMode=process
+OOMScoreAdjust=-999
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

misc/snapshotter/nydusd-config-localfs.json

@@ -1,25 +1,22 @@
 {
-    "device": {
-      "backend": {
-        "type": "localfs",
-        "config": {
-            "dir": "/tmp/blobs/"
-        }
-      },
-      "cache": {
-        "type": "blobcache",
-        "config": {
-          "work_dir": "/var/lib/nydus/cache"
-        }
+  "device": {
+    "backend": {
+      "type": "localfs",
+      "config": {
+        "dir": "/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/"
       }
     },
-    "mode": "direct",
-    "digest_validate": false,
-    "iostats_files": false,
-    "enable_xattr": true,
-    "fs_prefetch": {
-      "enable": true,
-      "threads_count": 2
+    "cache": {
+      "type": "blobcache"
     }
+  },
+  "mode": "direct",
+  "digest_validate": false,
+  "iostats_files": false,
+  "enable_xattr": true,
+  "amplify_io": 1048576,
+  "fs_prefetch": {
+    "enable": true,
+    "threads_count": 2
   }
-  
+}

misc/snapshotter/nydusd-config.fscache.json

@@ -2,9 +2,12 @@
     "type": "bootstrap",
     "config": {
         "backend_type": "registry",
-        "backend_config": {
-            "scheme": "https"
-        },
-        "cache_type": "fscache"
+        "backend_config": {},
+        "cache_type": "fscache",
+        "prefetch_config": {
+            "enable": true,
+            "threads_count": 8,
+            "merging_size": 1048576
+        }
     }
-}
+}

misc/snapshotter/nydusd-config.fusedev.json

@@ -3,27 +3,24 @@
     "backend": {
       "type": "registry",
       "config": {
-        "scheme": "https",
         "timeout": 5,
         "connect_timeout": 5,
         "retry_limit": 2
       }
     },
     "cache": {
-      "type": "blobcache",
-      "config": {
-        "work_dir": "/var/lib/nydus/cache"
-      }
+      "type": "blobcache"
     }
   },
   "mode": "direct",
   "digest_validate": false,
   "iostats_files": false,
   "enable_xattr": true,
+  "amplify_io": 1048576,
   "fs_prefetch": {
     "enable": true,
     "threads_count": 8,
     "merging_size": 1048576,
     "prefetch_all": true
   }
-}
+}

misc/snapshotter/overlays/k3s/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+patches:
+- path: mount_k3s_conf.yaml

misc/snapshotter/overlays/k3s/mount_k3s_conf.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+spec:
+  template:
+    spec:
+      volumes:
+        - name: containerd-conf
+          hostPath:
+            path: /var/lib/rancher/k3s/agent/etc/containerd/

misc/snapshotter/overlays/rke2/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+patches:
+- path: mount_rke2_conf.yaml

misc/snapshotter/overlays/rke2/mount_rke2_conf.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+spec:
+  template:
+    spec:
+      volumes:
+        - name: containerd-conf
+          hostPath:
+            path: /var/lib/rancher/rke2/agent/etc/containerd/

misc/snapshotter/snapshotter.sh

@@ -0,0 +1,347 @@
+#!/usr/bin/env bash
+# Copyright (c) 2023. Nydus Developers. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+SNAPSHOTTER_ARTIFACTS_DIR="/opt/nydus-artifacts"
+
+# Container runtime config, the default container runtime is containerd
+CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}"
+CONTAINER_RUNTIME_CONFIG="/etc/containerd/config.toml"
+
+# Common nydus snapshotter config options
+FS_DRIVER="${FS_DRIVER:-fusedev}"
+SNAPSHOTTER_GRPC_SOCKET="${SNAPSHOTTER_GRPC_SOCKET:-/run/containerd-nydus/containerd-nydus-grpc.sock}"
+
+# The directory about nydus and nydus snapshotter
+NYDUS_CONFIG_DIR="${NYDUS_CONFIG_DIR:-/etc/nydus}"
+NYDUS_LIB_DIR="${NYDUS_LIB_DIR:-/var/lib/containerd/io.containerd.snapshotter.v1.nydus}"
+NYDUS_BINARY_DIR="${NYDUS_BINARY_DIR:-/usr/local/bin}"
+SNAPSHOTTER_SCRYPT_DIR="${SNAPSHOTTER_SCRYPT_DIR:-/opt/nydus}"
+
+# The binary about nydus-snapshotter
+SNAPSHOTTER_BINARY="${SNAPSHOTTER_BINARY:-${NYDUS_BINARY_DIR}/containerd-nydus-grpc}"
+
+# The config about nydus snapshotter
+SNAPSHOTTER_CONFIG="${SNAPSHOTTER_CONFIG:-${NYDUS_CONFIG_DIR}/config.toml}"
+# The systemd service config about nydus snapshotter
+SNAPSHOTTER_SERVICE="${SNAPSHOTTER_SERVICE:-/etc/systemd/system/nydus-snapshotter.service}"
+# If true, the script would read the config from env.
+ENABLE_CONFIG_FROM_VOLUME="${ENABLE_CONFIG_FROM_VOLUME:-false}"
+# If true, the script would enable the "runtime specific snapshotter" in containerd config.
+ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER="${ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER:-false}"
+# If true, the snapshotter would be running as a systemd service
+ENABLE_SYSTEMD_SERVICE="${ENABLE_SYSTEMD_SERVICE:-false}"
+
+COMMANDLINE=""
+
+# If we fail for any reason a message will be displayed
+die() {
+    msg="$*"
+    echo "ERROR: $msg" >&2
+    exit 1
+}
+
+print_usage() {
+    echo "Usage: $0 [deploy/cleanup]"
+}
+
+wait_service_active(){
+    local wait_time="$1"
+    local sleep_time="$2"
+    local service="$3"
+
+    nsenter -t 1 -m systemctl restart $service
+
+    # Wait for containerd to be running
+    while [ "$wait_time" -gt 0 ]; do
+        if nsenter -t 1 -m systemctl is-active --quiet $service; then
+            echo "$service is running"
+            return 0  
+        else
+            sleep "$sleep_time"
+            wait_time=$((wait_time-sleep_time))
+        fi
+    done
+
+    echo "Timeout reached. $service may not be running."
+    nsenter -t 1 -m systemctl status $service
+    return 1  
+}
+
+function fs_driver_handler() {
+    if [ "${ENABLE_CONFIG_FROM_VOLUME}" == "true" ]; then
+        SNAPSHOTTER_CONFIG="${NYDUS_CONFIG_DIR}/config.toml"
+    else
+        case "${FS_DRIVER}" in
+        fusedev) 
+            sed -i -e "s|nydusd_config = .*|nydusd_config = \"${NYDUS_CONFIG_DIR}/nydusd-fusedev.json\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|fs_driver = .*|fs_driver = \"fusedev\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|daemon_mode = .*|daemon_mode = \"multiple\"|" "${SNAPSHOTTER_CONFIG}" 
+            ;;
+        fscache) 
+            sed -i -e "s|nydusd_config = .*|nydusd_config = \"${NYDUS_CONFIG_DIR}/nydusd-fscache.json\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|fs_driver = .*|fs_driver = \"fscache\"|" "${SNAPSHOTTER_CONFIG}"
+            sed -i -e "s|daemon_mode = .*|daemon_mode = \"multiple\"|" "${SNAPSHOTTER_CONFIG}"  
+            ;;
+        blockdev) 
+            sed -i -e "s|fs_driver = .*|fs_driver = \"blockdev\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|enable_kata_volume = .*|enable_kata_volume = true|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|enable_tarfs = .*|enable_tarfs = true|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|daemon_mode = .*|daemon_mode = \"none\"|" "${SNAPSHOTTER_CONFIG}"  
+            sed -i -e "s|export_mode = .*|export_mode = \"layer_block_with_verity\"|" "${SNAPSHOTTER_CONFIG}"  
+            ;;
+        proxy) 
+            sed -i -e "s|fs_driver = .*|fs_driver = \"proxy\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|enable_kata_volume = .*|enable_kata_volume = true|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|daemon_mode = .*|daemon_mode = \"none\"|" "${SNAPSHOTTER_CONFIG}"  
+            ;;
+        *) die "invalid fs driver ${FS_DRIVER}" ;;
+        esac
+    fi
+    COMMANDLINE+=" --config ${SNAPSHOTTER_CONFIG}"
+}
+
+function configure_snapshotter() {
+
+    echo "configuring snapshotter"
+
+    # Copy the container runtime config to a backup
+    cp "$CONTAINER_RUNTIME_CONFIG" "$CONTAINER_RUNTIME_CONFIG".bak.nydus
+
+
+    # When trying to edit the config file that is mounted by docker with `sed -i`, the error would happend:
+    # sed: cannot rename /etc/containerd/config.tomlpmdkIP: Device or resource busy  
+    # The reason is that `sed`` with option `-i` creates new file, and then replaces the old file with the new one, 
+    # which definitely will change the file inode. But the file is mounted by docker, which means we are not allowed to 
+    # change its inode from within docker container.
+    # 
+    # So we copy the original file to a backup, make changes to the backup, and then overwrite the original file with the backup.
+    cp "$CONTAINER_RUNTIME_CONFIG" "$CONTAINER_RUNTIME_CONFIG".bak
+    # Check and add nydus proxy plugin in the config
+    if grep -q '\[proxy_plugins.nydus\]' "$CONTAINER_RUNTIME_CONFIG".bak; then
+        echo "the config has configured the nydus proxy plugin!"
+    else
+        echo "Not found nydus proxy plugin!"
+        cat <<EOF >>"$CONTAINER_RUNTIME_CONFIG".bak
+        
+    [proxy_plugins.nydus]
+        type = "snapshot"
+        address = "$SNAPSHOTTER_GRPC_SOCKET"
+EOF
+    fi
+
+    if grep -q 'disable_snapshot_annotations' "$CONTAINER_RUNTIME_CONFIG".bak; then
+        sed -i -e "s|disable_snapshot_annotations = .*|disable_snapshot_annotations = false|" \
+                "${CONTAINER_RUNTIME_CONFIG}".bak
+    else
+        sed -i '/\[plugins\..*\.containerd\]/a\disable_snapshot_annotations = false' \
+                "${CONTAINER_RUNTIME_CONFIG}".bak
+    fi
+    if grep -q 'discard_unpacked_layers' "$CONTAINER_RUNTIME_CONFIG".bak; then
+        sed -i -e "s|discard_unpacked_layers = .*|discard_unpacked_layers = false|" \
+                "${CONTAINER_RUNTIME_CONFIG}".bak
+    else
+        sed -i '/\[plugins\..*\.containerd\]/a\discard_unpacked_layers = false' \
+                "${CONTAINER_RUNTIME_CONFIG}".bak
+    fi
+
+    if [ "${ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER}" == "false" ]; then
+        sed -i -e '/\[plugins\..*\.containerd\]/,/snapshotter =/ s/snapshotter = "[^"]*"/snapshotter = "nydus"/' "${CONTAINER_RUNTIME_CONFIG}".bak
+    fi
+    
+    cat "${CONTAINER_RUNTIME_CONFIG}".bak >  "${CONTAINER_RUNTIME_CONFIG}"
+}
+
+function install_snapshotter() {
+    echo "install nydus snapshotter artifacts"
+    find "${SNAPSHOTTER_ARTIFACTS_DIR}${NYDUS_BINARY_DIR}" -type f -exec install -Dm 755 -t "${NYDUS_BINARY_DIR}" "{}"  \;
+    find "${SNAPSHOTTER_ARTIFACTS_DIR}${NYDUS_CONFIG_DIR}" -type f -exec install -Dm 644 -t "${NYDUS_CONFIG_DIR}" "{}"  \;
+    install -D -m 644 "${SNAPSHOTTER_ARTIFACTS_DIR}${SNAPSHOTTER_SCRYPT_DIR}/snapshotter.sh" "${SNAPSHOTTER_SCRYPT_DIR}/snapshotter.sh"
+    if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then
+        install -D -m 644 "${SNAPSHOTTER_ARTIFACTS_DIR}${SNAPSHOTTER_SERVICE}" "${SNAPSHOTTER_SERVICE}"
+    fi
+    if [ "${ENABLE_CONFIG_FROM_VOLUME}" == "true" ]; then
+        find "/etc/nydus-snapshotter" -type f -exec install -Dm 644 -t "${NYDUS_CONFIG_DIR}" "{}"  \;
+    fi
+}
+
+function deploy_snapshotter() {
+    echo "deploying snapshotter"
+    install_snapshotter
+
+    COMMANDLINE="${SNAPSHOTTER_BINARY}"
+    fs_driver_handler
+    configure_snapshotter
+    if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then
+        echo "running snapshotter as systemd service"
+        sed -i "s|^ExecStart=.*$|ExecStart=$COMMANDLINE|" "${SNAPSHOTTER_SERVICE}"
+        nsenter -t 1 -m systemctl daemon-reload
+        nsenter -t 1 -m systemctl enable nydus-snapshotter.service
+        wait_service_active 30 5 nydus-snapshotter
+    else
+        echo "running snapshotter as standalone process"
+        ${COMMANDLINE} &
+    fi
+    wait_service_active 30 5 ${CONTAINER_RUNTIME}
+
+}
+
+function remove_images() {
+    local SNAPSHOTTER="nydus"
+    local NAMESPACE="k8s.io"
+    local ctr_args="nsenter -t 1 -m ctr"
+
+    if [[ " k3s k3s-agent rke2-agent rke2-server " =~ " ${CONTAINER_RUNTIME} " ]]; then
+        ctr_args+=" --address /run/k3s/containerd/containerd.sock"
+    fi
+    ctr_args+=" --namespace $NAMESPACE"
+
+    # List all snapshots for nydus snapshotter
+    local SNAPSHOTS=$($ctr_args snapshot --snapshotter $SNAPSHOTTER ls | awk 'NR>1 {print $1}')
+    echo "Images associated with snapshotter $SNAPSHOTTER:"
+
+    # Loop through each snapshot and find associated contents
+    for SNAPSHOT in $SNAPSHOTS; do
+        local CONTENTS=$($ctr_args content ls | grep $SNAPSHOT | awk '{print $1}')
+        echo "Snapshot: $SNAPSHOT, Contents: $CONTENTS"
+        if [ -z "$CONTENTS" ]; then
+            continue
+        fi
+        # Loop through each content and find associated digests of images
+        for CONTENT in $CONTENTS; do
+            local DIGESTS=$($ctr_args image ls | grep $CONTENT | awk '{print $3}')
+            echo "Content: $CONTENT, Digests: $DIGESTS"
+            if [ -z "$DIGESTS" ]; then
+                continue
+            fi
+            # Loop through each digest and find associated image references
+            for DIGEST in $DIGESTS; do
+                local IMAGES=$($ctr_args image ls | grep $DIGEST | awk '{print $1}')
+                echo "Digest: $DIGEST, Images: $IMAGES"
+                if [ -z "$IMAGES" ]; then
+                    continue
+                fi
+                for IMAGE in $IMAGES; do
+                    # Delete the image
+                    $ctr_args images rm $IMAGE > /dev/null 2>&1 || true
+                    echo "Image $IMAGE removed"
+                done
+            done
+            # Delete the content
+            $ctr_args content rm $CONTENT > /dev/null 2>&1 || true
+            echo "content $CONTENT removed"
+        done
+        # Delete the snapshot
+        $ctr_args snapshot --snapshotter $SNAPSHOTTER rm $SNAPSHOT > /dev/null 2>&1 || true
+        echo "snapshot $SNAPSHOT removed"
+    done
+    echo "INFO: Images removed"
+}
+
+function cleanup_snapshotter() {
+    echo "cleaning up snapshotter"
+
+    pid=$(ps -ef | grep containerd-nydus-grpc | grep -v grep | awk '{print $1}' || true)
+    if [ ! -z "$pid" ]; then
+        remove_images
+    fi
+    echo "Recover containerd config"
+    cat "$CONTAINER_RUNTIME_CONFIG".bak.nydus >"$CONTAINER_RUNTIME_CONFIG"
+    if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then
+        nsenter -t 1 -m systemctl stop nydus-snapshotter.service
+        nsenter -t 1 -m systemctl disable --now nydus-snapshotter.service
+        rm -f "${SNAPSHOTTER_SERVICE}" || true
+    else
+        kill -9 $pid || true
+    fi
+    wait_service_active 30 5 ${CONTAINER_RUNTIME}
+    echo "Removing nydus-snapshotter artifacts from host"
+    rm -f "${NYDUS_BINARY_DIR}"/nydus*
+    rm -rf "${NYDUS_CONFIG_DIR}"/*
+    rm -rf "${SNAPSHOTTER_SCRYPT_DIR}"/*
+    rm -rf "${NYDUS_LIB_DIR}"/*
+    echo "cleaned up snapshotter"
+}
+
+function get_container_runtime() {
+    local runtime=$(kubectl get node ${NODE_NAME} -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}')
+    if [ "$?" -ne 0 ]; then
+        die "\"$NODE_NAME\" is an invalid node name"
+    fi
+
+    if echo "$runtime" | grep -qE 'containerd.*-k3s'; then
+        if nsenter -t 1 -m systemctl is-active --quiet rke2-agent; then
+            echo "rke2-agent"
+        elif nsenter -t 1 -m systemctl is-active --quiet rke2-server; then
+            echo "rke2-server"
+        elif nsenter -t 1 -m systemctl is-active --quiet k3s-agent; then
+            echo "k3s-agent"
+        else
+            echo "k3s"
+        fi
+    elif nsenter -t 1 -m systemctl is-active --quiet k0scontroller; then
+        echo "k0s-controller"
+    elif nsenter -t 1 -m systemctl is-active --quiet k0sworker; then
+        echo "k0s-worker"
+    else
+        echo "$runtime" | awk -F '[:]' '{print $1}'
+    fi
+}
+
+function main() {
+    # script requires that user is root
+    euid=$(id -u)
+    if [[ $euid -ne 0 ]]; then
+        die "This script must be run as root"
+    fi
+
+    CONTAINER_RUNTIME=$(get_container_runtime)
+    if [[ " k3s k3s-agent rke2-agent rke2-server " =~ " ${CONTAINER_RUNTIME} " ]]; then
+        CONTAINER_RUNTIME_CONFIG_TMPL="${CONTAINER_RUNTIME_CONFIG}.tmpl"
+        if [ ! -f "${CONTAINER_RUNTIME_CONFIG_TMPL}" ]; then
+            cp "${CONTAINER_RUNTIME_CONFIG}" "${CONTAINER_RUNTIME_CONFIG_TMPL}"
+        fi
+
+        CONTAINER_RUNTIME_CONFIG="${CONTAINER_RUNTIME_CONFIG_TMPL}"
+    elif [ "${CONTAINER_RUNTIME}" == "containerd" ]; then
+        if [ ! -f "${CONTAINER_RUNTIME_CONFIG}" ]; then
+            mkdir -p $(dirname ${CONTAINER_RUNTIME_CONFIG}) || true
+            if [ -x $(command -v ${CONTAINER_RUNTIME}) ]; then
+                ${CONTAINER_RUNTIME} config default > ${CONTAINER_RUNTIME_CONFIG}
+            else
+                die "Not able to find an executable ${CONTAINER_RUNTIME} binary to create the default config"
+            fi
+        fi
+    else
+        die "${CONTAINER_RUNTIME} is a unsupported containe runtime"
+    fi
+
+    action=${1:-}
+    if [ -z "$action" ]; then
+        print_usage
+        die "invalid arguments"
+    fi
+
+    case "$action" in
+    deploy)
+        deploy_snapshotter
+        ;;
+    cleanup)
+        cleanup_snapshotter
+        ;;
+    *)
+        die "invalid arguments"
+        print_usage
+        ;;
+    esac
+
+    sleep infinity
+}
+
+main "$@"

pkg/auth/docker_test.go

@@ -9,7 +9,6 @@ package auth
 import (
 	"encoding/base64"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -42,13 +41,13 @@ const (
 var oldDockerConfig = os.Getenv("DOCKER_CONFIG")
 
 func setupDockerConfig() (string, error) {
-	dir, err := ioutil.TempDir("", "testdocker-")
+	dir, err := os.MkdirTemp("", "testdocker-")
 	if err != nil {
 		return "", err
 	}
 	os.Setenv("DOCKER_CONFIG", dir)
 
-	err = ioutil.WriteFile(filepath.Join(dir, configFile),
+	err = os.WriteFile(filepath.Join(dir, configFile),
 		[]byte(fmt.Sprintf(testConfigFmt, dockerHost, base64.StdEncoding.EncodeToString([]byte(dockerUser+":"+dockerPass)),
 			extraHost, base64.StdEncoding.EncodeToString([]byte(extraUser+":"+extraPass)))),
 		0600)

pkg/auth/image_proxy.go

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package auth
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/containerd/containerd/v2/defaults"
+	"github.com/containerd/containerd/v2/pkg/dialer"
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/log"
+	"github.com/containerd/stargz-snapshotter/service/keychain/cri"
+	"github.com/containerd/stargz-snapshotter/service/resolver"
+	distribution "github.com/distribution/reference"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/credentials/insecure"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+const DefaultImageServiceAddress = "/run/containerd/containerd.sock"
+
+// Should be concurrency safe
+var Credentials []resolver.Credential = make([]resolver.Credential, 0, 8)
+
+// This function is borrowed from stargz
+func newCRIConn(criAddr string) (*grpc.ClientConn, error) {
+	// TODO: make gRPC options configurable from config.toml
+	backoffConfig := backoff.DefaultConfig
+	backoffConfig.MaxDelay = 3 * time.Second
+	connParams := grpc.ConnectParams{
+		Backoff: backoffConfig,
+	}
+	gopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithConnectParams(connParams),
+		grpc.WithContextDialer(dialer.ContextDialer),
+		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
+		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
+	}
+	return grpc.NewClient(dialer.DialAddress(criAddr), gopts...)
+}
+
+// from stargz-snapshotter/cmd/containerd-stargz-grpc/main.go#main
+func AddImageProxy(ctx context.Context, rpc *grpc.Server, imageServiceAddress string) {
+	criAddr := DefaultImageServiceAddress
+	if imageServiceAddress != "" {
+		criAddr = imageServiceAddress
+	}
+
+	criCred, criServer := cri.NewCRIKeychain(ctx, func() (runtime.ImageServiceClient, error) {
+		conn, err := newCRIConn(criAddr)
+		if err != nil {
+			return nil, err
+		}
+
+		return runtime.NewImageServiceClient(conn), nil
+	})
+
+	runtime.RegisterImageServiceServer(rpc, criServer)
+
+	Credentials = append(Credentials, criCred)
+
+	log.G(ctx).WithField("target-image-service", criAddr).Info("setup image proxy keychain")
+}
+
+func FromCRI(host, ref string) (*PassKeyChain, error) {
+	if Credentials == nil {
+		return nil, errors.New("No Credentials parsers")
+	}
+
+	refSpec, err := parseReference(ref)
+	if err != nil {
+		log.L.WithError(err).Error("parse ref failed")
+		return nil, errors.Wrapf(err, "parse image reference %s", ref)
+	}
+
+	var u, p string
+	var keychain *PassKeyChain
+
+	for _, cred := range Credentials {
+		if username, secret, err := cred(host, refSpec); err != nil {
+			return nil, err
+		} else if !(username == "" && secret == "") {
+			u = username
+			p = secret
+
+			keychain = &PassKeyChain{
+				Username: u,
+				Password: p,
+			}
+
+			break
+		}
+	}
+
+	return keychain, nil
+}
+
+// from stargz-snapshotter/service/keychain/cri/cri.go
+func parseReference(ref string) (reference.Spec, error) {
+	namedRef, err := distribution.ParseDockerRef(ref)
+	if err != nil {
+		return reference.Spec{}, fmt.Errorf("failed to parse image reference %q: %w", ref, err)
+	}
+	return reference.Parse(namedRef.String())
+}

pkg/auth/image_proxy_test.go

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package auth
+
+import (
+	"context"
+	"net"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/containerd/v2/pkg/dialer"
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+type MockImageService struct {
+	runtime.UnimplementedImageServiceServer
+}
+
+func (*MockImageService) PullImage(_ context.Context, _ *runtime.PullImageRequest) (*runtime.PullImageResponse, error) {
+	return &runtime.PullImageResponse{}, nil
+}
+
+func TestFromImagePull(t *testing.T) {
+	var err error
+	assertions := assert.New(t)
+
+	ctx := context.TODO()
+	d := t.TempDir()
+
+	tagImage := "docker.io/library/busybox:latest"
+
+	// should return nil if no proxy
+	kc, err := FromCRI("docker.io", tagImage)
+	assertions.Nil(kc)
+	assertions.NoError(err)
+
+	// Mocking the end CRI request consumer.
+	mockRPC := grpc.NewServer()
+	mockSocket := filepath.Join(d, "mock.sock")
+	listenMock, err := net.Listen("unix", mockSocket)
+	assertions.NoError(err)
+
+	// Mocking the end CRI request consumer.
+	server := &MockImageService{}
+	runtime.RegisterImageServiceServer(mockRPC, server)
+
+	go func() {
+		err := mockRPC.Serve(listenMock)
+		assertions.NoError(err)
+	}()
+	defer mockRPC.Stop()
+
+	// The server of CRI image service proxy.
+	proxyRPC := grpc.NewServer()
+	proxySocket := filepath.Join(d, "proxy.sock")
+	listenProxy, err := net.Listen("unix", proxySocket)
+	assertions.NoError(err)
+	AddImageProxy(ctx, proxyRPC, mockSocket)
+	go func() {
+		err := proxyRPC.Serve(listenProxy)
+		assertions.NoError(err)
+	}()
+	defer proxyRPC.Stop()
+
+	// should return empty kc before pulling
+	kc, err = FromCRI("docker.io", tagImage)
+	assertions.Nil(kc)
+	assertions.NoError(err)
+
+	gopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithContextDialer(dialer.ContextDialer),
+	}
+
+	conn, err := grpc.NewClient(dialer.DialAddress(proxySocket), gopts...)
+	assertions.NoError(err)
+	criClient := runtime.NewImageServiceClient(conn)
+
+	_, err = criClient.PullImage(ctx, &runtime.PullImageRequest{
+		Image: &runtime.ImageSpec{
+			Image: tagImage,
+		},
+		Auth: &runtime.AuthConfig{
+			Username: "test",
+			Password: "passwd",
+		},
+	})
+	assertions.NoError(err)
+
+	// get correct kc after pulling
+	kc, err = FromCRI("docker.io", tagImage)
+	assertions.Equal("test", kc.Username)
+	assertions.Equal("passwd", kc.Password)
+	assertions.NoError(err)
+
+	// get empty kc with wrong tag
+	kc, err = FromCRI("docker.io", "docker.io/library/busybox:another")
+	assertions.Nil(kc)
+	assertions.NoError(err)
+
+	image2 := "ghcr.io/busybox:latest"
+
+	_, err = criClient.PullImage(ctx, &runtime.PullImageRequest{
+		Image: &runtime.ImageSpec{
+			Image: image2,
+		},
+		Auth: &runtime.AuthConfig{
+			Username: "test_1",
+			Password: "passwd_1",
+		},
+	})
+	assertions.NoError(err)
+
+	// get correct kc after pulling
+	kc, err = FromCRI("ghcr.io", image2)
+	assertions.NoError(err)
+	assertions.Equal(kc.Username, "test_1")
+	assertions.Equal(kc.Password, "passwd_1")
+
+	// should work with digest
+	digestImage := "docker.io/library/busybox@sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa"
+	_, err = criClient.PullImage(ctx, &runtime.PullImageRequest{
+		Image: &runtime.ImageSpec{
+			Image: digestImage,
+		},
+		Auth: &runtime.AuthConfig{
+			Username: "digest",
+			Password: "dpwd",
+		},
+	})
+	assertions.NoError(err)
+
+	// get correct kc after pulling
+	kc, err = FromCRI("docker.io", digestImage)
+	assertions.NoError(err)
+	assertions.Equal("digest", kc.Username)
+	assertions.Equal("dpwd", kc.Password)
+}

pkg/auth/keychain.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nydus-snapshotter/pkg/label"
+	distribution "github.com/distribution/reference"
 	"github.com/google/go-containerregistry/pkg/authn"
 )
 
@@ -35,7 +35,7 @@ type PassKeyChain struct {
 func FromBase64(str string) (PassKeyChain, error) {
 	decoded, err := base64.StdEncoding.DecodeString(str)
 	if err != nil {
-		return emptyPassKeyChain, nil
+		return emptyPassKeyChain, err
 	}
 	pair := strings.Split(string(decoded), sep)
 	if len(pair) != 2 {
@@ -61,7 +61,7 @@ func (kc PassKeyChain) TokenBase() bool {
 }
 
 // FromLabels finds image pull username and secret from snapshot labels.
-// Returned `nil` means no validated username and secrect are passed, it should
+// Returned `nil` means no valid username and secret is passed, it should
 // not override input nydusd configuration.
 func FromLabels(labels map[string]string) *PassKeyChain {
 	u, found := labels[label.NydusImagePullUsername]
@@ -80,35 +80,44 @@ func FromLabels(labels map[string]string) *PassKeyChain {
 	}
 }
 
-// GetRegistryKeyChain get image pull kaychain from (ordered):
+// GetRegistryKeyChain get image pull keychain from (ordered):
 // 1. username and secrets labels
-// 2. docker config
-// 3. k8s docker config secret
-func GetRegistryKeyChain(host string, labels map[string]string) *PassKeyChain {
+// 2. cri request
+// 3. docker config
+// 4. k8s docker config secret
+func GetRegistryKeyChain(host, ref string, labels map[string]string) *PassKeyChain {
 	kc := FromLabels(labels)
 	if kc != nil {
 		return kc
 	}
+
+	// TODO: Handle error
+	kc, _ = FromCRI(host, ref)
+	if kc != nil {
+		return kc
+	}
+
 	kc = FromDockerConfig(host)
 	if kc != nil {
 		return kc
 	}
+
 	return FromKubeSecretDockerConfig(host)
 }
 
 func GetKeyChainByRef(ref string, labels map[string]string) (*PassKeyChain, error) {
-	named, err := docker.ParseDockerRef(ref)
+	named, err := distribution.ParseDockerRef(ref)
 	if err != nil {
 		return nil, errors.Wrapf(err, "parse ref %s", ref)
 	}
 
-	host := docker.Domain(named)
-	keychain := GetRegistryKeyChain(host, labels)
+	host := distribution.Domain(named)
+	keychain := GetRegistryKeyChain(host, ref, labels)
 
 	return keychain, nil
 }
 
-func (kc PassKeyChain) Resolve(target authn.Resource) (authn.Authenticator, error) {
+func (kc PassKeyChain) Resolve(_ authn.Resource) (authn.Authenticator, error) {
 	return authn.FromConfig(kc.toAuthConfig()), nil
 }
 

pkg/auth/kubesecret.go

@@ -30,46 +30,47 @@ type KubeSecretListener struct {
 	informer      cache.SharedIndexInformer
 }
 
-func InitKubeSecretListener(ctx context.Context, kubeconfigPath string) {
+func InitKubeSecretListener(ctx context.Context, kubeconfigPath string) error {
 	configMu.Lock()
 	defer configMu.Unlock()
 	if kubeSecretListener != nil {
-		return
+		return nil
 	}
 	kubeSecretListener = &KubeSecretListener{
 		dockerConfigs: make(map[string]*configfile.ConfigFile),
 	}
-	go func() {
-		if kubeconfigPath != "" {
-			_, err := os.Stat(kubeconfigPath)
-			if err != nil && !os.IsNotExist(err) {
-				logrus.WithError(err).Warningf("kubeconfig does not exist, kubeconfigPath %s", kubeconfigPath)
-				return
-			} else if err != nil {
-				logrus.WithError(err).Warningf("failed to detect kubeconfig existence, kubeconfigPath %s", kubeconfigPath)
-				return
-			}
+
+	if kubeconfigPath != "" {
+		_, err := os.Stat(kubeconfigPath)
+		if err != nil && !os.IsNotExist(err) {
+			logrus.WithError(err).Warningf("kubeconfig does not exist, kubeconfigPath %s", kubeconfigPath)
+			return err
+		} else if err != nil {
+			logrus.WithError(err).Warningf("failed to detect kubeconfig existence, kubeconfigPath %s", kubeconfigPath)
+			return err
 		}
-		loadingRule := clientcmd.NewDefaultClientConfigLoadingRules()
-		loadingRule.ExplicitPath = kubeconfigPath
-		clientConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-			loadingRule,
-			&clientcmd.ConfigOverrides{},
-		).ClientConfig()
-		if err != nil {
-			logrus.WithError(err).Warningf("failed to load kubeconfig")
-			return
-		}
-		clientset, err := kubernetes.NewForConfig(clientConfig)
-		if err != nil {
-			logrus.WithError(err).Warningf("failed to create kubernetes client")
-			return
-		}
-		if err := kubeSecretListener.SyncKubeSecrets(ctx, clientset); err != nil {
-			logrus.WithError(err).Warningf("failed to sync secrets")
-			return
-		}
-	}()
+	}
+	loadingRule := clientcmd.NewDefaultClientConfigLoadingRules()
+	loadingRule.ExplicitPath = kubeconfigPath
+	clientConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		loadingRule,
+		&clientcmd.ConfigOverrides{},
+	).ClientConfig()
+	if err != nil {
+		logrus.WithError(err).Warningf("failed to load kubeconfig")
+		return err
+	}
+	clientset, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		logrus.WithError(err).Warningf("failed to create kubernetes client")
+		return err
+	}
+	if err := kubeSecretListener.SyncKubeSecrets(ctx, clientset); err != nil {
+		logrus.WithError(err).Warningf("failed to sync secrets")
+		return err
+	}
+
+	return nil
 }
 
 func (kubelistener *KubeSecretListener) addDockerConfig(key string, obj interface{}) error {
@@ -112,7 +113,7 @@ func (kubelistener *KubeSecretListener) SyncKubeSecrets(ctx context.Context, cli
 		cache.Indexers{},
 	)
 	kubelistener.informer = informer
-	kubelistener.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+	_, err := kubelistener.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(obj interface{}) {
 			key, err := cache.MetaNamespaceKeyFunc(obj)
 			if err != nil {
@@ -124,7 +125,7 @@ func (kubelistener *KubeSecretListener) SyncKubeSecrets(ctx context.Context, cli
 				return
 			}
 		},
-		UpdateFunc: func(old, new interface{}) {
+		UpdateFunc: func(_, new interface{}) {
 			key, err := cache.MetaNamespaceKeyFunc(new)
 			if err != nil {
 				logrus.WithError(err).Errorf("failed to get key for secret from cache")
@@ -143,6 +144,9 @@ func (kubelistener *KubeSecretListener) SyncKubeSecrets(ctx context.Context, cli
 			kubelistener.deleteDockerConfig(key)
 		}},
 	)
+	if err != nil {
+		return errors.Wrap(err, "add event handler to informer")
+	}
 	go kubelistener.informer.Run(ctx.Done())
 	if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) {
 		return fmt.Errorf("timed out for syncing cache")

pkg/auth/kubesecret_test.go

@@ -13,14 +13,14 @@ import (
 const (
 	testDockerConfigJSONFmt = `
 {
-        "auths": {
-                "%s": {
-                        "username": "%s",
-						"password": "%s",
-						"email": "%s",
-						"auth": "%s"
-                }
-        }
+	"auths": {
+		"%s": {
+			"username": "%s",
+			"password": "%s",
+			"email": "%s",
+			"auth": "%s"
+		}
+	}
 }
 `
 	dockerConfigKey = "testKey"
@@ -33,7 +33,8 @@ func TestGetCredentialsStore(t *testing.T) {
 	assert := assert.New(t)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	InitKubeSecretListener(ctx, "")
+	// Host may not has kubeconfig, so ignore the error and continue the test
+	_ = InitKubeSecretListener(ctx, "")
 	assert.NotNil(kubeSecretListener)
 
 	var obj interface{} = &corev1.Secret{

pkg/backend/backend.go

@@ -9,20 +9,28 @@ package backend
 import (
 	"context"
 	"fmt"
-	"io"
 
+	"github.com/containerd/containerd/v2/core/content"
 	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 const (
-	BackendTypeOSS = "oss"
+	BackendTypeOSS     = "oss"
+	BackendTypeS3      = "s3"
+	BackendTypeLocalFS = "localfs"
 )
 
-// Backend uploads blobs generated by nydus-image builder to a backend storage such as:
-// - oss: A object storage backend, which uses its SDK to upload blob file.
+var (
+	// We always use multipart upload for backend, and limit the
+	// multipart chunk size to 500MB by default.
+	MultipartChunkSize int64 = 500 * 1024 * 1024
+)
+
+// Backend uploads blobs generated by nydus-image builder to a backend storage.
 type Backend interface {
 	// Push pushes specified blob file to remote storage backend.
-	Push(ctx context.Context, blobReader io.Reader, blobDigest digest.Digest) error
+	Push(ctx context.Context, cs content.Store, desc ocispec.Descriptor) error
 	// Check checks whether a blob exists in remote storage backend,
 	// blob exists -> return (blobPath, nil)
 	// blob not exists -> return ("", err)
@@ -35,12 +43,14 @@ type Backend interface {
 // registry as per OCI distribution specification. But nydus can also make OSS or
 // other storage services as backend storage. Pass config as byte slice here because
 // we haven't find a way to represent all backend config at the same time.
-func NewBackend(_type string, config []byte) (Backend, error) {
+func NewBackend(_type string, config []byte, forcePush bool) (Backend, error) {
 	switch _type {
 	case BackendTypeOSS:
-		return newOSSBackend(config)
+		return newOSSBackend(config, forcePush)
+	case BackendTypeS3:
+		return newS3Backend(config, forcePush)
 	case BackendTypeLocalFS:
-		return newLocalFSBackend(config)
+		return newLocalFSBackend(config, forcePush)
 	default:
 		return nil, fmt.Errorf("unsupported backend type %s", _type)
 	}

pkg/backend/localfs.go

@@ -14,20 +14,19 @@ import (
 	"os"
 	"path"
 
-	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/errdefs"
 	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
-const (
-	BackendTypeLocalFS = "localfs"
-)
-
 type LocalFSBackend struct {
-	dir string
+	dir       string
+	forcePush bool
 }
 
-func newLocalFSBackend(rawConfig []byte) (*LocalFSBackend, error) {
+func newLocalFSBackend(rawConfig []byte, forcePush bool) (*LocalFSBackend, error) {
 	var configMap map[string]string
 	if err := json.Unmarshal(rawConfig, &configMap); err != nil {
 		return nil, errors.Wrap(err, "parse LocalFS storage backend configuration")
@@ -39,7 +38,8 @@ func newLocalFSBackend(rawConfig []byte) (*LocalFSBackend, error) {
 	}
 
 	return &LocalFSBackend{
-		dir: dir,
+		dir:       dir,
+		forcePush: forcePush,
 	}, nil
 }
 
@@ -47,12 +47,22 @@ func (b *LocalFSBackend) dstPath(blobID string) string {
 	return path.Join(b.dir, blobID)
 }
 
-func (b *LocalFSBackend) Push(ctx context.Context, blobReader io.Reader, blobDigest digest.Digest) error {
+func (b *LocalFSBackend) Push(ctx context.Context, cs content.Store, desc ocispec.Descriptor) error {
+	if _, err := b.Check(desc.Digest); err == nil && !b.forcePush {
+		return nil
+	}
+
 	if err := os.MkdirAll(b.dir, 0755); err != nil {
 		return errors.Wrap(err, "create directory in localfs backend")
 	}
 
-	blobID := blobDigest.Hex()
+	ra, err := cs.ReaderAt(ctx, desc)
+	if err != nil {
+		return errors.Wrap(err, "get reader from content store")
+	}
+	defer ra.Close()
+
+	blobID := desc.Digest.Hex()
 	dstPath := b.dstPath(blobID)
 
 	dstFile, err := os.Create(dstPath)
@@ -61,7 +71,8 @@ func (b *LocalFSBackend) Push(ctx context.Context, blobReader io.Reader, blobDig
 	}
 	defer dstFile.Close()
 
-	if _, err := io.Copy(dstFile, blobReader); err != nil {
+	sr := io.NewSectionReader(ra, 0, ra.Size())
+	if _, err := io.Copy(dstFile, sr); err != nil {
 		return errors.Wrapf(err, "copy blob to %s", dstPath)
 	}
 

pkg/backend/oss.go

@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package backend
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/errdefs"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+type OSSBackend struct {
+	// OSS storage does not support directory. Therefore add a prefix to each object
+	// to make it a path-like object.
+	objectPrefix string
+	bucket       *oss.Bucket
+	forcePush    bool
+}
+
+func newOSSBackend(rawConfig []byte, forcePush bool) (*OSSBackend, error) {
+	var configMap map[string]string
+	if err := json.Unmarshal(rawConfig, &configMap); err != nil {
+		return nil, errors.Wrap(err, "Parse OSS storage backend configuration")
+	}
+
+	endpoint, ok1 := configMap["endpoint"]
+	bucketName, ok2 := configMap["bucket_name"]
+
+	// Below fields are not mandatory.
+	accessKeyID := configMap["access_key_id"]
+	accessKeySecret := configMap["access_key_secret"]
+	objectPrefix := configMap["object_prefix"]
+
+	if !ok1 || !ok2 {
+		return nil, fmt.Errorf("no endpoint or bucket is specified")
+	}
+
+	client, err := oss.New(endpoint, accessKeyID, accessKeySecret)
+	if err != nil {
+		return nil, errors.Wrap(err, "Create client")
+	}
+
+	bucket, err := client.Bucket(bucketName)
+	if err != nil {
+		return nil, errors.Wrap(err, "Create bucket")
+	}
+
+	return &OSSBackend{
+		objectPrefix: objectPrefix,
+		bucket:       bucket,
+		forcePush:    forcePush,
+	}, nil
+}
+
+// Ported from https://github.com/aliyun/aliyun-oss-go-sdk/blob/v2.2.6/oss/utils.go#L259
+func splitFileByPartSize(blobSize, chunkSize int64) ([]oss.FileChunk, error) {
+	if chunkSize <= 0 {
+		return nil, errors.New("invalid chunk size")
+	}
+
+	var chunkN = blobSize / chunkSize
+	if chunkN >= 10000 {
+		return nil, errors.New("too many parts, please increase chunk size")
+	}
+
+	var chunks []oss.FileChunk
+	var chunk = oss.FileChunk{}
+	for i := int64(0); i < chunkN; i++ {
+		chunk.Number = int(i + 1)
+		chunk.Offset = i * chunkSize
+		chunk.Size = chunkSize
+		chunks = append(chunks, chunk)
+	}
+
+	if blobSize%chunkSize > 0 {
+		chunk.Number = len(chunks) + 1
+		chunk.Offset = int64(len(chunks)) * chunkSize
+		chunk.Size = blobSize % chunkSize
+		chunks = append(chunks, chunk)
+	}
+
+	return chunks, nil
+}
+
+// Upload nydus blob to oss storage backend.
+func (b *OSSBackend) push(ctx context.Context, cs content.Store, desc ocispec.Descriptor) error {
+	blobID := desc.Digest.Hex()
+	blobObjectKey := b.objectPrefix + blobID
+
+	ra, err := cs.ReaderAt(ctx, desc)
+	if err != nil {
+		return errors.Wrapf(err, "get reader for compression blob %q", desc.Digest)
+	}
+	defer ra.Close()
+
+	if exist, err := b.bucket.IsObjectExist(blobObjectKey); err != nil {
+		return errors.Wrap(err, "check object existence")
+	} else if exist && !b.forcePush {
+		return nil
+	}
+
+	chunks, err := splitFileByPartSize(ra.Size(), MultipartChunkSize)
+	if err != nil {
+		return errors.Wrap(err, "split blob by part num")
+	}
+
+	imur, err := b.bucket.InitiateMultipartUpload(blobObjectKey)
+	if err != nil {
+		return errors.Wrap(err, "initiate multipart upload")
+	}
+	partsChan := make(chan oss.UploadPart, len(chunks))
+
+	g := new(errgroup.Group)
+	for _, chunk := range chunks {
+		ck := chunk
+		g.Go(func() error {
+			p, err := b.bucket.UploadPart(imur, io.NewSectionReader(ra, ck.Offset, ck.Size), ck.Size, ck.Number)
+			if err != nil {
+				return errors.Wrap(err, "upload part")
+			}
+			partsChan <- p
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		_ = b.bucket.AbortMultipartUpload(imur)
+		close(partsChan)
+		return errors.Wrap(err, "upload parts")
+	}
+	close(partsChan)
+
+	parts := make([]oss.UploadPart, 0, 16)
+	for p := range partsChan {
+		parts = append(parts, p)
+	}
+
+	_, err = b.bucket.CompleteMultipartUpload(imur, parts)
+	if err != nil {
+		return errors.Wrap(err, "complete multipart upload")
+	}
+
+	return nil
+}
+
+func (b *OSSBackend) Push(ctx context.Context, cs content.Store, desc ocispec.Descriptor) error {
+	backoff := time.Second
+	for {
+		err := b.push(ctx, cs, desc)
+		if err != nil {
+			select {
+			case <-ctx.Done():
+				return err
+			default:
+			}
+		} else {
+			return nil
+		}
+		if backoff >= 8*time.Second {
+			return err
+		}
+		time.Sleep(backoff)
+		backoff *= 2
+	}
+}
+
+func (b *OSSBackend) Check(blobDigest digest.Digest) (string, error) {
+	blobID := blobDigest.Hex()
+	blobObjectKey := b.objectPrefix + blobID
+	if exist, err := b.bucket.IsObjectExist(blobObjectKey); err != nil {
+		return "", err
+	} else if exist {
+		return blobID, nil
+	}
+	return "", errdefs.ErrNotFound
+}
+
+func (b *OSSBackend) Type() string {
+	return BackendTypeOSS
+}

pkg/backend/s3.go

@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package backend
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	awscfg "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/errdefs"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+type S3Backend struct {
+	// objectPrefix is the path prefix of the uploaded object.
+	// For example, if the blobID which should be uploaded is "abc",
+	// and the objectPrefix is "path/to/my-registry/", then the object key will be
+	// "path/to/my-registry/abc".
+	objectPrefix       string
+	bucketName         string
+	endpointWithScheme string
+	region             string
+	accessKeySecret    string
+	accessKeyID        string
+	forcePush          bool
+}
+
+type S3Config struct {
+	AccessKeyID     string `json:"access_key_id,omitempty"`
+	AccessKeySecret string `json:"access_key_secret,omitempty"`
+	Endpoint        string `json:"endpoint,omitempty"`
+	Scheme          string `json:"scheme,omitempty"`
+	BucketName      string `json:"bucket_name,omitempty"`
+	Region          string `json:"region,omitempty"`
+	ObjectPrefix    string `json:"object_prefix,omitempty"`
+}
+
+func newS3Backend(rawConfig []byte, forcePush bool) (*S3Backend, error) {
+	cfg := &S3Config{}
+	if err := json.Unmarshal(rawConfig, cfg); err != nil {
+		return nil, errors.Wrap(err, "parse S3 storage backend configuration")
+	}
+	if cfg.Endpoint == "" {
+		cfg.Endpoint = "s3.amazonaws.com"
+	}
+	if cfg.Scheme == "" {
+		cfg.Scheme = "https"
+	}
+	endpointWithScheme := fmt.Sprintf("%s://%s", cfg.Scheme, cfg.Endpoint)
+
+	if cfg.BucketName == "" || cfg.Region == "" {
+		return nil, fmt.Errorf("invalid S3 configuration: missing 'bucket_name' or 'region'")
+	}
+
+	return &S3Backend{
+		objectPrefix:       cfg.ObjectPrefix,
+		bucketName:         cfg.BucketName,
+		region:             cfg.Region,
+		endpointWithScheme: endpointWithScheme,
+		accessKeySecret:    cfg.AccessKeySecret,
+		accessKeyID:        cfg.AccessKeyID,
+		forcePush:          forcePush,
+	}, nil
+}
+
+func (b *S3Backend) client() (*s3.Client, error) {
+	s3AWSConfig, err := awscfg.LoadDefaultConfig(context.TODO())
+	if err != nil {
+		return nil, errors.Wrap(err, "load default AWS config")
+	}
+
+	client := s3.NewFromConfig(s3AWSConfig, func(o *s3.Options) {
+		o.BaseEndpoint = &b.endpointWithScheme
+		o.Region = b.region
+		o.UsePathStyle = true
+		if len(b.accessKeySecret) > 0 && len(b.accessKeyID) > 0 {
+			o.Credentials = credentials.NewStaticCredentialsProvider(b.accessKeyID, b.accessKeySecret, "")
+		}
+		o.UsePathStyle = true
+	})
+	return client, nil
+}
+
+func (b *S3Backend) existObject(ctx context.Context, objectKey string) (bool, error) {
+	client, err := b.client()
+	if err != nil {
+		return false, errors.Wrap(err, "failed to create s3 client")
+	}
+	_, err = client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: &b.bucketName,
+		Key:    &objectKey,
+	})
+	if err != nil {
+		var responseError *awshttp.ResponseError
+		if errors.As(err, &responseError) && responseError.ResponseError.HTTPStatusCode() == http.StatusNotFound {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func (b *S3Backend) Push(ctx context.Context, cs content.Store, desc ocispec.Descriptor) error {
+	blobID := desc.Digest.Hex()
+	blobObjectKey := b.objectPrefix + blobID
+
+	if exist, err := b.existObject(ctx, blobObjectKey); err != nil {
+		return errors.Wrap(err, "check object existence")
+	} else if exist && !b.forcePush {
+		return nil
+	}
+
+	ra, err := cs.ReaderAt(ctx, desc)
+	if err != nil {
+		return errors.Wrap(err, "get reader from content store")
+	}
+	defer ra.Close()
+	reader := content.NewReader(ra)
+
+	client, err := b.client()
+	if err != nil {
+		return errors.Wrap(err, "failed to create s3 client")
+	}
+
+	uploader := manager.NewUploader(client, func(u *manager.Uploader) {
+		u.PartSize = MultipartChunkSize
+	})
+	if _, err := uploader.Upload(ctx, &s3.PutObjectInput{
+		Bucket:            aws.String(b.bucketName),
+		Key:               aws.String(blobObjectKey),
+		Body:              reader,
+		ChecksumAlgorithm: types.ChecksumAlgorithmCrc32,
+	}); err != nil {
+		return errors.Wrap(err, "push blob to s3 backend")
+	}
+
+	return nil
+}
+
+func (b *S3Backend) Check(blobDigest digest.Digest) (string, error) {
+	blobID := blobDigest.Hex()
+	objectKey := b.objectPrefix + blobDigest.Hex()
+	if exist, err := b.existObject(context.Background(), objectKey); err != nil {
+		return "", err
+	} else if exist {
+		return blobID, nil
+	}
+	return "", errdefs.ErrNotFound
+}
+
+func (b *S3Backend) Type() string {
+	return BackendTypeS3
+}

pkg/backend/s3_test.go

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package backend
+
+import (
+	"reflect"
+	"testing"
+)
+
+func Test_newS3Backend(t *testing.T) {
+	type args struct {
+		rawConfig []byte
+	}
+
+	tests := []struct {
+		name    string
+		args    args
+		want    *S3Backend
+		wantErr bool
+	}{
+		{
+			name: "test1, no error",
+			args: args{
+				rawConfig: []byte(`{
+					"endpoint": "localhost:9000",
+					"scheme": "http",
+					"bucket_name": "nydus",
+					"region": "us-east-1",
+					"object_prefix": "path/to/my-registry/",
+					"access_key_id": "minio",
+					"access_key_secret": "minio123"
+				}`),
+			},
+			want: &S3Backend{
+				objectPrefix:       "path/to/my-registry/",
+				bucketName:         "nydus",
+				endpointWithScheme: "http://localhost:9000",
+				region:             "us-east-1",
+				accessKeySecret:    "minio123",
+				accessKeyID:        "minio",
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := newS3Backend(tt.args.rawConfig, false)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("newS3Backend() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("newS3Backend() = %+#v\nwant %+#v\n\n", got, tt.want)
+			}
+		})
+	}
+}

pkg/cache/db.go

@@ -1,13 +0,0 @@
-package cache
-
-import (
-	"github.com/containerd/nydus-snapshotter/pkg/store"
-)
-
-type DB interface {
-	AddSnapshot(imageID string, blobs []string) error
-	DelSnapshot(imageID string) error
-	GC(delFunc func(blob string) error) ([]string, error)
-}
-
-var _ DB = &store.CacheStore{}

pkg/cache/manager.go

@@ -1,29 +1,46 @@
+/*
+ * Copyright (c) 2022. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 package cache
 
 import (
+	"context"
 	"os"
+	"path"
 	"time"
 
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/nydus-snapshotter/config"
-	"github.com/containerd/nydus-snapshotter/pkg/store"
 	"github.com/pkg/errors"
+
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/continuity/fs"
+	"github.com/containerd/log"
+	"github.com/containerd/nydus-snapshotter/pkg/store"
 )
 
+const (
+	imageDiskFileSuffix = ".image.disk"
+	layerDiskFileSuffix = ".layer.disk"
+	chunkMapFileSuffix  = ".chunk_map"
+	metaFileSuffix      = ".blob.meta"
+	// Blob cache is suffixed after nydus v2.1
+	dataFileSuffix = ".blob.data"
+)
+
+// Disk cache manager for fusedev.
 type Manager struct {
-	db       DB
-	store    *Store
 	cacheDir string
 	period   time.Duration
 	eventCh  chan struct{}
-	fsDriver string
 }
 
 type Opt struct {
+	Disabled bool
 	CacheDir string
 	Period   time.Duration
 	Database *store.Database
-	FsDriver string
 }
 
 func NewManager(opt Opt) (*Manager, error) {
@@ -32,32 +49,13 @@ func NewManager(opt Opt) (*Manager, error) {
 		return nil, errors.Wrapf(err, "failed to create cache dir %s", opt.CacheDir)
 	}
 
-	db, err := store.NewCacheStore(opt.Database)
-	if err != nil {
-		return nil, err
-	}
-	s := NewStore(opt.CacheDir)
-
 	eventCh := make(chan struct{})
 	m := &Manager{
-		db:       db,
-		store:    s,
 		cacheDir: opt.CacheDir,
 		period:   opt.Period,
 		eventCh:  eventCh,
-		fsDriver: opt.FsDriver,
 	}
 
-	// For fscache backend, the cache is maintained by the kernel fscache module,
-	// so here we ignore gc for now, and in the future we need another design
-	// to remove the cache.
-	if opt.FsDriver == config.FsDriverFscache {
-		return m, nil
-	}
-
-	go m.runGC()
-	log.L.Info("gc goroutine start...")
-
 	return m, nil
 }
 
@@ -65,44 +63,60 @@ func (m *Manager) CacheDir() string {
 	return m.cacheDir
 }
 
-func (m *Manager) SchedGC() {
-	if m.fsDriver == config.FsDriverFscache {
-		return
+// Report each blob disk usage
+// TODO: For fscache cache files, the cache files are managed by nydusd and Linux kernel
+// We don't know how it manages cache files. A method to address this is to query nydusd.
+// So we can't report cache usage in the case of fscache now
+func (m *Manager) CacheUsage(ctx context.Context, blobID string) (snapshots.Usage, error) {
+	var usage snapshots.Usage
+
+	blobCachePath := path.Join(m.cacheDir, blobID)
+	blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix)
+	// For backward compatibility
+	blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix)
+	blobChunkMapSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix+chunkMapFileSuffix)
+	blobMeta := path.Join(m.cacheDir, blobID+metaFileSuffix)
+	imageDisk := path.Join(m.cacheDir, blobID+imageDiskFileSuffix)
+	layerDisk := path.Join(m.cacheDir, blobID+layerDiskFileSuffix)
+
+	stuffs := []string{blobCachePath, blobChunkMap, blobCacheSuffixedPath, blobChunkMapSuffixedPath, blobMeta, imageDisk, layerDisk}
+
+	for _, f := range stuffs {
+		du, err := fs.DiskUsage(ctx, f)
+		if err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				log.L.Debugf("Cache %s does not exist", f)
+				continue
+			}
+			return snapshots.Usage{}, err
+		}
+		usage.Add(snapshots.Usage(du))
 	}
-	m.eventCh <- struct{}{}
+
+	return usage, nil
 }
 
-func (m *Manager) runGC() {
-	tick := time.NewTicker(m.period)
-	defer tick.Stop()
-	for {
-		select {
-		case <-m.eventCh:
-			if err := m.gc(); err != nil {
-				log.L.Infof("[event] cache gc err, %v", err)
-			}
-			tick.Reset(m.period)
-		case <-tick.C:
-			if err := m.gc(); err != nil {
-				log.L.Infof("[tick] cache gc err, %v", err)
+func (m *Manager) RemoveBlobCache(blobID string) error {
+	blobCachePath := path.Join(m.cacheDir, blobID)
+	blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix)
+	blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix)
+	blobChunkMapSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix+chunkMapFileSuffix)
+	blobMeta := path.Join(m.cacheDir, blobID+metaFileSuffix)
+	imageDisk := path.Join(m.cacheDir, blobID+imageDiskFileSuffix)
+	layerDisk := path.Join(m.cacheDir, blobID+layerDiskFileSuffix)
+
+	// NOTE: Delete chunk bitmap file before data blob
+	stuffs := []string{blobChunkMap, blobChunkMapSuffixedPath, blobMeta, blobCachePath, blobCacheSuffixedPath, imageDisk, layerDisk}
+
+	for _, f := range stuffs {
+		err := os.Remove(f)
+		if err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				log.L.Debugf("file %s doest not exist.", f)
+				continue
 			}
+			return err
 		}
 	}
-}
-
-func (m *Manager) gc() error {
-	delBlobs, err := m.db.GC(m.store.DelBlob)
-	if err != nil {
-		return errors.Wrapf(err, "cache gc err")
-	}
-	log.L.Debugf("remove %d unused blobs successfully", len(delBlobs))
 	return nil
 }
-
-func (m *Manager) AddSnapshot(imageID string, blobs []string) error {
-	return m.db.AddSnapshot(imageID, blobs)
-}
-
-func (m *Manager) DelSnapshot(imageID string) error {
-	return m.db.DelSnapshot(imageID)
-}

pkg/cache/store.go

@@ -1,51 +0,0 @@
-package cache
-
-import (
-	"os"
-	"path/filepath"
-
-	"github.com/pkg/errors"
-)
-
-const (
-	chunkMapFileSuffix = ".chunk_map"
-	metaFileSuffix     = ".blob.meta"
-)
-
-type Store struct {
-	cacheDir string
-}
-
-func NewStore(cacheDir string) *Store {
-	return &Store{cacheDir: cacheDir}
-}
-
-func (cs *Store) DelBlob(blob string) error {
-	blobPath := cs.blobPath(blob)
-
-	// Remove the blob chunkmap file named $blob_id.chunk_map first.
-	chunkMapPath := blobPath + chunkMapFileSuffix
-	if err := os.Remove(chunkMapPath); err != nil {
-		// Older versions of nydusd do not support chunkmap, and there
-		// is no chunkmap file generation, so just ignore the error.
-		if !os.IsNotExist(err) {
-			return errors.Wrapf(err, "remove blob chunkmap %v err", chunkMapPath)
-		}
-	}
-
-	// Then remove the blob file named $blob_id.
-	if err := os.Remove(blobPath); err != nil {
-		return errors.Wrapf(err, "remove blob %v err", blobPath)
-	}
-
-	metaPath := blobPath + metaFileSuffix
-	if err := os.Remove(metaPath); err != nil {
-		return errors.Wrapf(err, "remove blob meta file %v err", metaPath)
-	}
-
-	return nil
-}
-
-func (cs *Store) blobPath(blob string) string {
-	return filepath.Join(cs.cacheDir, blob)
-}

pkg/cgroup/cgroup.go

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package cgroup
+
+import (
+	"errors"
+
+	"github.com/containerd/cgroups/v3"
+	v1 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v1"
+	v2 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v2"
+)
+
+const (
+	defaultSlice = "system.slice"
+)
+
+var (
+	ErrCgroupNotSupported = errors.New("cgroups: cgroup not supported")
+)
+
+type Config struct {
+	MemoryLimitInBytes int64
+}
+
+type DaemonCgroup interface {
+	// Delete the current cgroup.
+	Delete() error
+	// Add a process to current cgroup.
+	AddProc(pid int) error
+}
+
+func createCgroup(name string, config Config) (DaemonCgroup, error) {
+	if cgroups.Mode() == cgroups.Unified {
+		return v2.NewCgroup(defaultSlice, name, config.MemoryLimitInBytes)
+	}
+
+	return v1.NewCgroup(defaultSlice, name, config.MemoryLimitInBytes)
+}
+
+func supported() bool {
+	return cgroups.Mode() != cgroups.Unavailable
+}
+
+func displayMode() string {
+	switch cgroups.Mode() {
+	case cgroups.Legacy:
+		return "legacy"
+	case cgroups.Hybrid:
+		return "hybrid"
+	case cgroups.Unified:
+		return "unified"
+	case cgroups.Unavailable:
+		return "unavailable"
+	default:
+		return "unknown"
+	}
+}