Merge pull request #644 from DataDog/fricounet/referrer-detect-shared-layer

[mountRemote] include intermediate snapshots in mount slice during referrer detection

.github/codecov.yml

@@ -0,0 +1,23 @@
+coverage:
+  status:
+    patch: off
+    project:
+      default:
+        enabled: yes
+        target: auto # auto compares coverage to the previous base commit
+        # adjust accordingly based on how flaky your tests are
+        # this allows a 0.3% drop from the previous base commit coverage
+        threshold: 0.3%
+
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: true # if true: only post the comment if coverage changes
+
+codecov:
+  require_ci_to_pass: false
+  notify:
+    wait_for_ci: true
+# When modifying this file, please validate using
+
+# curl -X POST --data-binary @codecov.yml https://codecov.io/validate

.github/workflows/ci.yml

@@ -2,62 +2,70 @@ name: CI
 
 on:
   push:
-    branches: ["*"]
+    branches: ["**", "stable/**"]
   pull_request:
-    branches: [main]
+    branches: ["**", "stable/**"]
+
+env:
+  CARGO_TERM_COLOR: always
 
 jobs:
   build:
     name: Build and Lint
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.20.1"
       - name: Check out code
-        uses: actions/checkout@v3
-      - name: cache go mod
-        uses: actions/cache@v3
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.59
+          skip-cache: true
       - name: Build
         run: |
-          go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2
-          export PATH=$PATH:$(go env GOPATH)/bin
           make
           make test
-          make check
-
+  build-optimizer:
+    name: Build optimizer
+    timeout-minutes: 10
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: Build
+        run: |
+          rustup component add rustfmt clippy
+          make build-optimizer
   smoke:
     name: Smoke
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.19.6"
       - name: Check out code
-        uses: actions/checkout@v3
-      - name: cache go mod
-        uses: actions/cache@v3
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
       - name: Set up containerd
-        uses: crazy-max/ghaction-setup-containerd@v2
+        uses: crazy-max/ghaction-setup-containerd@v3
       - name: Build
         run: |
           # Download nydus components
-          NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
+          NYDUS_VER=v$(curl -fsSL --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//')
+          wget -q https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
           tar xzvf nydus-static-$NYDUS_VER-linux-amd64.tgz
           mkdir -p /usr/bin
           sudo mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/
@@ -67,25 +75,19 @@ jobs:
   cross-build-test:
     name: Cross Build Test
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         GOOS: ["linux", "windows", "darwin"]
         GOARCH: ["amd64", "arm64"]
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.19.6"
       - name: Check out code
-        uses: actions/checkout@v3
-      - name: cache go mod
-        uses: actions/cache@v3
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
       - name: Build
         run: |
           make -e GOOS=${{ matrix.GOOS }} GOARCH=${{ matrix.GOARCH }} converter
@@ -93,26 +95,20 @@ jobs:
   coverage:
     name: Code coverage
     timeout-minutes: 10
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [build]
     steps:
-      - name: Install Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.19.6"
       - name: Checkout code
-        uses: actions/checkout@v3
-      - name: cache go mod
-        uses: actions/cache@v3
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
       - name: Run unit tests.
         run: make cover
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
           files: ./coverage.txt

.github/workflows/e2e.yml

@@ -19,24 +19,41 @@ env:
   REGISTRY: ghcr.io
 
 jobs:
-  run-e2e:
-    runs-on: ubuntu-latest
+  run-e2e-for-cgroups-v1:
+    runs-on: ubuntu-22.04
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.19.6"
       - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: cache go mod
-        uses: actions/cache@v3
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
       - name: Log in to container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run e2e test
+        run: |
+          TAG=$GITHUB_REF_NAME
+          [ "$TAG" == "main" ] && TAG="latest"
+          [ "$GITHUB_EVENT_NAME" == "pull_request" ] && TAG="local"
+          make integration
+  
+  run-e2e-for-cgroups-v2:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: Log in to container registry
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}

.github/workflows/k8s-e2e.yml

@@ -10,134 +10,38 @@ on:
 env:
   DOCKER_USER: testuser
   DOCKER_PASSWORD: testpassword
+  NAMESPACE: nydus-system
 
 jobs:
   e2e_tests_k8s:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     timeout-minutes: 30
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Set up Go
-        uses: actions/setup-go@v3
+      - name: Setup Golang
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.19.6"
-      - name: Setup Kind
-        uses: engineerd/setup-kind@v0.5.0
-        with:
-          version: v0.16.0
-          config: tests/e2e/k8s/kind.yaml
-      - name: Build nydus snapshotter dev image
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: Test
         run: |
-          make
-          cp bin/containerd-nydus-grpc ./
-          cp misc/snapshotter/* ./
-          ls -tl ./
-          NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          docker build --build-arg NYDUS_VER=${NYDUS_VER} -t local-dev:e2e .
-
-          ## load local test image into kind node
-          kind load docker-image local-dev:e2e
-      - name: Setup registry
-        run: |
-          mkdir auth
-          docker run \
-            --entrypoint htpasswd \
-            httpd:2 -Bbn ${{ env.DOCKER_USER }} ${{ env.DOCKER_PASSWORD }} > auth/htpasswd
-
-          docker run -d \
-            -p 5000:5000 \
-            --restart=always \
-            --name registry \
-            -v "$(pwd)"/auth:/auth \
-            -e "REGISTRY_AUTH=htpasswd" \
-            -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
-            -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-            registry:2
-      - name: Login to GitHub Container Registry
-        run: |
-          registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
-          ## sudo cat can't modify daemon.json
-          sudo chmod a+w /etc/docker/daemon.json
-          cat << EOF > /etc/docker/daemon.json
-          {
-            "exec-opts": ["native.cgroupdriver=cgroupfs"],
-            "cgroup-parent": "/actions_job",
-            "insecure-registries" : [ "${registry_ip}:5000" ]
-          }
-          EOF
-          sudo systemctl restart docker
-          docker login --username=${{ env.DOCKER_USER }} --password=${{ env.DOCKER_PASSWORD }} $registry_ip:5000
-      - name: Setup nydus snapshotter
-        run: |
-          kubectl create -f tests/e2e/k8s/snapshotter-${{ inputs.auth-type }}.yaml
-          export ns=nydus-system
-          p=`kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name`
-          echo "snapshotter pod name ${p}"
-          kubectl -n $ns wait po $p --for=condition=ready --timeout=2m
-
-          # change snapshotter to nydus after nydus snapshotter started
-          registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
-          docker cp kind-control-plane:/etc/containerd/config.toml containerd.config.toml.bak
-          sed -i -e 's|snapshotter = "overlayfs"|snapshotter = "nydus"|' containerd.config.toml.bak
-          cat << EOF >> containerd.config.toml.bak
-          [proxy_plugins]
-            [proxy_plugins.nydus]
-              type = "snapshot"
-              address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
-
-          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${registry_ip}:5000"]
-              endpoint = ["http://${registry_ip}:5000"]
-          EOF
-
-          docker cp containerd.config.toml.bak kind-control-plane:/etc/containerd/config.toml.bak
-          docker exec kind-control-plane sh -c "cat /etc/containerd/config.toml.bak > /etc/containerd/config.toml"
-          docker exec kind-control-plane systemctl restart containerd
-      - name: Install Nydus binaries and convert nydus image
-        run: |
-          NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
-          tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz
-          sudo cp nydus-static/nydusify nydus-static/nydus-image /usr/local/bin/
-
-          registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
-
-          sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
-                 --source busybox:latest \
-                 --target ${registry_ip}:5000/busybox:nydus-v6-latest \
-                 --fs-version 6
-      - name: Run E2E test
-        run: |
-          registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
-          kubectl create --namespace nydus-system secret generic regcred \
-              --from-file=.dockerconfigjson=$HOME/.docker/config.json \
-              --type=kubernetes.io/dockerconfigjson
-
-          sed -e "s|REGISTRY_IP|${registry_ip}|" tests/e2e/k8s/test-pod.yaml.tpl > tests/e2e/k8s/test-pod.yaml
-
-          if [[ "${{ inputs.auth-type }}" == "cri" ]]; then
-            docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet'
-            docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet'
-          fi
-
-          kubectl apply -f tests/e2e/k8s/test-pod.yaml
-          kubectl wait po test-pod -n nydus-system --for=condition=ready --timeout=1m
-          kubectl delete -f tests/e2e/k8s/test-pod.yaml
+          AUTH_TYPE='${{ inputs.auth-type }}'
+          ./tests/helpers/kind.sh
       - name: Dump logs
         if: failure()
         continue-on-error: true
         run: |
           log_dir="/tmp/nydus-log"
           mkdir -p $log_dir
-          export ns=nydus-system
-          for p in `kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do
-           kubectl -n $ns get pod $p -o yaml >> $log_dir/nydus-pods.conf
-           kubectl -n $ns describe pod $p  >> $log_dir/nydus-pods.conf
-           kubectl -n $ns logs $p -c nydus-snapshotter >> $log_dir/nydus-snapshotter.log || echo "failed to get snapshotter log"
+          for p in `kubectl --namespace "$NAMESPACE" get pods --no-headers -o custom-columns=NAME:metadata.name`; do
+           kubectl --namespace "$NAMESPACE" get pod $p -o yaml >> $log_dir/nydus-pods.conf
+           kubectl --namespace "$NAMESPACE" describe pod $p  >> $log_dir/nydus-pods.conf
+           kubectl --namespace "$NAMESPACE" logs $p -c nydus-snapshotter >> $log_dir/nydus-snapshotter.log || echo "failed to get snapshotter log"
           done
-          kubectl -n $ns get secrets -o yaml >> $log_dir/nydus-secrets.log
+          kubectl --namespace "$NAMESPACE" get secrets -o yaml >> $log_dir/nydus-secrets.log
 
           docker exec kind-control-plane cat /etc/containerd/config.toml >> $log_dir/containerd-config.toml
           docker exec kind-control-plane containerd config dump >> $log_dir/containerd-config-dump.toml
@@ -150,9 +54,10 @@ jobs:
 
           cat ~/.docker/config.json > $log_dir/docker.config.json || echo "~/.docker/config.json  not found"
       - name: Upload Logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: k8s-e2e-tests-logs
           path: |
             /tmp/nydus-log
+          overwrite: true

.github/workflows/optimizer.yml

@@ -0,0 +1,109 @@
+name: optimizer test
+
+on:
+  push:
+    branches:
+      - "main"
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+  pull_request:
+    branches:
+      - "main"
+  schedule:
+    # Trigger test every day at 00:03 clock UTC
+    - cron: "3 0 * * *"
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  run_optimizer:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          cache-dependency-path: "go.sum"
+      - name: cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            tools/optimizer-server/target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('tools/optimizer-server/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo
+      - name: containerd runc and crictl
+        run: |
+          sudo wget -q https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.0/crictl-v1.26.0-linux-amd64.tar.gz
+          sudo tar zxvf ./crictl-v1.26.0-linux-amd64.tar.gz -C /usr/local/bin
+          sudo install -D -m 755 misc/optimizer/crictl.yaml /etc/crictl.yaml
+          sudo wget -q https://github.com/containerd/containerd/releases/download/v1.7.0/containerd-static-1.7.0-linux-amd64.tar.gz
+          sudo systemctl stop containerd
+          sudo tar -zxf ./containerd-static-1.7.0-linux-amd64.tar.gz -C /usr/
+          sudo install -D -m 755 misc/optimizer/containerd-config.toml /etc/containerd/config.toml
+          sudo systemctl restart containerd
+          sudo wget -q https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 -O /usr/bin/runc
+          sudo chmod +x /usr/bin/runc
+      - name: Setup CNI
+        run: |
+          wget -q https://github.com/containernetworking/plugins/releases/download/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+          sudo mkdir -p /opt/cni/bin
+          sudo tar xzf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/
+          sudo install -D -m 755 misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist
+      - name: Build and install optimizer
+        run: |
+          rustup component add rustfmt clippy
+          make optimizer
+          sudo chown -R $(id -un):$(id -gn) . ~/.cargo/
+          pwd
+          ls -lh bin/*optimizer*
+          sudo make install-optimizer
+          sudo install -D -m 755 misc/example/optimizer-nri-plugin.conf /etc/nri/conf.d/02-optimizer-nri-plugin.conf
+          sudo systemctl restart containerd
+          systemctl status containerd --no-pager -l
+      - name: Wait containerd ready
+        run: |
+          unset READY
+          for i in $(seq 30); do
+            if eval "timeout 180 ls /run/containerd/containerd.sock"; then
+                READY=true
+                break
+            fi
+            echo "Fail(${i}). Retrying..."
+            sleep 1
+          done
+          if [ "$READY" != "true" ];then
+            echo "containerd is not ready"
+            exit 1
+          fi
+      - name: Generate accessed files list
+        run: |
+          sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
+          sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
+          sleep 20
+          sudo crictl rmp  -f --all
+          tree /opt/nri/optimizer/results/
+          count=$(cat /opt/nri/optimizer/results/library/nginx:1.23.3 | wc -l)
+          expected=$(cat misc/optimizer/script/file_list.txt | wc -l)
+          echo "count: $count expected minimum value: $expected"
+          if [ $count -lt $expected ]; then
+            echo "failed to generate accessed files list for nginx:1.23.3"
+            cat misc/optimizer/script/file_list.txt
+            exit 1
+          fi
+          cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv
+      - name: Dump logs
+        if: failure()
+        continue-on-error: true
+        run: |
+          systemctl status containerd --no-pager -l
+          journalctl -xeu containerd --no-pager
+          

.github/workflows/release.yml

@@ -12,94 +12,175 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - build-os: linux
+            build-arch: amd64
+            build-linker: x86-64
+          - build-os: linux
+            build-arch: arm64
+            build-linker: aarch64
+          - build-os: linux
+            build-arch: s390x
+            build-linker: s390x
+          - build-os: linux
+            build-arch: ppc64le
+            build-linker: powerpc64le
+          - build-os: linux
+            build-arch: riscv64
+            build-linker: riscv64
+          - build-os: linux
+            build-arch: static
+            build-linker: static
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.19.6"
+          go-version: "1.22.5"
       - name: cache go mod
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
+          key: ${{ matrix.build-os }}-go-${{ hashFiles('go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go
-      - name: build nydus-snapshotter
+            ${{ matrix.build-os }}-go
+      - name: cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            tools/optimizer-server/target/
+          key: ${{ matrix.build-os }}-cargo-${{ hashFiles('tools/optimizer-server/Cargo.lock') }}
+          restore-keys: |
+            ${{ matrix.build-os }}-cargo
+      - name: install gnu gcc linker
+        run: |
+          if [ "${{ matrix.build-linker }}" != "static" ]; then
+            sudo apt-get install -qq gcc-${{ matrix.build-linker }}-linux-gnu
+          fi
+      - name: build nydus-snapshotter and optimizer
         run: |
           go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2
           export PATH=$PATH:$(go env GOPATH)/bin
-          make
+          if [ "${{ matrix.build-arch }}" == "static" ]; then
+            make static-package
+          else
+            make package GOOS=${{ matrix.build-os }} GOARCH=${{ matrix.build-arch }}
+          fi
       - name: upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: nydus-snapshotter_artifacts
+          name: release-tars-${{ matrix.build-os }}-${{ matrix.build-arch }}
           path: |
-            bin/containerd-nydus-grpc
+            package/*.tar.gz*
+          overwrite: true
   upload:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [build]
     steps:
-      - uses: actions/checkout@v3
-      - name: install hub
-        run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub'
-          sudo mv hub /usr/local/bin/hub
-      - name: download artifacts
-        uses: actions/download-artifact@v3
+      - uses: actions/checkout@v4
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
         with:
-          name: nydus-snapshotter_artifacts
-          path: nydus-snapshotter
-      - name: upload artifacts
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="nydus-snapshotter-$tag-x86_64.tgz"
-          chmod +x nydus-snapshotter/*
-          tar cf - nydus-snapshotter | gzip > ${tarball}
-          echo "tag=$tag" >> $GITHUB_ENV
-          echo "tarball=$tarball" >> $GITHUB_ENV
-
-          tarball_shasum="$tarball.sha256sum"
-          sha256sum $tarball > $tarball_shasum
-          echo "tarball_shasum=$tarball_shasum" >> $GITHUB_ENV
+          path: builds
       - name: Release
         uses: softprops/action-gh-release@v1
         with:
-          name: "Nydus Snapshotter ${{ env.tag }} Release"
+          name: "Nydus Snapshotter ${{ github.ref_name }} Release"
           generate_release_notes: true
           files: |
-            ${{ env.tarball }}
-            ${{ env.tarball_shasum }}
+            builds/release-tars-**/*
 
   publish-image:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: [build]
+    strategy:
+      matrix:
+        include:
+          - build-os: linux
+            build-arch: amd64
+          - build-os: linux
+            build-arch: arm64
+          - build-os: linux
+            build-arch: s390x
+          - build-os: linux
+            build-arch: ppc64le
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to the container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: download artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: nydus-snapshotter_artifacts
+          name: release-tars-${{ matrix.build-os }}-${{ matrix.build-arch }}
           path: misc/snapshotter
+      - name: unpack static release
+        run: |
+          cd misc/snapshotter && tar -zxf *.tar.gz && mv bin/* . && rm -rf bin
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v4
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Get the nydusd version
+        run: |
+          export NYDUS_STABLE_VER=$(curl -fsSL https://api.github.com/repos/dragonflyoss/nydus/releases/latest | jq -r .tag_name)
+          echo "NYDUS_STABLE_VER=$NYDUS_STABLE_VER" >> "$GITHUB_ENV"
+          printf 'nydus version is: %s\n' "$NYDUS_STABLE_VER"
       - name: build and push nydus-snapshotter image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: misc/snapshotter
           file: misc/snapshotter/Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          platforms: ${{ matrix.build-os }}/${{ matrix.build-arch }}
+          provenance: false
+          tags: |
+            ${{ fromJSON(steps.meta.outputs.json).tags[0] }}-${{ matrix.build-arch }}
+            ${{ fromJSON(steps.meta.outputs.json).tags[1] }}-${{ matrix.build-arch }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: NYDUS_VER=${{ env.NYDUS_STABLE_VER }}
+
+  publish-manifest:
+    runs-on: ubuntu-22.04
+    needs: [publish-image]
+    steps:
+      - name: Log in to the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Publish manifest for multi-arch image
+        run: |
+          IFS=',' read -ra tags <<< "$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ',')"
+          for tag in "${tags[@]}"; do
+            docker manifest create "${tag}" \
+              --amend "${tag}-amd64" \
+              --amend "${tag}-arm64" \
+              --amend "${tag}-s390x" \
+              --amend "${tag}-ppc64le"
+            docker manifest push "${tag}"
+          done

.gitignore

@@ -5,3 +5,4 @@ coverage.txt
 tests/output/
 smoke.tests
 tools/optimizer-server/target
+vendor/

.golangci.yml

@@ -2,13 +2,29 @@
 
 run:
   concurrency: 4
-  deadline: 5m
+  timeout: 5m
   issues-exit-code: 1
   tests: true
-  skip-dirs:
+
+issues:
+  exclude-dirs:
     - misc
+    # The package is ported from containerd project, let's skip it.
+    - pkg/remote/remotes
 
 linters-settings:
+  depguard:
+    rules:
+      main:
+        deny:
+          - pkg: "github.com/containerd/containerd/errdefs"
+            desc: The containerd errdefs package was migrated to a separate module. Use github.com/containerd/errdefs instead.
+          - pkg: "github.com/containerd/containerd/log"
+            desc: The containerd log package was migrated to a separate module. Use github.com/containerd/log instead.
+          - pkg: "github.com/containerd/containerd/platforms"
+            desc: The containerd platforms package was migrated to a separate module. Use github.com/containerd/platforms instead.
+          - pkg: "github.com/containerd/containerd/reference/docker"
+            desc: The containerd platforms package was migrated to a separate module. Use github.com/distribution/reference instead.
   #   govet:
   #     check-shadowing: true
   #     enable:
@@ -32,13 +48,14 @@ linters-settings:
 
 linters:
   enable:
+    - depguard      # Checks for imports that shouldn't be used.
     - staticcheck
     - unconvert
     - gofmt
     - goimports
     - revive
     - ineffassign
-    - vet
+    - govet
     - unused
     - misspell
     - bodyclose
@@ -50,11 +67,11 @@ linters:
     # - goerr113
     - exportloopref
     # - gosec
-    - govet
     - gocritic
-    # - prealloc
+    - prealloc
     - tenv
     # - funlen
     - exhaustive
     - errcheck
   disable:
+    - gosec

MAINTAINERS

@@ -12,3 +12,5 @@ imeoer, Yan Song, yansong.ys@antgroup.com
 #
 # REVIEWERS
 # GitHub ID, Name, Email address
+sctb512, Bin Tang, tangbin.bin@bytedance.com
+

Makefile

@@ -1,5 +1,5 @@
-all: clear build
-optimizer: clear-optimizer build-optimizer
+all: clean build
+optimizer: clean-optimizer build-optimizer
 
 PKG = github.com/containerd/nydus-snapshotter
 PACKAGES ?= $(shell go list ./... | grep -v /tests)
@@ -16,6 +16,10 @@ BUILD_TIMESTAMP=$(shell date '+%Y-%m-%dT%H:%M:%S')
 VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
 REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
 
+
+RELEASE=nydus-snapshotter-v$(VERSION:v%=%)-${GOOS}-${GOARCH}
+STATIC_RELEASE=nydus-snapshotter-v$(VERSION:v%=%)-${GOOS}-static
+
 # Relpace test target images for e2e tests.
 ifdef E2E_TEST_TARGET_IMAGES_FILE
 ENV_TARGET_IMAGES_FILE = --env-file ${E2E_TEST_TARGET_IMAGES_FILE}
@@ -43,44 +47,75 @@ SOURCE_NYDUSD_CONFIG=misc/snapshotter/nydusd-config.${FS_DRIVER}.json
 SNAPSHOTTER_SYSTEMD_UNIT_SERVICE=misc/snapshotter/nydus-snapshotter.${FS_DRIVER}.service
 
 LDFLAGS = -s -w -X ${PKG}/version.Version=${VERSION} -X ${PKG}/version.Revision=$(REVISION) -X ${PKG}/version.BuildTimestamp=$(BUILD_TIMESTAMP)
+DEBUG_LDFLAGS = -X ${PKG}/version.Version=${VERSION} -X ${PKG}/version.Revision=$(REVISION) -X ${PKG}/version.BuildTimestamp=$(BUILD_TIMESTAMP)
 
 CARGO ?= $(shell which cargo)
 OPTIMIZER_SERVER = tools/optimizer-server
 OPTIMIZER_SERVER_TOML = ${OPTIMIZER_SERVER}/Cargo.toml
-OPTIMIZER_SERVER_BIN = ${OPTIMIZER_SERVER}/target/release/optimizer-server
-
+OPTIMIZER_SERVER_BIN = ${OPTIMIZER_SERVER}/bin/optimizer-server
 .PHONY: build
 build:
 	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+
+.PHONY: static
+static:
+	CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+
+debug:
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(DEBUG_LDFLAGS)" -gcflags "-N -l" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(DEBUG_LDFLAGS)" -gcflags "-N -l" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
 
 .PHONY: build-optimizer
 build-optimizer:
-	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/02-optimizer-nri-plugin ./cmd/optimizer-nri-plugin
-	${CARGO} fmt --manifest-path ${OPTIMIZER_SERVER_TOML} -- --check
-	${CARGO} build --release --manifest-path ${OPTIMIZER_SERVER_TOML} && cp ${OPTIMIZER_SERVER_BIN} ./bin
-	${CARGO} clippy --manifest-path ${OPTIMIZER_SERVER_TOML} --bins -- -Dwarnings
+	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/optimizer-nri-plugin ./cmd/optimizer-nri-plugin
+	make -C tools/optimizer-server release OS=$(GOOS) ARCH=$(GOARCH) && cp ${OPTIMIZER_SERVER_BIN} ./bin
 
 static-release:
 	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/optimizer-nri-plugin ./cmd/optimizer-nri-plugin
+	make -C tools/optimizer-server static-release && cp ${OPTIMIZER_SERVER_BIN} ./bin
+
+package/$(RELEASE).tar.gz: build build-optimizer
+	mkdir -p package
+	rm -rf package/$(RELEASE) package/$(RELEASE).tar.gz
+	tar -czf package/$(RELEASE).tar.gz bin
+	rm -rf package/$(RELEASE)
+
+package/$(STATIC_RELEASE).tar.gz: static-release
+	@mkdir -p package
+	@rm -rf package/$(STATIC_RELEASE) package/$(STATIC_RELEASE).tar.gz
+	@tar -czf package/$(STATIC_RELEASE).tar.gz bin
+	@rm -rf package/$(STATIC_RELEASE)
+
+package: package/$(RELEASE).tar.gz
+	cd package && sha256sum $(RELEASE).tar.gz >$(RELEASE).tar.gz.sha256sum
+
+static-package: package/$(STATIC_RELEASE).tar.gz
+	@cd package && sha256sum $(STATIC_RELEASE).tar.gz >$(STATIC_RELEASE).tar.gz.sha256sum
 
 # Majorly for cross build for converter package since it is imported by other projects
 converter:
 	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/converter ./cmd/converter
 
-.PHONY: clear
-clear:
+.PHONY: clean
+clean:
 	rm -f bin/*
 	rm -rf _out
 
-.PHONY: clear-optimizer
-clear-optimizer:
-	rm -rf bin/02-optimizer-nri-plugin
-	${CARGO} clean --manifest-path ${OPTIMIZER_SERVER_TOML}
+.PHONY: clean-optimizer
+clean-optimizer:
+	rm -rf bin/optimizer-nri-plugin bin/optimizer-server
+	make -C tools/optimizer-server clean
 
 .PHONY: install
 install:
 	@echo "+ $@ bin/containerd-nydus-grpc"
 	@sudo install -D -m 755 bin/containerd-nydus-grpc /usr/local/bin/containerd-nydus-grpc
+	@echo "+ $@ bin/nydus-overlayfs"
+	@sudo install -D -m 755 bin/nydus-overlayfs /usr/local/bin/nydus-overlayfs
 
 	@if [ ! -e ${NYDUSD_CONFIG} ]; then echo "+ $@ SOURCE_NYDUSD_CONFIG"; sudo install -D -m 664 ${SOURCE_NYDUSD_CONFIG} ${NYDUSD_CONFIG}; fi
 	@if [ ! -e ${SNAPSHOTTER_CONFIG} ]; then echo "+ $@ ${SOURCE_SNAPSHOTTER_CONFIG}"; sudo install -D -m 664 ${SOURCE_SNAPSHOTTER_CONFIG} ${SNAPSHOTTER_CONFIG}; fi
@@ -93,9 +128,9 @@ install:
 	@if which systemctl >/dev/null; then sudo systemctl enable /etc/systemd/system/nydus-snapshotter.service; sudo systemctl restart nydus-snapshotter; fi
 
 install-optimizer:
-	sudo install -D -m 755 bin/02-optimizer-nri-plugin /opt/nri/plugins/02-optimizer-nri-plugin
+	sudo install -D -m 755 bin/optimizer-nri-plugin /opt/nri/plugins/02-optimizer-nri-plugin
 	sudo install -D -m 755 bin/optimizer-server /usr/local/bin/optimizer-server
-	sudo install -D -m 755 misc/example/02-optimizer-nri-plugin.conf /etc/nri/conf.d/02-optimizer-nri-plugin.conf
+	sudo install -D -m 755 misc/example/optimizer-nri-plugin.conf /etc/nri/conf.d/02-optimizer-nri-plugin.conf
 
 	@sudo mkdir -p /opt/nri/optimizer/results
 
@@ -124,8 +159,9 @@ smoke:
 .PHONY: integration
 integration:
 	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
-	$(SUDO) DOCKER_BUILDKIT=1 docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile .
-	$(SUDO) docker run --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \
+	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+	$(SUDO) docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile .
+	$(SUDO) docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \
 	-v `go env GOCACHE`:/root/.cache/go-build -v `pwd`:/nydus-snapshotter \
 	-v /usr/src/linux-headers-${KERNEL_VER}:/usr/src/linux-headers-${KERNEL_VER} \
 	${ENV_TARGET_IMAGES_FILE}  \

README.md

@@ -1,27 +1,35 @@
+[**[⬇️ Download]**](https://github.com/containerd/nydus-snapshotter/releases)
+[**[📖 Website]**](https://nydus.dev/)
+[**[☸ Quick Start (Kubernetes)**]](https://github.com/containerd/nydus-snapshotter/blob/main/docs/run_nydus_in_kubernetes.md)
+[**[🤓 Quick Start (nerdctl)**]](https://github.com/containerd/nerdctl/blob/master/docs/nydus.md)
+[**[❓ FAQs & Troubleshooting]**](https://github.com/dragonflyoss/nydus/wiki/FAQ)
+
 # Nydus Snapshotter
 
-<p><img src="https://github.com/dragonflyoss/image-service/blob/master/misc/logo.svg" width="170"></p>
+<p><img src="https://github.com/dragonflyoss/nydus/blob/master/misc/logo.svg" width="170"></p>
 
 [![Release Version](https://img.shields.io/github/v/release/containerd/nydus-snapshotter?style=flat)](https://github.com/containerd/nydus-snapshotter/releases)
 [![LICENSE](https://img.shields.io/github/license/containerd/nydus-snapshotter.svg?style=flat)](https://github.com/containerd/nydus-snapshotter/blob/main/LICENSE)
 ![CI](https://github.com/containerd/nydus-snapshotter/actions/workflows/ci.yml/badge.svg?event=push)
 [![Go Report Card](https://goreportcard.com/badge/github.com/containerd/nydus-snapshotter?style=flat)](https://goreportcard.com/report/github.com/containerd/nydus-snapshotter)
 [![Twitter](https://img.shields.io/twitter/url?style=social&url=https%3A%2F%2Ftwitter.com%2Fdragonfly_oss)](https://twitter.com/dragonfly_oss)
-[![Nydus Stars](https://img.shields.io/github/stars/dragonflyoss/image-service?label=Nydus%20Stars&style=social)](https://github.com/dragonflyoss/image-service)
+[![Nydus Stars](https://img.shields.io/github/stars/dragonflyoss/nydus?label=Nydus%20Stars&style=social)](https://github.com/dragonflyoss/nydus)
 
 Nydus-snapshotter is a **non-core** sub-project of containerd.
 
 Nydus snapshotter is an external plugin of containerd for [Nydus image service](https://nydus.dev) which implements a chunk-based content-addressable filesystem on top of a called `RAFS (Registry Acceleration File System)` format that improves the current OCI image specification, in terms of container launching speed, image space, and network bandwidth efficiency, as well as data integrity with several runtime backends: FUSE, virtiofs and in-kernel [EROFS](https://www.kernel.org/doc/html/latest/filesystems/erofs.html).
 
-Nydus supports lazy pulling feature since pulling image is one of the time-consuming steps in the container lifecycle. Lazy pulling here means a container can run even the image is partially available and necessary chunks of the image are fetched on-demand. Apart from that, Nydus also supports [(e)Stargz](https://github.com/containerd/stargz-snapshotter) and OCI (by using zran) lazy pulling directly **WITHOUT** any explicit conversion.
+Nydus supports lazy pulling feature since pulling image is one of the time-consuming steps in the container lifecycle. Lazy pulling here means a container can run even the image is partially available and necessary chunks of the image are fetched on-demand. Apart from that, Nydus also supports [(e)Stargz](https://github.com/containerd/stargz-snapshotter) and OCI (by using [zran](https://github.com/dragonflyoss/nydus/blob/master/docs/nydus-zran.md)) lazy pulling directly **WITHOUT** any explicit conversion.
 
-For more details about how to build Nydus container image, please refer to [nydusify](https://github.com/dragonflyoss/image-service/blob/master/docs/nydusify.md) conversion tool and [acceld](https://github.com/goharbor/acceleration-service).
+For more details about how to build Nydus container image, please refer to [nydusify](https://github.com/dragonflyoss/nydus/blob/master/docs/nydusify.md) conversion tool and [acceld](https://github.com/goharbor/acceleration-service).
 
-## Architecture Based on FUSE
+## Architecture
+
+### Architecture Based on FUSE
 
 ![fuse arch](./docs/diagram/nydus_fuse_arch.svg)
 
-## Architecture Based on Fscache/Erofs
+### Architecture Based on Fscache/Erofs
 
 ![fscache arch](./docs/diagram/nydus_fscache_erofs_arch.svg)
 
@@ -54,17 +62,17 @@ Restart your containerd service making the change take effect. Assume that your
 systemctl restart containerd
 ```
 
-## Get Nydus Binaries
+### Get Nydus Binaries
 
-Get `nydusd` `nydus-image` and `nydusctl` binaries from [nydus releases page](https://github.com/dragonflyoss/image-service/releases).
+Get `nydusd` `nydus-image` and `nydusctl` binaries from [nydus releases page](https://github.com/dragonflyoss/nydus/releases).
 It's suggested to install the binaries to your system path. `nydusd` is FUSE userspace daemon and a vhost-user-fs backend. Nydus-snapshotter
 will fork a nydusd process when necessary.
 
-## Configure Nydus
+### Configure Nydus
 
 Please follow instructions to [configure nydus](./docs/configure_nydus.md) in order to make it work properly in your environment.
 
-## Start Nydus Snapshotter
+### Start Nydus Snapshotter
 
 Nydus-snapshotter is implemented as a [proxy plugin](https://github.com/containerd/containerd/blob/04985039cede6aafbb7dfb3206c9c4d04e2f924d/PLUGINS.md#proxy-plugins) (`containerd-nydus-grpc`) for containerd.
 
@@ -83,7 +91,7 @@ Or you can start nydus-snapshotter manually.
 # Otherwise, provide them in below command line.
 # `address` is the domain socket that you configured in containerd configuration file
 # `--nydusd-config` is the path to `nydusd` configuration file
-# The default nydus-snapshotter work directory is located at `/var/lib/containerd-nydus`
+# The default nydus-snapshotter work directory is located at `/var/lib/containerd/io.containerd.snapshotter.v1.nydus`
 
 $ sudo ./containerd-nydus-grpc --config /etc/nydus/config.toml --nydusd-config /etc/nydus/nydusd-config.json --log-to-stdout
 ```
@@ -98,12 +106,12 @@ TYPE                            ID                       PLATFORMS      STATUS
 io.containerd.snapshotter.v1    nydus                    -              ok
 ```
 
-## Optimize Nydus Image as per Workload
+### Optimize Nydus Image as per Workload
 
 Nydus usually prefetch image data to local filesystem before a real user on-demand read. It helps to improve the performance and availability. A containerd NRI plugin [container image optimizer](docs/optimize_nydus_image.md) can be used to generate nydus image building suggestions to optimize your nydus image making the nydusd runtime match your workload IO pattern. The optimized nydus image has
 a better performance.
 
-## Quickly Start Container with Lazy Pulling
+## Quickstart Container with Lazy Pulling
 
 ### Start Container on single Node
 
@@ -130,9 +138,17 @@ Use `crictl` to debug starting container via Kubernetes CRI. Dry run [steps](./d
 
 We can also use the `nydus-snapshotter` container image when we want to put Nydus stuffs inside a container. See the [nydus-snapshotter example](./misc/example/README.md) for how to setup and use it.
 
-## Integrate with Dragonfly to Distribute Images in P2P
+## Integrate with Dragonfly to Distribute Images by P2P
 
-Nydus is also a sub-project of [Dragonfly](https://github.com/dragonflyoss/Dragonfly2). So it closely works with Dragonfly to distribute container images in a fast and efficient P2P fashion to reduce network latency and lower the pressure on a single-point of the registry.
+Nydus is a sub-project of [Dragonfly](https://github.com/dragonflyoss/dragonfly). So it closely works with Dragonfly to distribute container images in a fast and efficient P2P fashion to reduce network latency and lower the pressure on a single-point of the registry.
+
+### Quickstart Dragonfly & Nydus in Kubernetes
+
+We recommend using the Dragonfly P2P data distribution system to further improve the runtime performance of Nydus images.
+
+If you want to deploy Dragonfly and Nydus at the same time, please refer to this **[Quick Start](https://github.com/dragonflyoss/helm-charts/blob/main/INSTALL.md)**.
+
+### Config Dragonfly mode
 
 Dragonfly supports both **mirror** mode and HTTP **proxy** mode to boost the containers startup. It is suggested to use Dragonfly mirror mode. To integrate with Dragonfly in the mirror mode, please provide registry mirror in nydusd's json configuration file in section `device.backend.mirrors`
 
@@ -141,34 +157,33 @@ Dragonfly supports both **mirror** mode and HTTP **proxy** mode to boost the con
   "mirrors": [
     {
       "host": "http://127.0.0.1:65001",
-      "headers": "https://index.docker.io/v1/",
-      "auth_through": false
+      "headers": "https://index.docker.io/v1/"
     }
   ]
 }
 ```
 
-`auth_through=false` means nydusd's authentication request will directly go to original registry rather than relayed by Dragonfly.
-
 ### Hot updating mirror configurations
+
 In addition to setting the registry mirror in nydusd's json configuration file, `nydus-snapshotter` also supports hot updating mirror configurations. You can set the configuration directory in nudus-snapshotter's toml configuration file with `remote.mirrors_config.dir`. The empty `remote.mirrors_config.dir` means disabling it.
+
 ```toml
 [remote.mirrors_config]
 dir = "/etc/nydus/certs.d"
 ```
+
 Configuration file is compatible with containerd's configuration file in toml format.
+
 ```toml
-server = "https://p2p-nydus.com"
 [host]
   [host."http://127.0.0.1:65001"]
-    auth_through = false
     [host."http://127.0.0.1:65001".header]
+      # NOTE: For Dragonfly, the HTTP scheme must be explicitly specified.
       X-Dragonfly-Registry = ["https://p2p-nydus.com"]
 ```
 
 Mirror configurations loaded from nydusd's json file will be overwritten before pulling image if the valid mirror configuration items loaded from `remote.mirrors_config.dir` are greater than 0.
 
-
 ## Community
 
 Nydus aims to form a **vendor-neutral opensource** image distribution solution to all communities.
@@ -178,12 +193,10 @@ We're very pleased to hear your use cases any time.
 Feel free to reach/join us via Slack and/or Dingtalk.
 
 - **Slack:** [Nydus Workspace](https://join.slack.com/t/nydusimageservice/shared_invite/zt-pz4qvl4y-WIh4itPNILGhPS8JqdFm_w)
-
 - **Twitter:** [@dragonfly_oss](https://twitter.com/dragonfly_oss)
-
 - **Dingtalk:** [34971767](https://qr.dingtalk.com/action/joingroup?code=v1,k1,ioWGzuDZEIO10Bf+/ohz4RcQqAkW0MtOwoG1nbbMxQg=&_dt_no_comment=1&origin=11)
 
-<img src="https://github.com/dragonflyoss/image-service/blob/master/misc/dingtalk.jpg" width="250" height="300"/>
+<img src="https://github.com/dragonflyoss/nydus/blob/master/misc/dingtalk.jpg" width="250" height="300"/>
 
 - **Technical Meeting:** Every Wednesday at 06:00 UTC (Beijing, Shanghai 14:00), please see our [HackMD](https://hackmd.io/@Nydus/Bk8u2X0p9) page for more information.
 

cmd/containerd-nydus-grpc/main.go

@@ -11,7 +11,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
@@ -30,7 +30,7 @@ func main() {
 		Version:     version.Version,
 		Flags:       flags.F,
 		HideVersion: true,
-		Action: func(c *cli.Context) error {
+		Action: func(_ *cli.Context) error {
 			if flags.Args.PrintVersion {
 				fmt.Println("Version:    ", version.Version)
 				fmt.Println("Revision:   ", version.Revision)
@@ -44,7 +44,7 @@ func main() {
 			var snapshotterConfig config.SnapshotterConfig
 
 			if err := defaultSnapshotterConfig.FillUpWithDefaults(); err != nil {
-				return errors.New("failed to fill up nydus configuration with defaults")
+				return errors.New("failed to generate nydus default configuration")
 			}
 
 			// Once snapshotter's configuration file is provided, parse it and let command line parameters override it.
@@ -52,28 +52,32 @@ func main() {
 				if c, err := config.LoadSnapshotterConfig(snapshotterConfigPath); err == nil {
 					// Command line parameters override the snapshotter's configurations for backwards compatibility
 					if err := config.ParseParameters(flags.Args, c); err != nil {
-						return errors.Wrap(err, "parse parameters")
+						return errors.Wrap(err, "failed to parse commandline options")
 					}
 					snapshotterConfig = *c
 				} else {
-					return errors.Wrapf(err, "Failed to load snapshotter's configuration at %q", snapshotterConfigPath)
+					return errors.Wrapf(err, "failed to load snapshotter configuration from %q", snapshotterConfigPath)
 				}
 			} else {
 				if err := config.ParseParameters(flags.Args, &snapshotterConfig); err != nil {
-					return errors.Wrap(err, "parse parameters")
+					return errors.Wrap(err, "failed to parse commandline options")
 				}
 			}
 
 			if err := config.MergeConfig(&snapshotterConfig, &defaultSnapshotterConfig); err != nil {
-				return errors.Wrap(err, "merge configurations")
+				return errors.Wrap(err, "failed to merge configurations")
 			}
 
 			if err := config.ValidateConfig(&snapshotterConfig); err != nil {
-				return errors.Wrapf(err, "validate configurations")
+				return errors.Wrapf(err, "failed to validate configurations")
 			}
 
 			if err := config.ProcessConfigurations(&snapshotterConfig); err != nil {
-				return errors.Wrap(err, "process configurations")
+				return errors.Wrap(err, "failed to process configurations")
+			}
+
+			if err := config.SetUpEnvironment(&snapshotterConfig); err != nil {
+				return errors.Wrap(err, "failed to setup environment")
 			}
 
 			ctx := logging.WithContext()
@@ -87,11 +91,11 @@ func main() {
 			}
 
 			if err := logging.SetUp(logConfig.LogLevel, logConfig.LogToStdout, logConfig.LogDir, logRotateArgs); err != nil {
-				return errors.Wrap(err, "set up logger")
+				return errors.Wrap(err, "failed to setup logger")
 			}
 
-			log.L.Infof("Start nydus-snapshotter. PID %d Version %s FsDriver %s DaemonMode %s",
-				os.Getpid(), version.Version, config.GetFsDriver(), snapshotterConfig.DaemonMode)
+			log.L.Infof("Start nydus-snapshotter. Version: %s, PID: %d, FsDriver: %s, DaemonMode: %s",
+				version.Version, os.Getpid(), config.GetFsDriver(), snapshotterConfig.DaemonMode)
 
 			return Start(ctx, &snapshotterConfig)
 		},

cmd/containerd-nydus-grpc/snapshotter.go

@@ -21,9 +21,9 @@ import (
 	"github.com/containerd/nydus-snapshotter/snapshot"
 
 	api "github.com/containerd/containerd/api/services/snapshots/v1"
-	"github.com/containerd/containerd/contrib/snapshotservice"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/snapshots"
+	"github.com/containerd/containerd/v2/contrib/snapshotservice"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/log"
 	"google.golang.org/grpc"
 )
 
@@ -64,7 +64,7 @@ func Serve(ctx context.Context, sn snapshots.Snapshotter, options ServeOptions,
 	}
 	rpc := grpc.NewServer()
 	if rpc == nil {
-		return errors.New("start RPC server")
+		return errors.New("start gRPC server")
 	}
 	api.RegisterSnapshotsServer(rpc, snapshotservice.FromSnapshotter(sn))
 	listener, err := net.Listen("unix", options.ListeningSocketPath)
@@ -86,7 +86,7 @@ func Serve(ctx context.Context, sn snapshots.Snapshotter, options ServeOptions,
 		}
 
 		if err := listener.Close(); err != nil {
-			log.L.Errorf("failed to close listener %s, err: %v", options.ListeningSocketPath, err)
+			log.L.Errorf("Failed to close listener %s, err: %v", options.ListeningSocketPath, err)
 		}
 	}()
 

cmd/nydus-overlayfs/main.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// Extra mount option to pass Nydus specific information from snapshotter to runtime through containerd.
+	extraOptionKey = "extraoption="
+	// Kata virtual volume infmation passed from snapshotter to runtime through containerd, superset of `extraOptionKey`.
+	// Please refer to `KataVirtualVolume` in https://github.com/kata-containers/kata-containers/blob/main/src/libs/kata-types/src/mount.rs
+	kataVolumeOptionKey = "io.katacontainers.volume="
+)
+
+var (
+	Version   = "v0.1"
+	BuildTime = "unknown"
+)
+
+/*
+containerd run fuse.mount format: nydus-overlayfs overlay /tmp/ctd-volume107067851
+-o lowerdir=/foo/lower2:/foo/lower1,upperdir=/foo/upper,workdir=/foo/work,extraoption={...},dev,suid]
+*/
+type mountArgs struct {
+	fsType  string
+	target  string
+	options []string
+}
+
+func parseArgs(args []string) (*mountArgs, error) {
+	margs := &mountArgs{
+		fsType: args[0],
+		target: args[1],
+	}
+	if margs.fsType != "overlay" {
+		return nil, errors.Errorf("invalid filesystem type %s for overlayfs", margs.fsType)
+	}
+	if len(margs.target) == 0 {
+		return nil, errors.New("empty overlayfs mount target")
+	}
+
+	if args[2] == "-o" && len(args[3]) != 0 {
+		for _, opt := range strings.Split(args[3], ",") {
+			// filter Nydus specific options
+			if strings.HasPrefix(opt, extraOptionKey) || strings.HasPrefix(opt, kataVolumeOptionKey) {
+				continue
+			}
+			margs.options = append(margs.options, opt)
+		}
+	}
+	if len(margs.options) == 0 {
+		return nil, errors.New("empty overlayfs mount options")
+	}
+
+	return margs, nil
+}
+
+func parseOptions(options []string) (int, string) {
+	flagsTable := map[string]int{
+		"async":         unix.MS_SYNCHRONOUS,
+		"atime":         unix.MS_NOATIME,
+		"bind":          unix.MS_BIND,
+		"defaults":      0,
+		"dev":           unix.MS_NODEV,
+		"diratime":      unix.MS_NODIRATIME,
+		"dirsync":       unix.MS_DIRSYNC,
+		"exec":          unix.MS_NOEXEC,
+		"mand":          unix.MS_MANDLOCK,
+		"noatime":       unix.MS_NOATIME,
+		"nodev":         unix.MS_NODEV,
+		"nodiratime":    unix.MS_NODIRATIME,
+		"noexec":        unix.MS_NOEXEC,
+		"nomand":        unix.MS_MANDLOCK,
+		"norelatime":    unix.MS_RELATIME,
+		"nostrictatime": unix.MS_STRICTATIME,
+		"nosuid":        unix.MS_NOSUID,
+		"rbind":         unix.MS_BIND | unix.MS_REC,
+		"relatime":      unix.MS_RELATIME,
+		"remount":       unix.MS_REMOUNT,
+		"ro":            unix.MS_RDONLY,
+		"rw":            unix.MS_RDONLY,
+		"strictatime":   unix.MS_STRICTATIME,
+		"suid":          unix.MS_NOSUID,
+		"sync":          unix.MS_SYNCHRONOUS,
+	}
+
+	var (
+		flags int
+		data  []string
+	)
+	for _, o := range options {
+		if f, exist := flagsTable[o]; exist {
+			flags |= f
+		} else {
+			data = append(data, o)
+		}
+	}
+	return flags, strings.Join(data, ",")
+}
+
+func run(args cli.Args) error {
+	margs, err := parseArgs(args.Slice())
+	if err != nil {
+		return errors.Wrap(err, "parse mount options")
+	}
+
+	flags, data := parseOptions(margs.options)
+	err = syscall.Mount(margs.fsType, margs.target, margs.fsType, uintptr(flags), data)
+	if err != nil {
+		return errors.Wrapf(err, "mount overlayfs by syscall")
+	}
+	return nil
+}
+
+func main() {
+	app := &cli.App{
+		Name:      "NydusOverlayfs",
+		Usage:     "FUSE mount helper for containerd to filter out Nydus specific options",
+		Version:   fmt.Sprintf("%s.%s", Version, BuildTime),
+		UsageText: "[Usage]: nydus-overlayfs overlay <target> -o <options>",
+		Action: func(c *cli.Context) error {
+			return run(c.Args())
+		},
+		Before: func(c *cli.Context) error {
+			if c.NArg() != 4 {
+				cli.ShowAppHelpAndExit(c, 1)
+			}
+			return nil
+		},
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Exit(0)
+}

cmd/optimizer-nri-plugin/main.go

@@ -16,11 +16,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containerd/log"
+	distribution "github.com/distribution/reference"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 
-	"github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nri/pkg/api"
 	"github.com/containerd/nri/pkg/stub"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
@@ -40,6 +40,7 @@ type PluginConfig struct {
 
 	ServerPath string `toml:"server_path"`
 	PersistDir string `toml:"persist_dir"`
+	Readable   bool   `toml:"readable"`
 	Timeout    int    `toml:"timeout"`
 	Overwrite  bool   `toml:"overwrite"`
 }
@@ -86,6 +87,12 @@ func buildFlags(args *PluginArgs) []cli.Flag {
 			Usage:       "the directory to persist accessed files list for container",
 			Destination: &args.Config.PersistDir,
 		},
+		&cli.BoolFlag{
+			Name:        "readable",
+			Value:       false,
+			Usage:       "whether to make the csv file human readable",
+			Destination: &args.Config.Readable,
+		},
 		&cli.IntFlag{
 			Name:        "timeout",
 			Value:       0,
@@ -115,18 +122,20 @@ type plugin struct {
 
 var (
 	cfg                  PluginConfig
-	log                  *logrus.Logger
 	logWriter            *syslog.Writer
-	_                    = stub.ConfigureInterface(&plugin{})
 	globalFanotifyServer = make(map[string]*fanotify.Server)
+
+	_ = stub.ConfigureInterface(&plugin{})
+	_ = stub.StartContainerInterface(&plugin{})
+	_ = stub.StopContainerInterface(&plugin{})
 )
 
 const (
 	imageNameLabel = "io.kubernetes.cri.image-name"
 )
 
-func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, error) {
-	log.Infof("got configuration data: %q from runtime %s %s", config, runtime, version)
+func (p *plugin) Configure(ctx context.Context, config, runtime, version string) (stub.EventMask, error) {
+	log.G(ctx).Infof("got configuration data: %q from runtime %s %s", config, runtime, version)
 	if config == "" {
 		return p.mask, nil
 	}
@@ -144,12 +153,12 @@ func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, err
 		return 0, errors.Wrap(err, "parse events in configuration")
 	}
 
-	log.Infof("configuration: %#v", cfg)
+	log.G(ctx).Infof("configuration: %#v", cfg)
 
 	return p.mask, nil
 }
 
-func (p *plugin) StartContainer(_ *api.PodSandbox, container *api.Container) error {
+func (p *plugin) StartContainer(_ context.Context, _ *api.PodSandbox, container *api.Container) error {
 	dir, imageName, err := GetImageName(container.Annotations)
 	if err != nil {
 		return err
@@ -165,7 +174,7 @@ func (p *plugin) StartContainer(_ *api.PodSandbox, container *api.Container) err
 		persistFile = fmt.Sprintf("%s.timeout%ds", persistFile, cfg.Timeout)
 	}
 
-	fanotifyServer := fanotify.NewServer(cfg.ServerPath, container.Pid, imageName, persistFile, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter)
+	fanotifyServer := fanotify.NewServer(cfg.ServerPath, container.Pid, imageName, persistFile, cfg.Readable, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter)
 
 	if err := fanotifyServer.RunServer(); err != nil {
 		return err
@@ -176,7 +185,7 @@ func (p *plugin) StartContainer(_ *api.PodSandbox, container *api.Container) err
 	return nil
 }
 
-func (p *plugin) StopContainer(_ *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) {
+func (p *plugin) StopContainer(_ context.Context, _ *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) {
 	var update = []*api.ContainerUpdate{}
 	_, imageName, err := GetImageName(container.Annotations)
 	if err != nil {
@@ -192,12 +201,12 @@ func (p *plugin) StopContainer(_ *api.PodSandbox, container *api.Container) ([]*
 }
 
 func GetImageName(annotations map[string]string) (string, string, error) {
-	named, err := docker.ParseDockerRef(annotations[imageNameLabel])
+	named, err := distribution.ParseDockerRef(annotations[imageNameLabel])
 	if err != nil {
 		return "", "", err
 	}
-	nameTagged := named.(docker.NamedTagged)
-	repo := docker.Path(nameTagged)
+	nameTagged := named.(distribution.NamedTagged)
+	repo := distribution.Path(nameTagged)
 
 	dir := filepath.Dir(repo)
 	image := filepath.Base(repo)
@@ -222,7 +231,7 @@ func main() {
 		Version:     version.Version,
 		Flags:       flags.F,
 		HideVersion: true,
-		Action: func(c *cli.Context) error {
+		Action: func(_ *cli.Context) error {
 			var (
 				opts []stub.Option
 				err  error
@@ -230,13 +239,13 @@ func main() {
 
 			cfg = flags.Args.Config
 
-			log = logrus.StandardLogger()
-			log.SetFormatter(&logrus.TextFormatter{
-				PadLevelText: true,
-			})
+			// FIXME(thaJeztah): ucontainerd's log does not set "PadLevelText: true"
+			_ = log.SetFormat(log.TextFormat)
+			ctx := log.WithLogger(context.Background(), log.L)
+
 			logWriter, err = syslog.New(syslog.LOG_INFO, "optimizer-nri-plugin")
 			if err == nil {
-				log.SetOutput(io.MultiWriter(os.Stdout, logWriter))
+				log.G(ctx).Logger.SetOutput(io.MultiWriter(os.Stdout, logWriter))
 			}
 
 			if flags.Args.PluginName != "" {
@@ -249,17 +258,17 @@ func main() {
 			p := &plugin{}
 
 			if p.mask, err = api.ParseEventMask(flags.Args.PluginEvents); err != nil {
-				log.Fatalf("failed to parse events: %v", err)
+				log.G(ctx).Fatalf("failed to parse events: %v", err)
 			}
 			cfg.Events = strings.Split(flags.Args.PluginEvents, ",")
 
 			if p.stub, err = stub.New(p, append(opts, stub.WithOnClose(p.onClose))...); err != nil {
-				log.Fatalf("failed to create plugin stub: %v", err)
+				log.G(ctx).Fatalf("failed to create plugin stub: %v", err)
 			}
 
 			err = p.stub.Run(context.Background())
 			if err != nil {
-				log.Errorf("plugin exited with error %v", err)
+				log.G(ctx).Errorf("plugin exited with error %v", err)
 				os.Exit(1)
 			}
 
@@ -268,9 +277,9 @@ func main() {
 	}
 	if err := app.Run(os.Args); err != nil {
 		if errdefs.IsConnectionClosed(err) {
-			log.Info("optimizer NRI plugin exited")
+			log.L.Info("optimizer NRI plugin exited")
 		} else {
-			log.WithError(err).Fatal("failed to start optimizer NRI plugin")
+			log.L.WithError(err).Fatal("failed to start optimizer NRI plugin")
 		}
 	}
 }

cmd/prefetchfiles-nri-plugin/main.go

@@ -0,0 +1,209 @@
+/*
+* Copyright (c) 2023. Nydus Developers. All rights reserved.
+*
+* SPDX-License-Identifier: Apache-2.0
+ */
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/syslog"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/containerd/log"
+	"github.com/containerd/nri/pkg/api"
+	"github.com/containerd/nri/pkg/stub"
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/version"
+)
+
+const (
+	endpointPrefetch               = "/api/v1/prefetch"
+	defaultEvents                  = "RunPodSandbox"
+	defaultSystemControllerAddress = "/run/containerd-nydus/system.sock"
+	defaultPrefetchConfigDir       = "/etc/nydus"
+	nydusPrefetchAnnotation        = "containerd.io/nydus-prefetch"
+)
+
+type PluginArgs struct {
+	PluginName    string
+	PluginIdx     string
+	SocketAddress string
+}
+
+type Flags struct {
+	Args *PluginArgs
+	Flag []cli.Flag
+}
+
+func buildFlags(args *PluginArgs) []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "name",
+			Usage:       "plugin name to register to NRI",
+			Destination: &args.PluginName,
+		},
+		&cli.StringFlag{
+			Name:        "idx",
+			Usage:       "plugin index to register to NRI",
+			Destination: &args.PluginIdx,
+		},
+		&cli.StringFlag{
+			Name:        "socket-addr",
+			Value:       defaultSystemControllerAddress,
+			Usage:       "unix domain socket address. If defined in the configuration file, there is no need to add ",
+			Destination: &args.SocketAddress,
+		},
+	}
+}
+
+func NewPluginFlags() *Flags {
+	var args PluginArgs
+	return &Flags{
+		Args: &args,
+		Flag: buildFlags(&args),
+	}
+}
+
+type plugin struct {
+	stub stub.Stub
+	mask stub.EventMask
+}
+
+var (
+	globalSocket string
+	logWriter    *syslog.Writer
+
+	_ = stub.RunPodInterface(&plugin{})
+)
+
+// sendDataOverHTTP sends the prefetch data to the specified endpoint over HTTP using a Unix socket.
+func sendDataOverHTTP(data string, endpoint, sock string) error {
+	url := fmt.Sprintf("http://unix%s", endpoint)
+
+	req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", sock)
+			},
+		},
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to send data, status code: %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+
+	return nil
+}
+
+func (p *plugin) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) error {
+	prefetchList, ok := pod.Annotations[nydusPrefetchAnnotation]
+	if !ok {
+		return nil
+	}
+
+	err := sendDataOverHTTP(prefetchList, endpointPrefetch, globalSocket)
+	if err != nil {
+		log.G(ctx).Errorf("failed to send data: %v", err)
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	flags := NewPluginFlags()
+
+	app := &cli.App{
+		Name:        "prefetch-nri-plugin",
+		Usage:       "NRI plugin for obtaining and transmitting prefetch files path",
+		Version:     version.Version,
+		Flags:       flags.Flag,
+		HideVersion: true,
+		Action: func(_ *cli.Context) error {
+			var (
+				opts []stub.Option
+				err  error
+			)
+
+			// FIXME(thaJeztah): ucontainerd's log does not set "PadLevelText: true"
+			_ = log.SetFormat(log.TextFormat)
+			ctx := log.WithLogger(context.Background(), log.L)
+
+			configFileName := "prefetchConfig.toml"
+			configDir := defaultPrefetchConfigDir
+			configFilePath := filepath.Join(configDir, configFileName)
+
+			config, err := toml.LoadFile(configFilePath)
+			if err != nil {
+				log.G(ctx).Warnf("failed to read config file: %v", err)
+			}
+
+			configSocketAddrRaw := config.Get("file_prefetch.socket_address")
+			if configSocketAddrRaw != nil {
+				if configSocketAddr, ok := configSocketAddrRaw.(string); ok {
+					globalSocket = configSocketAddr
+				} else {
+					log.G(ctx).Warnf("failed to read config: 'file_prefetch.socket_address' is not a string")
+				}
+			} else {
+				globalSocket = flags.Args.SocketAddress
+			}
+
+			logWriter, err = syslog.New(syslog.LOG_INFO, "prefetch-nri-plugin")
+			if err == nil {
+				log.G(ctx).Logger.SetOutput(io.MultiWriter(os.Stdout, logWriter))
+			}
+
+			if flags.Args.PluginName != "" {
+				opts = append(opts, stub.WithPluginName(flags.Args.PluginName))
+			}
+			if flags.Args.PluginIdx != "" {
+				opts = append(opts, stub.WithPluginIdx(flags.Args.PluginIdx))
+			}
+
+			p := &plugin{}
+
+			if p.mask, err = api.ParseEventMask(defaultEvents); err != nil {
+				log.G(ctx).Fatalf("failed to parse events: %v", err)
+			}
+
+			if p.stub, err = stub.New(p, opts...); err != nil {
+				log.G(ctx).Fatalf("failed to create plugin stub: %v", err)
+			}
+
+			err = p.stub.Run(context.Background())
+			if err != nil {
+				return errors.Wrap(err, "plugin exited")
+			}
+			return nil
+		},
+	}
+	if err := app.Run(os.Args); err != nil {
+		if errdefs.IsConnectionClosed(err) {
+			log.L.Info("prefetch NRI plugin exited")
+		} else {
+			log.L.WithError(err).Fatal("failed to start prefetch NRI plugin")
+		}
+	}
+}

config/config.go

@@ -10,13 +10,17 @@ package config
 import (
 	"os"
 
-	"github.com/imdario/mergo"
+	"dario.cat/mergo"
 	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
 
+	"github.com/containerd/nydus-snapshotter/internal/constant"
 	"github.com/containerd/nydus-snapshotter/internal/flags"
+	"github.com/containerd/nydus-snapshotter/pkg/cgroup"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/file"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/parser"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/sysinfo"
 )
 
 func init() {
@@ -30,27 +34,33 @@ func init() {
 type DaemonMode string
 
 const (
-	// One nydusd, one rafs instance
-	DaemonModeMultiple DaemonMode = "multiple"
-	// One nydusd serves multiple rafs instances
-	DaemonModeShared DaemonMode = "shared"
-	// No nydusd daemon is needed to be started. Snapshotter does not start any nydusd
-	// and only interacts with containerd with mount slice to pass necessary configuration
-	// to container runtime
-	DaemonModeNone    DaemonMode = "none"
-	DaemonModeInvalid DaemonMode = ""
+	// Spawn a dedicated nydusd for each RAFS instance.
+	DaemonModeMultiple DaemonMode = DaemonMode(constant.DaemonModeMultiple)
+	// Spawn a dedicated nydusd for each RAFS instance.
+	DaemonModeDedicated DaemonMode = DaemonMode(constant.DaemonModeDedicated)
+	// Share a global nydusd to serve all RAFS instances.
+	DaemonModeShared DaemonMode = DaemonMode(constant.DaemonModeShared)
+	// Do not spawn nydusd for RAFS instances.
+	//
+	// For tarfs and rund, there's no need to create nydusd to serve RAFS instances,
+	// the snapshotter just returns mount slices with additional information for runC/runD
+	// to manage those snapshots.
+	DaemonModeNone    DaemonMode = DaemonMode(constant.DaemonModeNone)
+	DaemonModeInvalid DaemonMode = DaemonMode(constant.DaemonModeInvalid)
 )
 
 func parseDaemonMode(m string) (DaemonMode, error) {
 	switch m {
 	case string(DaemonModeMultiple):
-		return DaemonModeMultiple, nil
+		return DaemonModeDedicated, nil
+	case string(DaemonModeDedicated):
+		return DaemonModeDedicated, nil
 	case string(DaemonModeShared):
 		return DaemonModeShared, nil
 	case string(DaemonModeNone):
 		return DaemonModeNone, nil
 	default:
-		return DaemonModeInvalid, errdefs.ErrInvalidArgument
+		return DaemonModeInvalid, errors.Errorf("invalid daemon mode %q", m)
 	}
 }
 
@@ -83,29 +93,49 @@ var recoverPolicyParser map[string]DaemonRecoverPolicy
 func ParseRecoverPolicy(p string) (DaemonRecoverPolicy, error) {
 	policy, ok := recoverPolicyParser[p]
 	if !ok {
-		return RecoverPolicyInvalid, errdefs.ErrNotFound
+		return RecoverPolicyInvalid, errors.Errorf("invalid recover policy %q", p)
 	}
 
 	return policy, nil
 }
 
 const (
-	FsDriverFusedev string = "fusedev"
-	FsDriverFscache string = "fscache"
+	FsDriverBlockdev string = constant.FsDriverBlockdev
+	FsDriverFusedev  string = constant.FsDriverFusedev
+	FsDriverFscache  string = constant.FsDriverFscache
+	FsDriverNodev    string = constant.FsDriverNodev
+	FsDriverProxy    string = constant.FsDriverProxy
 )
 
 type Experimental struct {
-	EnableStargz bool `toml:"enable_stargz"`
+	EnableStargz         bool        `toml:"enable_stargz"`
+	EnableReferrerDetect bool        `toml:"enable_referrer_detect"`
+	TarfsConfig          TarfsConfig `toml:"tarfs"`
+	EnableBackendSource  bool        `toml:"enable_backend_source"`
+}
+
+type TarfsConfig struct {
+	EnableTarfs       bool   `toml:"enable_tarfs"`
+	MountTarfsOnHost  bool   `toml:"mount_tarfs_on_host"`
+	TarfsHint         bool   `toml:"tarfs_hint"`
+	MaxConcurrentProc int    `toml:"max_concurrent_proc"`
+	ExportMode        string `toml:"export_mode"`
+}
+
+type CgroupConfig struct {
+	Enable      bool   `toml:"enable"`
+	MemoryLimit string `toml:"memory_limit"`
 }
 
 // Configure how to start and recover nydusd daemons
 type DaemonConfig struct {
 	NydusdPath       string `toml:"nydusd_path"`
-	NydusImagePath   string `toml:"nydusimage_path"`
 	NydusdConfigPath string `toml:"nydusd_config"`
+	NydusImagePath   string `toml:"nydusimage_path"`
 	RecoverPolicy    string `toml:"recover_policy"`
 	FsDriver         string `toml:"fs_driver"`
 	ThreadsNumber    int    `toml:"threads_number"`
+	LogRotationSize  int    `toml:"log_rotation_size"`
 }
 
 type LoggingConfig struct {
@@ -128,8 +158,10 @@ type ImageConfig struct {
 // Configure containerd snapshots interfaces and how to process the snapshots
 // requests from containerd
 type SnapshotConfig struct {
-	EnableNydusOverlayFS bool `toml:"enable_nydus_overlayfs"`
-	SyncRemove           bool `toml:"sync_remove"`
+	EnableNydusOverlayFS bool   `toml:"enable_nydus_overlayfs"`
+	NydusOverlayFSPath   string `toml:"nydus_overlayfs_path"`
+	EnableKataVolume     bool   `toml:"enable_kata_volume"`
+	SyncRemove           bool   `toml:"sync_remove"`
 }
 
 // Configure cache manager that manages the cache files lifecycle
@@ -155,6 +187,7 @@ type AuthConfig struct {
 type RemoteConfig struct {
 	AuthConfig         AuthConfig    `toml:"auth"`
 	ConvertVpcRegistry bool          `toml:"convert_vpc_registry"`
+	SkipSSLVerify      bool          `toml:"skip_ssl_verify"`
 	MirrorsConfig      MirrorsConfig `toml:"mirrors_config"`
 }
 
@@ -195,6 +228,7 @@ type SnapshotterConfig struct {
 	ImageConfig            ImageConfig            `toml:"image"`
 	CacheManagerConfig     CacheManagerConfig     `toml:"cache_manager"`
 	LoggingConfig          LoggingConfig          `toml:"log"`
+	CgroupConfig           CgroupConfig           `toml:"cgroup"`
 	Experimental           Experimental           `toml:"experimental"`
 }
 
@@ -206,10 +240,14 @@ func LoadSnapshotterConfig(path string) (*SnapshotterConfig, error) {
 	}
 	tree, err := toml.LoadFile(path)
 	if err != nil {
-		return nil, errors.Wrapf(err, "load snapshotter configuration file %q", path)
+		return nil, errors.Wrapf(err, "load toml configuration from file %q", path)
 	}
+
 	if err = tree.Unmarshal(&config); err != nil {
-		return nil, errors.Wrapf(err, "unmarshal snapshotter configuration file %q", path)
+		return nil, errors.Wrap(err, "unmarshal snapshotter configuration")
+	}
+	if config.Version != 1 {
+		return nil, errors.Errorf("unsupported configuration version %d", config.Version)
 	}
 	return &config, nil
 }
@@ -232,7 +270,7 @@ func ValidateConfig(c *SnapshotterConfig) error {
 		if c.ImageConfig.PublicKeyFile == "" {
 			return errors.New("public key file for signature validation is not provided")
 		} else if _, err := os.Stat(c.ImageConfig.PublicKeyFile); err != nil {
-			return errors.Wrapf(err, "find publicKey file %q", c.ImageConfig.PublicKeyFile)
+			return errors.Wrapf(err, "check publicKey file %q", c.ImageConfig.PublicKeyFile)
 		}
 	}
 
@@ -240,6 +278,18 @@ func ValidateConfig(c *SnapshotterConfig) error {
 		return errors.New("empty root directory")
 	}
 
+	if c.DaemonConfig.FsDriver != FsDriverFscache && c.DaemonConfig.FsDriver != FsDriverFusedev &&
+		c.DaemonConfig.FsDriver != FsDriverBlockdev && c.DaemonConfig.FsDriver != FsDriverNodev &&
+		c.DaemonConfig.FsDriver != FsDriverProxy {
+		return errors.Errorf("invalid filesystem driver %q", c.DaemonConfig.FsDriver)
+	}
+	if _, err := ParseRecoverPolicy(c.DaemonConfig.RecoverPolicy); err != nil {
+		return err
+	}
+	if c.DaemonConfig.ThreadsNumber > 1024 {
+		return errors.Errorf("nydusd worker thread number %d is too big, max 1024", c.DaemonConfig.ThreadsNumber)
+	}
+
 	if c.RemoteConfig.AuthConfig.EnableCRIKeychain && c.RemoteConfig.AuthConfig.EnableKubeconfigKeychain {
 		return errors.Wrapf(errdefs.ErrInvalidArgument,
 			"\"enable_cri_keychain\" and \"enable_kubeconfig_keychain\" can't be set at the same time")
@@ -251,7 +301,7 @@ func ValidateConfig(c *SnapshotterConfig) error {
 			return err
 		}
 		if !dirExisted {
-			return errors.Errorf("mirrors config directory %s is not existed", c.RemoteConfig.MirrorsConfig.Dir)
+			return errors.Errorf("mirrors config directory %s does not exist", c.RemoteConfig.MirrorsConfig.Dir)
 		}
 	}
 
@@ -308,10 +358,28 @@ func ParseParameters(args *flags.Args, cfg *SnapshotterConfig) error {
 	// empty
 
 	// --- snapshot configuration
-	// empty
+	if args.NydusOverlayFSPath != "" {
+		cfg.SnapshotsConfig.NydusOverlayFSPath = args.NydusOverlayFSPath
+	}
 
 	// --- metrics configuration
 	// empty
 
 	return nil
 }
+
+func ParseCgroupConfig(config CgroupConfig) (cgroup.Config, error) {
+	totalMemory, err := sysinfo.GetTotalMemoryBytes()
+	if err != nil {
+		return cgroup.Config{}, errors.Wrap(err, "Failed  to get total memory bytes")
+	}
+
+	memoryLimitInBytes, err := parser.MemoryConfigToBytes(config.MemoryLimit, totalMemory)
+	if err != nil {
+		return cgroup.Config{}, err
+	}
+
+	return cgroup.Config{
+		MemoryLimitInBytes: memoryLimitInBytes,
+	}, nil
+}

config/config_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/containerd/nydus-snapshotter/internal/constant"
 	"github.com/containerd/nydus-snapshotter/internal/flags"
 	"github.com/stretchr/testify/assert"
 )
@@ -22,15 +23,18 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) {
 	A.NoError(err)
 
 	exampleConfig := SnapshotterConfig{
-		Version:        1,
-		Root:           "/var/lib/containerd-nydus",
-		Address:        "/run/containerd-nydus/containerd-nydus-grpc.sock",
-		DaemonMode:     "multiple",
-		Experimental:   Experimental{EnableStargz: false},
+		Version:    1,
+		Root:       "/var/lib/containerd/io.containerd.snapshotter.v1.nydus",
+		Address:    "/run/containerd-nydus/containerd-nydus-grpc.sock",
+		DaemonMode: "dedicated",
+		Experimental: Experimental{
+			EnableStargz:         false,
+			EnableReferrerDetect: false,
+		},
 		CleanupOnClose: false,
 		SystemControllerConfig: SystemControllerConfig{
 			Enable:  true,
-			Address: "/var/run/containerd-nydus/system.sock",
+			Address: "/run/containerd-nydus/system.sock",
 			DebugConfig: DebugConfig{
 				ProfileDuration: 5,
 				PprofAddress:    "",
@@ -43,9 +47,11 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) {
 			RecoverPolicy:    "restart",
 			NydusdConfigPath: "/etc/nydus/nydusd-config.fusedev.json",
 			ThreadsNumber:    4,
+			LogRotationSize:  100,
 		},
 		SnapshotsConfig: SnapshotConfig{
 			EnableNydusOverlayFS: false,
+			NydusOverlayFSPath:   "nydus-overlayfs",
 			SyncRemove:           false,
 		},
 		RemoteConfig: RemoteConfig{
@@ -55,7 +61,7 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) {
 				KubeconfigPath:           "",
 			},
 			MirrorsConfig: MirrorsConfig{
-				Dir: "/etc/nydus/certs.d",
+				Dir: "",
 			},
 		},
 		ImageConfig: ImageConfig{
@@ -73,19 +79,24 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) {
 			RotateLogLocalTime:  true,
 			RotateLogMaxAge:     7,
 			RotateLogMaxBackups: 5,
-			RotateLogMaxSize:    1,
+			RotateLogMaxSize:    100,
 			LogToStdout:         false,
 		},
 		MetricsConfig: MetricsConfig{
 			Address: ":9110",
 		},
+		CgroupConfig: CgroupConfig{
+			Enable:      true,
+			MemoryLimit: "",
+		},
 	}
 
 	A.EqualValues(cfg, &exampleConfig)
 
-	var args = flags.Args{}
+	args := flags.Args{}
 	args.RootDir = "/var/lib/containerd/nydus"
 	exampleConfig.Root = "/var/lib/containerd/nydus"
+
 	err = ParseParameters(&args, cfg)
 	A.NoError(err)
 	A.EqualValues(cfg, &exampleConfig)
@@ -174,21 +185,21 @@ func TestMergeConfig(t *testing.T) {
 
 	err = MergeConfig(&snapshotterConfig1, &defaultSnapshotterConfig)
 	A.NoError(err)
-	A.Equal(snapshotterConfig1.Root, defaultRootDir)
+	A.Equal(snapshotterConfig1.Root, constant.DefaultRootDir)
 	A.Equal(snapshotterConfig1.LoggingConfig.LogDir, "")
 	A.Equal(snapshotterConfig1.CacheManagerConfig.CacheDir, "")
 
-	A.Equal(snapshotterConfig1.DaemonMode, DefaultDaemonMode)
-	A.Equal(snapshotterConfig1.SystemControllerConfig.Address, defaultSystemControllerAddress)
-	A.Equal(snapshotterConfig1.LoggingConfig.LogLevel, DefaultLogLevel)
-	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxSize, defaultRotateLogMaxSize)
-	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxBackups, defaultRotateLogMaxBackups)
-	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxAge, defaultRotateLogMaxAge)
-	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogCompress, defaultRotateLogCompress)
+	A.Equal(snapshotterConfig1.DaemonMode, constant.DefaultDaemonMode)
+	A.Equal(snapshotterConfig1.SystemControllerConfig.Address, constant.DefaultSystemControllerAddress)
+	A.Equal(snapshotterConfig1.LoggingConfig.LogLevel, constant.DefaultLogLevel)
+	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxSize, constant.DefaultRotateLogMaxSize)
+	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxBackups, constant.DefaultRotateLogMaxBackups)
+	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogMaxAge, constant.DefaultRotateLogMaxAge)
+	A.Equal(snapshotterConfig1.LoggingConfig.RotateLogCompress, constant.DefaultRotateLogCompress)
 
-	A.Equal(snapshotterConfig1.DaemonConfig.NydusdConfigPath, defaultNydusDaemonConfigPath)
+	A.Equal(snapshotterConfig1.DaemonConfig.NydusdConfigPath, constant.DefaultNydusDaemonConfigPath)
 	A.Equal(snapshotterConfig1.DaemonConfig.RecoverPolicy, RecoverPolicyRestart.String())
-	A.Equal(snapshotterConfig1.CacheManagerConfig.GCPeriod, defaultGCPeriod)
+	A.Equal(snapshotterConfig1.CacheManagerConfig.GCPeriod, constant.DefaultGCPeriod)
 
 	var snapshotterConfig2 SnapshotterConfig
 	snapshotterConfig2.Root = "/snapshotter/root"
@@ -211,6 +222,7 @@ func TestProcessConfigurations(t *testing.T) {
 	A.NoError(err)
 	err = ValidateConfig(&snapshotterConfig1)
 	A.NoError(err)
+
 	err = ProcessConfigurations(&snapshotterConfig1)
 	A.NoError(err)
 
@@ -224,9 +236,21 @@ func TestProcessConfigurations(t *testing.T) {
 	A.NoError(err)
 	err = ValidateConfig(&snapshotterConfig2)
 	A.NoError(err)
+
 	err = ProcessConfigurations(&snapshotterConfig2)
 	A.NoError(err)
 
 	A.Equal(snapshotterConfig2.LoggingConfig.LogDir, filepath.Join(snapshotterConfig2.Root, "logs"))
 	A.Equal(snapshotterConfig2.CacheManagerConfig.CacheDir, filepath.Join(snapshotterConfig2.Root, "cache"))
+
+	var snapshotterConfig3 SnapshotterConfig
+	snapshotterConfig3.Root = "./snapshotter/root"
+
+	err = MergeConfig(&snapshotterConfig3, &defaultSnapshotterConfig)
+	A.NoError(err)
+	err = ValidateConfig(&snapshotterConfig3)
+	A.NoError(err)
+
+	err = ProcessConfigurations(&snapshotterConfig3)
+	A.NoError(err)
 }

config/daemonconfig/daemonconfig.go

@@ -10,6 +10,8 @@ package daemonconfig
 import (
 	"encoding/json"
 	"os"
+	"reflect"
+	"strings"
 
 	"github.com/pkg/errors"
 
@@ -31,7 +33,7 @@ type DaemonConfig interface {
 	Supplement(host, repo, snapshotID string, params map[string]string)
 	// Provide auth
 	FillAuth(kc *auth.PassKeyChain)
-	StorageBackendType() string
+	StorageBackend() (StorageBackendType, *BackendConfig)
 	UpdateMirrors(mirrorsConfigDir, registryHost string) error
 	DumpString() (string, error)
 	DumpFile(path string) error
@@ -53,18 +55,18 @@ func NewDaemonConfig(fsDriver, path string) (DaemonConfig, error) {
 		}
 		return cfg, nil
 	default:
-		return nil, errors.Errorf("unsupported, fs driver %s", fsDriver)
+		return nil, errors.Errorf("unsupported, fs driver %q", fsDriver)
 	}
 }
 
 type MirrorConfig struct {
 	Host                string            `json:"host,omitempty"`
 	Headers             map[string]string `json:"headers,omitempty"`
-	AuthThrough         bool              `json:"auth_through,omitempty"`
 	HealthCheckInterval int               `json:"health_check_interval,omitempty"`
 	FailureLimit        uint8             `json:"failure_limit,omitempty"`
 	PingURL             string            `json:"ping_url,omitempty"`
 }
+
 type BackendConfig struct {
 	// Localfs backend configs
 	BlobFile     string `json:"blob_file,omitempty"`
@@ -75,16 +77,16 @@ type BackendConfig struct {
 	// Registry backend configs
 	Host               string         `json:"host,omitempty"`
 	Repo               string         `json:"repo,omitempty"`
-	Auth               string         `json:"auth,omitempty"`
-	RegistryToken      string         `json:"registry_token,omitempty"`
+	Auth               string         `json:"auth,omitempty" secret:"true"`
+	RegistryToken      string         `json:"registry_token,omitempty" secret:"true"`
 	BlobURLScheme      string         `json:"blob_url_scheme,omitempty"`
 	BlobRedirectedHost string         `json:"blob_redirected_host,omitempty"`
 	Mirrors            []MirrorConfig `json:"mirrors,omitempty"`
 
 	// OSS backend configs
 	EndPoint        string `json:"endpoint,omitempty"`
-	AccessKeyID     string `json:"access_key_id,omitempty"`
-	AccessKeySecret string `json:"access_key_secret,omitempty"`
+	AccessKeyID     string `json:"access_key_id,omitempty" secret:"true"`
+	AccessKeySecret string `json:"access_key_secret,omitempty" secret:"true"`
 	BucketName      string `json:"bucket_name,omitempty"`
 	ObjectPrefix    string `json:"object_prefix,omitempty"`
 
@@ -124,6 +126,9 @@ type DeviceConfig struct {
 // We don't have to persist configuration file for fscache since its configuration
 // is passed through HTTP API.
 func DumpConfigFile(c interface{}, path string) error {
+	if config.IsBackendSourceEnabled() {
+		c = serializeWithSecretFilter(c)
+	}
 	b, err := json.Marshal(c)
 	if err != nil {
 		return errors.Wrapf(err, "marshal config")
@@ -146,9 +151,9 @@ func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string,
 		return errors.Wrapf(err, "parse image %s", imageID)
 	}
 
-	backend := c.StorageBackendType()
+	backendType, _ := c.StorageBackend()
 
-	switch backend {
+	switch backendType {
 	case backendTypeRegistry:
 		registryHost := image.Host
 		if vpcRegistry {
@@ -174,8 +179,58 @@ func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string,
 	case backendTypeLocalfs:
 	case backendTypeOss:
 	default:
-		return errors.Errorf("unknown backend type %s", backend)
+		return errors.Errorf("unknown backend type %s", backendType)
 	}
 
 	return nil
 }
+
+func serializeWithSecretFilter(obj interface{}) map[string]interface{} {
+	result := make(map[string]interface{})
+	value := reflect.ValueOf(obj)
+	typeOfObj := reflect.TypeOf(obj)
+
+	if value.Kind() == reflect.Ptr {
+		value = value.Elem()
+		typeOfObj = typeOfObj.Elem()
+	}
+
+	for i := 0; i < value.NumField(); i++ {
+		field := value.Field(i)
+		fieldType := typeOfObj.Field(i)
+		secretTag := fieldType.Tag.Get("secret")
+		jsonTags := strings.Split(fieldType.Tag.Get("json"), ",")
+		omitemptyTag := false
+
+		for _, tag := range jsonTags {
+			if tag == "omitempty" {
+				omitemptyTag = true
+				break
+			}
+		}
+
+		if secretTag == "true" {
+			continue
+		}
+
+		if field.Kind() == reflect.Ptr && field.IsNil() {
+			continue
+		}
+
+		if omitemptyTag && reflect.DeepEqual(reflect.Zero(field.Type()).Interface(), field.Interface()) {
+			continue
+		}
+
+		//nolint:exhaustive
+		switch fieldType.Type.Kind() {
+		case reflect.Struct:
+			result[jsonTags[0]] = serializeWithSecretFilter(field.Interface())
+		case reflect.Ptr:
+			result[jsonTags[0]] = serializeWithSecretFilter(field.Elem().Interface())
+		default:
+			result[jsonTags[0]] = field.Interface()
+		}
+	}
+
+	return result
+}

config/daemonconfig/daemonconfig_test.go

@@ -46,6 +46,7 @@ func TestLoadConfig(t *testing.T) {
   "digest_validate": true,
   "iostats_files": true,
   "enable_xattr": true,
+  "amplify_io": 1048576,
   "fs_prefetch": {
     "enable": true,
     "threads_count": 10,
@@ -62,3 +63,93 @@ func TestLoadConfig(t *testing.T) {
 	require.Equal(t, cfg.Device.Backend.Config.SkipVerify, true)
 	require.Equal(t, cfg.Device.Backend.Config.Proxy.CheckInterval, 5)
 }
+
+func TestAmplifyIo(t *testing.T) {
+	// Test non-zero value
+	input1 := []byte(`{"amplify_io": 1048576}`)
+	var cfg1 FuseDaemonConfig
+	err1 := json.Unmarshal(input1, &cfg1)
+	require.Nil(t, err1)
+	require.Equal(t, *cfg1.AmplifyIo, 1048576)
+	output1, _ := json.Marshal(cfg1)
+	require.Contains(t, string(output1), `"amplify_io":1048576`)
+
+	// Test zero value
+	input2 := []byte(`{"amplify_io": 0}`)
+	var cfg2 FuseDaemonConfig
+	err2 := json.Unmarshal(input2, &cfg2)
+	require.Nil(t, err2)
+	require.Equal(t, *cfg2.AmplifyIo, 0)
+	output2, _ := json.Marshal(cfg2)
+	require.Contains(t, string(output2), `"amplify_io":0`)
+
+	// Test nil value
+	input3 := []byte(`{}`)
+	var cfg3 FuseDaemonConfig
+	err3 := json.Unmarshal(input3, &cfg3)
+	require.Nil(t, err3)
+	require.Nil(t, cfg3.AmplifyIo)
+	output3, _ := json.Marshal(cfg3)
+	require.NotContains(t, string(output3), `amplify_io`)
+}
+
+func TestSerializeWithSecretFilter(t *testing.T) {
+	buf := []byte(`{
+  "device": {
+    "backend": {
+      "type": "registry",
+      "config": {
+        "skip_verify": true,
+        "host": "acr-nydus-registry-vpc.cn-hangzhou.cr.aliyuncs.com",
+        "repo": "test/myserver",
+        "auth": "token_token",
+        "blob_url_scheme": "http",
+        "proxy": {
+          "url": "http://p2p-proxy:65001",
+          "fallback": true,
+          "ping_url": "http://p2p-proxy:40901/server/ping",
+          "check_interval": 5
+        },
+        "timeout": 5,
+        "connect_timeout": 5,
+        "retry_limit": 0
+      }
+    },
+    "cache": {
+      "type": "blobcache",
+      "config": {
+        "work_dir": "/cache"
+      }
+    }
+  },
+  "mode": "direct",
+  "digest_validate": true,
+  "iostats_files": true,
+  "enable_xattr": true,
+  "fs_prefetch": {
+    "enable": true,
+    "threads_count": 10,
+    "merging_size": 131072
+  }
+}`)
+	var cfg FuseDaemonConfig
+	_ = json.Unmarshal(buf, &cfg)
+	filter := serializeWithSecretFilter(&cfg)
+	jsonData, err := json.Marshal(filter)
+	require.Nil(t, err)
+	var newCfg FuseDaemonConfig
+	err = json.Unmarshal(jsonData, &newCfg)
+	require.Nil(t, err)
+	require.Equal(t, newCfg.FSPrefetch, cfg.FSPrefetch)
+	require.Equal(t, newCfg.Device.Cache.CacheType, cfg.Device.Cache.CacheType)
+	require.Equal(t, newCfg.Device.Cache.Config, cfg.Device.Cache.Config)
+	require.Equal(t, newCfg.Mode, cfg.Mode)
+	require.Equal(t, newCfg.DigestValidate, cfg.DigestValidate)
+	require.Equal(t, newCfg.IOStatsFiles, cfg.IOStatsFiles)
+	require.Equal(t, newCfg.Device.Backend.Config.Host, cfg.Device.Backend.Config.Host)
+	require.Equal(t, newCfg.Device.Backend.Config.Repo, cfg.Device.Backend.Config.Repo)
+	require.Equal(t, newCfg.Device.Backend.Config.Proxy, cfg.Device.Backend.Config.Proxy)
+	require.Equal(t, newCfg.Device.Backend.Config.BlobURLScheme, cfg.Device.Backend.Config.BlobURLScheme)
+	require.Equal(t, newCfg.Device.Backend.Config.Auth, "")
+	require.NotEqual(t, newCfg.Device.Backend.Config.Auth, cfg.Device.Backend.Config.Auth)
+}

config/daemonconfig/fscache.go

@@ -11,7 +11,7 @@ import (
 	"os"
 	"path"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/containerd/nydus-snapshotter/pkg/auth"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/erofs"
 
@@ -79,8 +79,8 @@ func (c *FscacheDaemonConfig) UpdateMirrors(mirrorsConfigDir, registryHost strin
 	return nil
 }
 
-func (c *FscacheDaemonConfig) StorageBackendType() string {
-	return c.Config.BackendType
+func (c *FscacheDaemonConfig) StorageBackend() (string, *BackendConfig) {
+	return c.Config.BackendType, &c.Config.BackendConfig
 }
 
 // Each fscache/erofs has a configuration with different fscache ID built from snapshot ID.

config/daemonconfig/fuse.go

@@ -27,6 +27,7 @@ type FuseDaemonConfig struct {
 	EnableXattr     bool          `json:"enable_xattr,omitempty"`
 	AccessPattern   bool          `json:"access_pattern,omitempty"`
 	LatestReadFiles bool          `json:"latest_read_files,omitempty"`
+	AmplifyIo       *int          `json:"amplify_io,omitempty"`
 	FSPrefetch      `json:"fs_prefetch,omitempty"`
 	// (experimental) The nydus daemon could cache more data to increase hit ratio when enabled the warmup feature.
 	Warmup uint64 `json:"warmup,omitempty"`
@@ -59,7 +60,7 @@ func LoadFuseConfig(p string) (*FuseDaemonConfig, error) {
 	return &cfg, nil
 }
 
-func (c *FuseDaemonConfig) Supplement(host, repo, snapshotID string, params map[string]string) {
+func (c *FuseDaemonConfig) Supplement(host, repo, _ string, params map[string]string) {
 	c.Device.Backend.Config.Host = host
 	c.Device.Backend.Config.Repo = repo
 	c.Device.Cache.Config.WorkDir = params[CacheDir]
@@ -86,8 +87,8 @@ func (c *FuseDaemonConfig) UpdateMirrors(mirrorsConfigDir, registryHost string)
 	return nil
 }
 
-func (c *FuseDaemonConfig) StorageBackendType() string {
-	return c.Device.Backend.BackendType
+func (c *FuseDaemonConfig) StorageBackend() (string, *BackendConfig) {
+	return c.Device.Backend.BackendType, &c.Device.Backend.Config
 }
 
 func (c *FuseDaemonConfig) DumpString() (string, error) {

config/daemonconfig/mirrors.go

@@ -15,7 +15,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
 )
@@ -30,7 +30,6 @@ type HostFileConfig struct {
 	OverridePath bool                   `toml:"override_path"`
 
 	// The following configuration items are specific to nydus.
-	AuthThrough         bool   `toml:"auth_through,omitempty"`
 	HealthCheckInterval int    `toml:"health_check_interval,omitempty"`
 	FailureLimit        uint8  `toml:"failure_limit,omitempty"`
 	PingURL             string `toml:"ping_url,omitempty"`
@@ -41,7 +40,6 @@ type hostConfig struct {
 	Host   string
 	Header http.Header
 
-	AuthThrough         bool
 	HealthCheckInterval int
 	FailureLimit        uint8
 	PingURL             string
@@ -69,7 +67,6 @@ func parseMirrorsConfig(hosts []hostConfig) []MirrorConfig {
 
 	for i, host := range hosts {
 		parsedMirrors[i].Host = fmt.Sprintf("%s://%s", host.Scheme, host.Host)
-		parsedMirrors[i].AuthThrough = host.AuthThrough
 		parsedMirrors[i].HealthCheckInterval = host.HealthCheckInterval
 		parsedMirrors[i].FailureLimit = host.FailureLimit
 		parsedMirrors[i].PingURL = host.PingURL
@@ -141,23 +138,22 @@ func getSortedHosts(root *toml.Tree) ([]string, error) {
 	return list, nil
 }
 
+// parseHostConfig returns the parsed host configuration, make sure the server is not null.
 func parseHostConfig(server string, config HostFileConfig) (hostConfig, error) {
 	var (
 		result = hostConfig{}
 		err    error
 	)
 
-	if server != "" {
-		if !strings.HasPrefix(server, "http") {
-			server = "https://" + server
-		}
-		u, err := url.Parse(server)
-		if err != nil {
-			return hostConfig{}, fmt.Errorf("unable to parse server %v: %w", server, err)
-		}
-		result.Scheme = u.Scheme
-		result.Host = u.Host
+	if !strings.HasPrefix(server, "http") {
+		server = "https://" + server
 	}
+	u, err := url.Parse(server)
+	if err != nil {
+		return hostConfig{}, fmt.Errorf("unable to parse server %v: %w", server, err)
+	}
+	result.Scheme = u.Scheme
+	result.Host = u.Host
 
 	if config.Header != nil {
 		header := http.Header{}
@@ -177,7 +173,6 @@ func parseHostConfig(server string, config HostFileConfig) (hostConfig, error) {
 		result.Header = header
 	}
 
-	result.AuthThrough = config.AuthThrough
 	result.HealthCheckInterval = config.HealthCheckInterval
 	result.FailureLimit = config.FailureLimit
 	result.PingURL = config.PingURL
@@ -191,10 +186,6 @@ func parseHostsFile(b []byte) ([]hostConfig, error) {
 		return nil, fmt.Errorf("failed to parse TOML: %w", err)
 	}
 	c := struct {
-		HostFileConfig
-		// Server specifies the default server. When `host` is
-		// also specified, those hosts are tried first.
-		Server string `toml:"server"`
 		// HostConfigs store the per-host configuration
 		HostConfigs map[string]HostFileConfig `toml:"host"`
 	}{}
@@ -214,22 +205,16 @@ func parseHostsFile(b []byte) ([]hostConfig, error) {
 
 	// Parse hosts array
 	for _, host := range orderedHosts {
-		config := c.HostConfigs[host]
-
-		parsed, err := parseHostConfig(host, config)
-		if err != nil {
-			return nil, err
+		if host != "" {
+			config := c.HostConfigs[host]
+			parsed, err := parseHostConfig(host, config)
+			if err != nil {
+				return nil, err
+			}
+			hosts = append(hosts, parsed)
 		}
-		hosts = append(hosts, parsed)
 	}
 
-	// Parse root host config and append it as the last element
-	parsed, err := parseHostConfig(c.Server, c.HostFileConfig)
-	if err != nil {
-		return nil, err
-	}
-	hosts = append(hosts, parsed)
-
 	return hosts, nil
 }
 

config/daemonconfig/mirrors_test.go

@@ -43,7 +43,6 @@ func TestLoadMirrorConfig(t *testing.T) {
 	buf1 := []byte(`server = "https://default-docker.hub.com"
 	[host]
 	  [host."http://default-p2p-mirror1:65001"]
-		auth_through = true
 		[host."http://default-p2p-mirror1:65001".header]
 		  X-Dragonfly-Registry = ["https://default-docker.hub.com"]
 	`)
@@ -51,12 +50,9 @@ func TestLoadMirrorConfig(t *testing.T) {
 	assert.NoError(t, err)
 	mirrors, err = LoadMirrorsConfig(mirrorsConfigDir, registryHost)
 	require.NoError(t, err)
-	require.Equal(t, len(mirrors), 2)
+	require.Equal(t, len(mirrors), 1)
 	require.Equal(t, mirrors[0].Host, "http://default-p2p-mirror1:65001")
-	require.Equal(t, mirrors[0].AuthThrough, true)
 	require.Equal(t, mirrors[0].Headers["X-Dragonfly-Registry"], "https://default-docker.hub.com")
-	require.Equal(t, mirrors[1].Host, "https://default-docker.hub.com")
-	require.Equal(t, mirrors[1].AuthThrough, false)
 
 	err = os.MkdirAll(registryHostConfigDir, os.ModePerm)
 	assert.NoError(t, err)
@@ -64,7 +60,6 @@ func TestLoadMirrorConfig(t *testing.T) {
 	buf2 := []byte(`server = "https://docker.hub.com"
 	[host]
 	  [host."http://p2p-mirror1:65001"]
-		auth_through = true
 		[host."http://p2p-mirror1:65001".header]
 		  X-Dragonfly-Registry = ["https://docker.hub.com"]
 	`)
@@ -72,10 +67,20 @@ func TestLoadMirrorConfig(t *testing.T) {
 	assert.NoError(t, err)
 	mirrors, err = LoadMirrorsConfig(mirrorsConfigDir, registryHost)
 	require.NoError(t, err)
-	require.Equal(t, len(mirrors), 2)
+	require.Equal(t, len(mirrors), 1)
 	require.Equal(t, mirrors[0].Host, "http://p2p-mirror1:65001")
-	require.Equal(t, mirrors[0].AuthThrough, true)
 	require.Equal(t, mirrors[0].Headers["X-Dragonfly-Registry"], "https://docker.hub.com")
-	require.Equal(t, mirrors[1].Host, "https://docker.hub.com")
-	require.Equal(t, mirrors[1].AuthThrough, false)
+
+	buf3 := []byte(`
+		[host."http://p2p-mirror2:65001"]
+		[host."http://p2p-mirror2:65001".header]
+			X-Dragonfly-Registry = ["https://docker.hub.com"]
+	`)
+	err = os.WriteFile(filepath.Join(registryHostConfigDir, "hosts.toml"), buf3, 0600)
+	assert.NoError(t, err)
+	mirrors, err = LoadMirrorsConfig(mirrorsConfigDir, registryHost)
+	require.NoError(t, err)
+	require.Equal(t, len(mirrors), 1)
+	require.Equal(t, mirrors[0].Host, "http://p2p-mirror2:65001")
+	require.Equal(t, mirrors[0].Headers["X-Dragonfly-Registry"], "https://docker.hub.com")
 }

config/default.go

@@ -8,80 +8,60 @@ package config
 
 import (
 	"os/exec"
-)
 
-const (
-	DefaultDaemonMode string = string(DaemonModeMultiple)
-
-	DefaultLogLevel string = "info"
-	defaultGCPeriod string = "24h"
-
-	defaultNydusDaemonConfigPath string = "/etc/nydus/nydusd-config.json"
-	nydusdBinaryName             string = "nydusd"
-	nydusImageBinaryName         string = "nydus-image"
-
-	defaultRootDir                 = "/var/lib/containerd-nydus"
-	defaultSystemControllerAddress = "/var/run/containerd-nydus/system.sock"
-
-	// Log rotation
-	defaultRotateLogMaxSize    = 200 // 200 megabytes
-	defaultRotateLogMaxBackups = 10
-	defaultRotateLogMaxAge     = 0 // days
-	defaultRotateLogLocalTime  = true
-	defaultRotateLogCompress   = true
+	"github.com/containerd/nydus-snapshotter/internal/constant"
 )
 
 func (c *SnapshotterConfig) FillUpWithDefaults() error {
-	c.Root = defaultRootDir
+	c.Version = 1
+	c.Root = constant.DefaultRootDir
+	c.Address = constant.DefaultAddress
 
 	// essential configuration
 	if c.DaemonMode == "" {
-		c.DaemonMode = DefaultDaemonMode
+		c.DaemonMode = constant.DefaultDaemonMode
 	}
 
 	// system controller configuration
-	c.SystemControllerConfig.Address = defaultSystemControllerAddress
+	c.SystemControllerConfig.Address = constant.DefaultSystemControllerAddress
 
 	// logging configuration
 	logConfig := &c.LoggingConfig
 	if logConfig.LogLevel == "" {
-		logConfig.LogLevel = DefaultLogLevel
+		logConfig.LogLevel = constant.DefaultLogLevel
 	}
-	logConfig.RotateLogMaxSize = defaultRotateLogMaxSize
-	logConfig.RotateLogMaxBackups = defaultRotateLogMaxBackups
-	logConfig.RotateLogMaxAge = defaultRotateLogMaxAge
-	logConfig.RotateLogLocalTime = defaultRotateLogLocalTime
-	logConfig.RotateLogCompress = defaultRotateLogCompress
+	logConfig.RotateLogMaxSize = constant.DefaultRotateLogMaxSize
+	logConfig.RotateLogMaxBackups = constant.DefaultRotateLogMaxBackups
+	logConfig.RotateLogMaxAge = constant.DefaultRotateLogMaxAge
+	logConfig.RotateLogLocalTime = constant.DefaultRotateLogLocalTime
+	logConfig.RotateLogCompress = constant.DefaultRotateLogCompress
 
 	// daemon configuration
 	daemonConfig := &c.DaemonConfig
 	if daemonConfig.NydusdConfigPath == "" {
-		daemonConfig.NydusdConfigPath = defaultNydusDaemonConfigPath
+		daemonConfig.NydusdConfigPath = constant.DefaultNydusDaemonConfigPath
 	}
 	daemonConfig.RecoverPolicy = RecoverPolicyRestart.String()
+	daemonConfig.FsDriver = constant.DefaultFsDriver
+	daemonConfig.LogRotationSize = constant.DefaultDaemonRotateLogMaxSize
 
 	// cache configuration
 	cacheConfig := &c.CacheManagerConfig
 	if cacheConfig.GCPeriod == "" {
-		cacheConfig.GCPeriod = defaultGCPeriod
+		cacheConfig.GCPeriod = constant.DefaultGCPeriod
 	}
 
 	return c.SetupNydusBinaryPaths()
 }
 
 func (c *SnapshotterConfig) SetupNydusBinaryPaths() error {
-	// when using DaemonMode = none, nydusd and nydus-image binaries are not required
-	if c.DaemonMode == string(DaemonModeNone) {
-		return nil
-	}
-
 	// resolve nydusd path
-	if path, err := exec.LookPath(nydusdBinaryName); err == nil {
+	if path, err := exec.LookPath(constant.NydusdBinaryName); err == nil {
 		c.DaemonConfig.NydusdPath = path
 	}
 
 	// resolve nydus-image path
-	if path, err := exec.LookPath(nydusImageBinaryName); err == nil {
+	if path, err := exec.LookPath(constant.NydusImageBinaryName); err == nil {
 		c.DaemonConfig.NydusImagePath = path
 	}
 

config/global.go

@@ -10,32 +10,40 @@
 package config
 
 import (
+	"os"
 	"path/filepath"
 	"time"
 
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/nydus-snapshotter/internal/logging"
+	"github.com/containerd/log"
 	"github.com/pkg/errors"
+
+	"github.com/containerd/nydus-snapshotter/internal/logging"
+	"github.com/containerd/nydus-snapshotter/pkg/utils/mount"
 )
 
 var (
 	globalConfig GlobalConfig
 )
 
-// Retain the configurations that must be parsed and converted and the
-// configurations that are not easy to access from some modules.
-// Or avoid calculating repeatedly
+// Global cached configuration information to help:
+// - access configuration information without passing a configuration object
+// - avoid frequent generation of information from configuration information
 type GlobalConfig struct {
 	origin           *SnapshotterConfig
 	SnapshotsDir     string
 	DaemonMode       DaemonMode
 	SocketRoot       string
 	ConfigRoot       string
+	RootMountpoint   string
 	DaemonThreadsNum int
 	CacheGCPeriod    time.Duration
 	MirrorsConfig    MirrorsConfig
 }
 
+func IsFusedevSharedModeEnabled() bool {
+	return globalConfig.DaemonMode == DaemonModeShared
+}
+
 func GetDaemonMode() DaemonMode {
 	return globalConfig.DaemonMode
 }
@@ -44,6 +52,10 @@ func GetSnapshotsRootDir() string {
 	return globalConfig.SnapshotsDir
 }
 
+func GetRootMountpoint() string {
+	return globalConfig.RootMountpoint
+}
+
 func GetSocketRoot() string {
 	return globalConfig.SocketRoot
 }
@@ -72,6 +84,10 @@ func GetLogLevel() string {
 	return globalConfig.origin.LoggingConfig.LogLevel
 }
 
+func GetDaemonLogRotationSize() int {
+	return globalConfig.origin.DaemonConfig.LogRotationSize
+}
+
 func GetDaemonThreadsNumber() int {
 	return globalConfig.origin.DaemonConfig.ThreadsNumber
 }
@@ -80,6 +96,10 @@ func GetLogToStdout() bool {
 	return globalConfig.origin.LoggingConfig.LogToStdout
 }
 
+func IsBackendSourceEnabled() bool {
+	return globalConfig.origin.Experimental.EnableBackendSource && globalConfig.origin.SystemControllerConfig.Enable
+}
+
 func IsSystemControllerEnabled() bool {
 	return globalConfig.origin.SystemControllerConfig.Enable
 }
@@ -96,6 +116,57 @@ func GetDaemonProfileCPUDuration() int64 {
 	return globalConfig.origin.SystemControllerConfig.DebugConfig.ProfileDuration
 }
 
+func GetSkipSSLVerify() bool {
+	return globalConfig.origin.RemoteConfig.SkipSSLVerify
+}
+
+const (
+	TarfsLayerVerityOnly      string = "layer_verity_only"
+	TarfsImageVerityOnly      string = "image_verity_only"
+	TarfsLayerBlockDevice     string = "layer_block"
+	TarfsImageBlockDevice     string = "image_block"
+	TarfsLayerBlockWithVerity string = "layer_block_with_verity"
+	TarfsImageBlockWithVerity string = "image_block_with_verity"
+)
+
+func GetTarfsMountOnHost() bool {
+	return globalConfig.origin.Experimental.TarfsConfig.MountTarfsOnHost
+}
+
+func GetTarfsExportEnabled() bool {
+	switch globalConfig.origin.Experimental.TarfsConfig.ExportMode {
+	case TarfsLayerVerityOnly, TarfsLayerBlockDevice, TarfsLayerBlockWithVerity:
+		return true
+	case TarfsImageVerityOnly, TarfsImageBlockDevice, TarfsImageBlockWithVerity:
+		return true
+	default:
+		return false
+	}
+}
+
+// Returns (wholeImage, generateBlockImage, withVerityInfo)
+// wholeImage: generate tarfs for the whole image instead of of a specific layer.
+// generateBlockImage: generate a block image file.
+// withVerityInfo: generate disk verity information.
+func GetTarfsExportFlags() (bool, bool, bool) {
+	switch globalConfig.origin.Experimental.TarfsConfig.ExportMode {
+	case "layer_verity_only":
+		return false, false, true
+	case "image_verity_only":
+		return true, false, true
+	case "layer_block":
+		return false, true, false
+	case "image_block":
+		return true, true, false
+	case "layer_block_with_verity":
+		return false, true, true
+	case "image_block_with_verity":
+		return true, true, true
+	default:
+		return false, false, false
+	}
+}
+
 func ProcessConfigurations(c *SnapshotterConfig) error {
 	if c.LoggingConfig.LogDir == "" {
 		c.LoggingConfig.LogDir = filepath.Join(c.Root, logging.DefaultLogDirName)
@@ -109,13 +180,14 @@ func ProcessConfigurations(c *SnapshotterConfig) error {
 	globalConfig.SnapshotsDir = filepath.Join(c.Root, "snapshots")
 	globalConfig.ConfigRoot = filepath.Join(c.Root, "config")
 	globalConfig.SocketRoot = filepath.Join(c.Root, "socket")
+	globalConfig.RootMountpoint = filepath.Join(c.Root, "mnt")
 
 	globalConfig.MirrorsConfig = c.RemoteConfig.MirrorsConfig
 
 	if c.CacheManagerConfig.GCPeriod != "" {
 		d, err := time.ParseDuration(c.CacheManagerConfig.GCPeriod)
 		if err != nil {
-			return errors.Errorf("invalid GC period %s", c.CacheManagerConfig.GCPeriod)
+			return errors.Errorf("invalid GC period '%s'", c.CacheManagerConfig.GCPeriod)
 		}
 		globalConfig.CacheGCPeriod = d
 	}
@@ -126,7 +198,7 @@ func ProcessConfigurations(c *SnapshotterConfig) error {
 	}
 
 	if c.DaemonConfig.FsDriver == FsDriverFscache && m != DaemonModeShared {
-		log.L.Infof("fscache driver must enforce \"shared\" daemon mode, change it forcefully from %s", m)
+		log.L.Infof("fscache driver only supports 'shared' mode, override daemon mode from '%s' to 'shared'", m)
 		m = DaemonModeShared
 	}
 
@@ -134,3 +206,16 @@ func ProcessConfigurations(c *SnapshotterConfig) error {
 
 	return nil
 }
+
+func SetUpEnvironment(c *SnapshotterConfig) error {
+	if err := os.MkdirAll(c.Root, 0700); err != nil {
+		return errors.Wrapf(err, "create root dir %s", c.Root)
+	}
+
+	realPath, err := mount.NormalizePath(c.Root)
+	if err != nil {
+		return errors.Wrapf(err, "invalid root path")
+	}
+	c.Root = realPath
+	return nil
+}

docs/configure_nydus.md

@@ -1,10 +1,10 @@
 # Configure Nydus-snapshotter
 
-Nydus-snapshotter can receive a toml file as its configurations to start providing image service through CLI parameter `--config`. A example configuration file can be found [here](../misc/snapshotter/config.toml). Besides nydus-snapshotter's configuration, `nydusd`'s configuration has to be provided to nydus-snapshotter too. Nydusd is started by nydus-snapshotter and it is configured by the provided json configuration file. A minimal configuration file can be found [here](../misc/snapshotter/nydusd-config.fusedev.json)
+Nydus-snapshotter can receive a toml file as its configurations to start providing image service through CLI parameter `--config`. An example configuration file can be found [here](../misc/snapshotter/config.toml). Besides nydus-snapshotter's configuration, `nydusd`'s configuration has to be provided to nydus-snapshotter too. Nydusd is started by nydus-snapshotter and it is configured by the provided json configuration file. A minimal configuration file can be found [here](../misc/snapshotter/nydusd-config.fusedev.json)
 
 ## Authentication
 
-As [contianerd#3731](https://github.com/containerd/containerd/issues/3731) discussed, containerd doesn't share credentials with third snapshotters now. Like [stargz snapshotter](https://github.com/containerd/stargz-snapshotter/blob/main/docs/overview.md#authentication), nydus-snapshotter supports 3 main ways to access registries with custom configurations. You can use configuration file to enable them.
+As [containerd#3731](https://github.com/containerd/containerd/issues/3731) discussed, containerd doesn't share credentials with third snapshotters now. Like [stargz snapshotter](https://github.com/containerd/stargz-snapshotter/blob/main/docs/overview.md#authentication), nydus-snapshotter supports 3 main ways to access registries with custom configurations. You can use configuration file to enable them.
 
 The snapshotter will try to get image pull keychain in the following order if such way is enabled:
 
@@ -128,9 +128,12 @@ The Nydus snapshotter will get the new secret and parse the authorization. If yo
 
 ## Metrics
 
-Nydusd records metrics in its own format. The metrics are exported via a HTTP server on top of unix domain socket. Nydus-snapshotter fetches the metrics and convert them in to Prometheus format which is exported via a network address. Nydus-snapshotter by default does not fetch metrics from nydusd. You can enable the nydusd metrics download by assigning an network address to `metrics.address` in nydus-snapshotter's toml [configuration file](../misc/snapshotter/config.toml).
+Nydusd records metrics in its own format. The metrics are exported via a HTTP server on top of unix domain socket. Nydus-snapshotter fetches the metrics and convert them in to Prometheus format which is exported via a network address. Nydus-snapshotter by default does not fetch metrics from nydusd. You can enable the nydusd metrics download by assigning a network address to `metrics.address` in nydus-snapshotter's toml [configuration file](../misc/snapshotter/config.toml).
+
+Once this entry is enabled, not only nydusd metrics, but also some information about the nydus-snapshotter 
+runtime and snapshot related events are exported in Prometheus format as well.
 
 ## Diagnose
 
 A system controller can be ran insides nydus-snapshotter.
-By setting `system.enable` to `true`,  nydus-snapshotter will start a simple HTTP serve on unix domain socket `system.address` path and exports some internal working status to users. The address defaults to `/var/run/containerd-nydus/system.sock`
+By setting `system.enable` to `true`,  nydus-snapshotter will start a simple HTTP server on unix domain socket `system.address` path and exports some internal working status to users. The address defaults to `/var/run/containerd-nydus/system.sock`

docs/optimize_nydus_image.md

@@ -1,6 +1,6 @@
 # Optimize a nydus image
 
-To improve the prefetch hit rate, we can specify a prefetch table when converting an OCI image to a nydus image. The prefetch table is workload related and we should generate the list of accessed files in order when the workload is running. Nydus-snapshotter has implemented an optimizer with Containerd NRI plugin to optimize the nydus image. The optimizer is image format independent and requires NRI 2.0, which is available in containerd 1.7.0 (not yet released). The optimizer subscribes container events to watch what files are opened and read, etc. during container application is starting up. This enables nydus image build tools to optimize the nydus image making it put the necessary and prioritized files into a special region of nydus image with extra image metadata. So nydus runtime can pull this files into local disk in top priority, thus to boost the performance further.
+To improve the prefetch hit rate, we can specify a prefetch table when converting an OCI image to a nydus image. The prefetch table is workload related and we should generate the list of accessed files in order when the workload is running. Nydus-snapshotter has implemented an optimizer with Containerd NRI plugin to optimize the nydus image. The optimizer is image format independent and requires NRI 2.0, which is available in containerd (>=v1.7.0). The optimizer subscribes container events to watch what files are opened and read, etc. during container application is starting up. This enables nydus image build tools to optimize the nydus image making it put the necessary and prioritized files into a special region of nydus image with extra image metadata. So nydus runtime can pull this files into local disk in top priority, thus to boost the performance further.
 
 ## Requirements
 
@@ -30,6 +30,8 @@ This command installs the optimizer's default toml configuration file in `/etc/n
 ```toml
 # The directory to persist accessed files list for container.
 persist_dir = "/opt/nri/optimizer/results"
+# Whether to make the csv file human readable.
+readable = false
 # The path of optimizer server binary.
 server_path = "/usr/local/bin/optimizer-server"
 # The timeout to kill optimizer server, 0 to disable it.
@@ -54,7 +56,7 @@ EOF
 
 ```
 
-Containerd will load all NRI plugins in the `plugin_path` directory on startup. If you want to start an NRI plugin manually, please add the following configuration to allow other NRI plugins to connect via `socket_path`.
+Containerd will load all NRI plugins in the `plugin_path` directory on startup. If you want to start a NRI plugin manually, please add the following configuration to allow other NRI plugins to connect via `socket_path`.
 
 ```console
 sudo tee /etc/nri/nri.conf <<- EOF
@@ -97,18 +99,18 @@ crictl run nginx.yaml pod.yaml
 
 ```
 
-The result file for the nginx image is `/opt/nri/optimizer/results/nginx:latest`.
+The result file for the nginx image is `/opt/nri/optimizer/results/library/nginx:latest`.
 
 ## Build Nydus Image with Optimizer's Suggestions
 
-Nydus provides a [nydusify](https://github.com/dragonflyoss/image-service/blob/master/docs/nydusify.md) CLI tool to convert OCI images from the source registry or local file system to nydus format and push them to the target registry.
+Nydus provides a [nydusify](https://github.com/dragonflyoss/nydus/blob/master/docs/nydusify.md) CLI tool to convert OCI images from the source registry or local file system to nydus format and push them to the target registry.
 
 We can install the `nydusify` cli tool from the nydus package.
 
 ```console
-VERSION=v2.1.4
+VERSION=v2.3.0
 
-wget https://github.com/dragonflyoss/image-service/releases/download/$VERSION/nydus-static-$VERSION-linux-amd64.tgz
+wget https://github.com/dragonflyoss/nydus/releases/download/$VERSION/nydus-static-$VERSION-linux-amd64.tgz
 tar -zxvf nydus-static-$VERSION-linux-amd64.tgz
 sudo install -D -m 755 nydus-static/nydusify /usr/local/bin/nydusify
 sudo install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image
@@ -123,7 +125,7 @@ sudo nydusify convert --source nginx --target sctb512/nginx:nydusv6 --fs-version
 With the `--prefetch-patterns` argument, we can specify the list of files to be written in the front of the nydus image and be prefetched in order when starting a container.
 
 ```console
-sudo nydusify convert --source nginx --target sctb512/nginx:optimized-nydusv6 --fs-version 6 --prefetch-patterns < /opt/nri/optimizer/results/nginx:latest
+sudo nydusify convert --source nginx --target sctb512/nginx:optimized-nydusv6 --fs-version 6 --prefetch-patterns < /opt/nri/optimizer/results/library/nginx:latest
 ```
 
 On a host with nydus-snapshotter installed and configured properly, start a container with an optimized nydus image.

docs/run_nydus_in_kubernetes.md

@@ -1,3 +1,9 @@
+# Run Dragonfly & Nydus in Kubernetes
+
+We recommend using the Dragonfly P2P data distribution system to further improve the runtime performance of Nydus images.
+
+If you want to deploy Dragonfly and Nydus at the same time through Helm, please refer to the **[Quick Start](https://github.com/dragonflyoss/helm-charts/blob/main/INSTALL.md)**.
+
 # Run Nydus snapshotter in Kubernetes
 
 This document will introduce how to run Nydus snapshotter in Kubernetes cluster, you can use helm to deploy Nydus snapshotter container.
@@ -18,7 +24,7 @@ networking:
   ipFamily: dual
 nodes:
   - role: control-plane
-    image: kindest/node:v1.23.4
+    image: kindest/node:v1.30.2
     extraMounts:
       - hostPath: ./containerd-config.toml
         containerPath: /etc/containerd/config.toml
@@ -28,6 +34,10 @@ nodes:
 
 Next, we also need a config for containerd(`containerd-config.toml`).
 
+**NOTE:** It may be necessary to explain here why `disable_snapshot_annotations` and `discard_unpacked_layers` need to be configured in containerd.
+- `disable_snapshot_annotations`: This variable disables to pass additional annotations (image related information) to snapshotters in containerd (default value is `true`). In nydus snapshotter, we need these annotations to pull images. Therefore, we need to set it to `false`.
+- `discard_unpacked_layers`: This variable allows GC to remove layers from the content store after successfully unpacking these layers to the snapshotter in containerd (default value is `true`). In nydus snapshotter, we need to preserve layers for demand pulling and sharing even after they are unpacked. Therefore, we need to set it to `false`.
+
 ```toml
 version = 2
 [debug]

docs/setup_snapshotter_by_daemonset.md

@@ -0,0 +1,165 @@
+# Setup Nydus Snapshotter by DaemonSet
+
+This document will guide you through the simple steps of setting up and cleaning up the nydus snapshotter in a kubernetes cluster that runs on the host.
+
+## Steps for Setting up Nydus Snapshotter 
+
+To begin, let's clone the Nydus Snapshotter repository.
+
+```bash
+git clone https://github.com/containerd/nydus-snapshotter
+cd nydus-snapshotter
+```
+
+We can build the docker image locally. (optional)
+```bash
+$ export NYDUS_VER=$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name)
+$ make  # build snapshotter binaries
+$ cp bin/* misc/snapshotter/
+$ pushd misc/snapshotter/
+$ docker build --build-arg NYDUS_VER="${NYDUS_VER}" -t ghcr.io/containerd/nydus-snapshotter:latest .
+$ popd
+```
+**NOTE:** By default, the nydus snapshotter would use the latest release nydus version. If you want to use a specific version, you can set `NYDUS_VER` on your side.
+
+Next, we can configure access control for nydus snapshotter.
+```bash
+kubectl apply -f misc/snapshotter/nydus-snapshotter-rbac.yaml
+```
+
+Afterward, we can deploy a DaemonSet for nydus snapshotter, according to the kubernetes flavour you're using.
+
+```bash
+# Vanilla kubernetes
+kubectl apply -f misc/snapshotter/base/nydus-snapshotter.yaml
+```
+
+```bash
+# k3s
+kubectl apply -k misc/snapshotter/overlays/k3s/
+```
+
+```bash
+# rke2
+kubectl apply -k misc/snapshotter/overlays/rke2/
+```
+
+Then, we can confirm that nydus snapshotter is running through the DaemonSet.
+```bash
+$ kubectl get pods -n nydus-system 
+NAME                      READY   STATUS    RESTARTS   AGE
+nydus-snapshotter-26rf7   1/1     Running   0          18s
+```
+
+Finally, we can view the logs in the pod.
+```bash
+$ kubectl logs nydus-snapshotter-26rf7 -n nydus-system
+install nydus snapshotter artifacts
+there is no proxy plugin!
+Created symlink /etc/systemd/system/multi-user.target.wants/nydus-snapshotter.service → /etc/systemd/system/nydus-snapshotter.service.
+```
+
+And we can see the nydus snapshotter service on the host.
+```bash
+$ systemctl status nydus-snapshotter
+● nydus-snapshotter.service - nydus snapshotter
+     Loaded: loaded (/etc/systemd/system/nydus-snapshotter.service; enabled; vendor preset: enabled)
+    Drop-In: /etc/systemd/system/nydus-snapshotter.service.d
+             └─proxy.conf
+     Active: active (running) since Wed 2024-01-17 16:14:22 UTC; 56s ago
+   Main PID: 1100169 (containerd-nydu)
+      Tasks: 11 (limit: 96376)
+     Memory: 8.6M
+        CPU: 35ms
+     CGroup: /system.slice/nydus-snapshotter.service
+             └─1100169 /opt/nydus/bin/containerd-nydus-grpc --config /etc/nydus/config.toml
+
+Jan 17 16:14:22 worker systemd[1]: Started nydus snapshotter.
+Jan 17 16:14:22 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:22.998798369Z" level=info msg="Start nydus-snapshotter. Version: v0.7.0-308-g106a6cb, PID: 1100169, FsDriver: fusedev, DaemonMode: dedicated"
+Jan 17 16:14:23 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:23.000186538Z" level=info msg="Run daemons monitor..."
+```
+
+**NOTE:** By default, the nydus snapshotter operates as a systemd service. If you prefer to run nydus snapshotter as a standalone process, you can set `ENABLE_SYSTEMD_SERVICE` to `false` in `nydus-snapshotter.yaml`.
+
+## Steps for Cleaning up Nydus Snapshotter 
+
+We use `preStop`` hook in the DaemonSet to uninstall nydus snapshotter and roll back the containerd configuration.
+
+```bash
+# Vanilla kubernetes
+$ kubectl delete -f misc/snapshotter/base/nydus-snapshotter.yaml 
+```
+
+```bash
+# k3s
+$ kubectl delete -k misc/snapshotter/overlays/k3s/ 
+```
+
+```bash
+# rke2
+$ kubectl delete -k misc/snapshotter/overlays/rke2/
+```
+
+```bash
+$ kubectl delete -f misc/snapshotter/nydus-snapshotter-rbac.yaml 
+$ systemd restart containerd.service
+```
+
+## Customized Setup
+
+As we know, nydus snapshotter supports four filesystem drivers (fs_driver): `fusedev`, `fscache`, `blockdev`, `proxy`. Within the container image, we have included configurations for these snapshotter drivers, as well as the corresponding nydusd configurations. By default, the fusedev driver is enabled in the nydus snapshotter, using the snapshotter configuration [`config-fusedev.toml`](../misc/snapshotter/config-fusedev.toml) and the nydusd configuration [`nydusd-config.fusedev.json`](../misc/snapshotter/nydusd-config.fusedev.json).
+
+### Other filesystem driver with related default configuration
+
+If we want to setup the nydus snapshotter with the default configuration for different fs_driver (such as `proxy`), we can modify the values in the `Configmap` in `nydus-snapshotter.yaml`:
+```yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
+  namespace: nydus-snapshotter
+data:
+  FS_DRIVER: "proxy"
+  NYDUSD_DAEMON_MODE: "none"
+```
+
+Then we can run the nydus snapshotter enabling `proxy` `fs_driver` with the snapshotter configuration [`config-proxy.toml`](../misc/snapshotter/config-proxy.toml).
+
+**NOTE:** The fs_driver (`blockdev` and `proxy`) do not need nydusd, so they do not need nydusd config.
+
+### Same filesystem with different snapshotter configuration and different nydusd configuration
+
+If we want to setup the nydus snapshotter for the same fs_driver (such as `fusedev`) with different snapshotter configuration and different nydusd configuration, we can enable `ENABLE_CONFIG_FROM_VOLUME` and add the snapshotter configuration [`config.toml`](../misc/snapshotter/config.toml) and nydusd configuration [`nydusd-config.json`](../misc/snapshotter/nydusd-config.fusedev.json) in the `Configmap` in `nydus-snapshotter.yaml`:
+
+```yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
+  namespace: nydus-snapshotter
+data:
+  ENABLE_CONFIG_FROM_VOLUME: "true"
+
+  config.toml: |-
+    # The snapshotter config content copied here
+
+  nydusd-config.json: |-
+    # The nydusd config content copied here
+```
+
+**NOTE:** We need to set `nydusd_config` to `/etc/nydus/nydusd-config.json` in the `config.toml`, so that snapshotter can find the nydusd configuration from configmap.
+
+### Customized Options
+
+| Options                             | Type   | Default                               | Comment                                                                                                                                         |
+| ----------------------------------- | ------ | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| FS_DRIVER                           | string | "fusedev"                             | the filesystem driver of snapshotter                                                                                                            |
+| ENABLE_CONFIG_FROM_VOLUME           | bool   | false                                 | enabling to use the configurations from volume                                                                                                  |
+| ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER | bool   | false                                 | enabling to skip to set `plugins."io.containerd.grpc.v1.cri".containerd` to `nydus` for runtime specific snapshotter feature in containerd 1.7+ |
+| ENABLE_SYSTEMD_SERVICE              | bool   | true                                  | enabling to run nydus snapshotter as a systemd service                                                                                          |

docs/tarfs.md

@@ -0,0 +1,175 @@
+# Nydus Tarfs Mode
+
+`Nydus Tarfs Mode` or `Tarfs` is a working mode for Nydus Image, which uses tar files as Nydus data blobs instead of generating native Nydus data blobs.
+
+### Enable Tarfs
+`Nydus Tarfs Mode` is still an experiment feature, please edit the snapshotter configuration file to enable the feature:
+```
+[experimental.tarfs]
+enable_tarfs = true
+```
+
+### Generate Raw Disk Image for Each Layer of a Container Image
+`Tarfs` supports generating a raw disk image for each layer of a container image, which can be directly mounted as EROFS filesystem through loopdev. Please edit the snapshotter configuration file to enable this submode:
+```
+[experimental.tarfs]
+enable_tarfs = true
+export_mode = "layer_block"
+```
+
+This is an example to generate and verify raw disk image for each layer of a container image:
+```
+$ containerd-nydus-grpc --config /etc/nydus/config.toml &
+$ nerdctl run --snapshotter nydus --rm nginx
+
+# Show mounted rootfs a container
+$ mount
+/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
+
+# Show loop devices used to mount layers and bootstrap for a container image
+$ losetup 
+NAME SIZELIMIT OFFSET AUTOCLEAR RO BACK-FILE                                                                       DIO LOG-SEC
+/dev/loop11 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa 0     512
+/dev/loop12 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3 0     512
+/dev/loop13 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd 0     512
+/dev/loop14 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4 0     512
+/dev/loop15 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665 0     512
+/dev/loop16 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75 0     512
+/dev/loop17 0      0         0  0 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/fs/image/image.boot                         0     512
+
+# Files without suffix are tar files, files with suffix `layer.disk` are raw disk image for container image layers
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
+total 376800
+-rw-r--r-- 1 root root      3584 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+-rw-r--r-- 1 root root    527872 Aug 30 23:18 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk
+-rw-r--r-- 1 root root  77814784 Aug 30 23:18 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5
+-rw-r--r-- 1 root root  78863360 Aug 30 23:18 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5.layer.disk
+-rw-r--r-- 1 root root      4608 Aug 30 23:18 96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd
+-rw-r--r-- 1 root root    528896 Aug 30 23:18 96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd.layer.disk
+-rw-r--r-- 1 root root      2560 Aug 30 23:18 a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4
+-rw-r--r-- 1 root root    526848 Aug 30 23:18 a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4.layer.disk
+-rw-r--r-- 1 root root      7168 Aug 30 23:18 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75
+-rw-r--r-- 1 root root    531456 Aug 30 23:18 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.layer.disk
+-rw-r--r-- 1 root root      5120 Aug 30 23:18 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665
+-rw-r--r-- 1 root root    529408 Aug 30 23:18 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665.layer.disk
+-rw-r--r-- 1 root root 112968704 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa
+-rw-r--r-- 1 root root 113492992 Aug 30 23:18 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa.layer.disk
+$ file /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3: POSIX tar archive
+
+# Mount the raw disk image for a container image layer
+$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3.layer.disk 
+$ mount -t erofs /dev/loop100 ./mnt/
+$ mount
+tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=1544836k,nr_inodes=386209,mode=700,inode64)
+/dev/loop17 on /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/7/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
+/dev/loop100 on /root/ws/nydus-snapshotter.git/mnt type erofs (ro,relatime,user_xattr,acl,cache_strategy=readaround)
+
+```
+
+### Generate Raw Disk Image for a Container Image
+`Tarfs` supports generating a raw disk image a container image, which can be directly mounted as EROFS filesystem through loopdev. Please edit the snapshotter configuration file to enable this submode:
+```
+[experimental.tarfs]
+enable_tarfs = true
+export_mode = "image_block"
+```
+
+This is an example to generate and verify raw disk image for a container image:
+```
+$ containerd-nydus-grpc --config /etc/nydus/config.toml &
+$ nerdctl run --snapshotter nydus --rm nginx
+
+# Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
+total 376320
+-rw-r--r-- 1 root root      3584 Aug 30 23:35 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+-rw-r--r-- 1 root root  77814784 Aug 30 23:35 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5
+-rw-r--r-- 1 root root      4608 Aug 30 23:35 96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd
+-rw-r--r-- 1 root root      2560 Aug 30 23:35 a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4
+-rw-r--r-- 1 root root      7168 Aug 30 23:35 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75
+-rw-r--r-- 1 root root 194518016 Aug 30 23:36 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+-rw-r--r-- 1 root root      5120 Aug 30 23:35 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665
+-rw-r--r-- 1 root root 112968704 Aug 30 23:36 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa
+
+```
+
+### Generate Raw Disk Image with dm-verity Information
+`Tarfs` supports generating raw disk images with dm-verity information, to enable runtime data integrity validation. Please change `export_mode` in snapshotter configuration file to `layer_block_with_verity` or `image_block_with_verity`.
+
+```
+[experimental.tarfs]
+enable_tarfs = true
+export_mode = "image_block_with_verity"
+```
+
+This is an example to generate and verify raw disk image for a container image with dm-verity information:
+```
+$ containerd-nydus-grpc --config /etc/nydus/config.toml &
+$ nerdctl run --snapshotter nydus --rm nginx
+
+# Files without suffix are tar files, files with suffix `image.disk` are raw disk image for a container image
+$ ls -l /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/
+total 388296
+-rw-r--r-- 1 root root      3584 Aug 30 23:45 055fa98b43638b67d10c58d41094d99c8696cc34b7a960c7a0cc5d9d152d12b3
+-rw-r--r-- 1 root root  77814784 Aug 30 23:46 52d2b7f179e32b4cbd579ee3c4958027988f9a8274850ab0c7c24661e3adaac5
+-rw-r--r-- 1 root root      4608 Aug 30 23:45 96576293dd2954ff84251aa0455687c8643358ba1b190ea1818f56b41884bdbd
+-rw-r--r-- 1 root root      2560 Aug 30 23:45 a7c4092be9044bd4eef78f27c95785ef3a9f345d01fd4512bc94ddaaefc359f4
+-rw-r--r-- 1 root root      7168 Aug 30 23:45 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75
+-rw-r--r-- 1 root root 206782464 Aug 30 23:46 da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+-rw-r--r-- 1 root root      5120 Aug 30 23:45 e3b6889c89547ec9ba653ab44ed32a99370940d51df956968c0d578dd61ab665
+-rw-r--r-- 1 root root 112968704 Aug 30 23:46 fd9f026c631046113bd492f69761c3ba6042c791c35a60e7c7f3b8f254592daa
+
+$ losetup /dev/loop100 /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+$  veritysetup open --no-superblock --format=1 -s "" --hash=sha256 --data-block-size=512 --hash-block-size=4096 --data-blocks 379918 --hash-offset 194519040 /dev/loop100 image1 /dev/loop100 8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd
+$ lsblk
+loop100   7:100  0 197.2M  0 loop  
+└─dm-0  252:0    0 185.5M  1 crypt 
+
+$ veritysetup status dm-0
+/dev/mapper/dm-0 is active and is in use.
+  type:        VERITY
+  status:      verified
+  hash type:   1
+  data block:  512
+  hash block:  4096
+  hash name:   sha256
+  salt:        -
+  data device: /dev/loop100
+  data loop:   /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+  size:        379918 sectors
+  mode:        readonly
+  hash device: /dev/loop100
+  hash loop:   /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/da761d9a302b21dc50767b67d46f737f5072fb4490c525b4a7ae6f18e1dbbf75.image.disk
+  hash offset: 379920 sectors
+  root hash:   8113799aaf9a5d14feca1eadc3b7e6ea98bdaf61e3a2e4a8ef8c24e26a551efd
+
+$ mount -t erofs /dev/dm-0 ./mnt/
+mount: /root/ws/nydus-snapshotter.git/mnt: WARNING: source write-protected, mounted read-only.
+$ ls -l mnt/
+total 14
+lrwxrwxrwx  1 root root    7 Aug 14 08:00 bin -> usr/bin
+drwxr-xr-x  2 root root   27 Jul 15 00:00 boot
+drwxr-xr-x  2 root root   27 Aug 14 08:00 dev
+drwxr-xr-x  2 root root  184 Aug 16 17:50 docker-entrypoint.d
+-rwxrwxr-x  1 root root 1620 Aug 16 17:50 docker-entrypoint.sh
+drwxr-xr-x 34 root root 1524 Aug 16 17:50 etc
+drwxr-xr-x  2 root root   27 Jul 15 00:00 home
+lrwxrwxrwx  1 root root    7 Aug 14 08:00 lib -> usr/lib
+lrwxrwxrwx  1 root root    9 Aug 14 08:00 lib32 -> usr/lib32
+lrwxrwxrwx  1 root root    9 Aug 14 08:00 lib64 -> usr/lib64
+lrwxrwxrwx  1 root root   10 Aug 14 08:00 libx32 -> usr/libx32
+drwxr-xr-x  2 root root   27 Aug 14 08:00 media
+drwxr-xr-x  2 root root   27 Aug 14 08:00 mnt
+drwxr-xr-x  2 root root   27 Aug 14 08:00 opt
+drwxr-xr-x  2 root root   27 Jul 15 00:00 proc
+drwx------  2 root root   66 Aug 14 08:00 root
+drwxr-xr-x  3 root root   43 Aug 14 08:00 run
+lrwxrwxrwx  1 root root    8 Aug 14 08:00 sbin -> usr/sbin
+drwxr-xr-x  2 root root   27 Aug 14 08:00 srv
+drwxr-xr-x  2 root root   27 Jul 15 00:00 sys
+drwxrwxrwt  2 root root   27 Aug 16 17:50 tmp
+drwxr-xr-x 14 root root  229 Aug 14 08:00 usr
+drwxr-xr-x 11 root root  204 Aug 14 08:00 var
+
+```

export/snapshotter/snapshotter.go

@@ -1,8 +1,10 @@
 package snapshotter
 
 import (
-	"github.com/containerd/containerd/platforms"
-	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/v2/plugins"
+	"github.com/containerd/platforms"
+	"github.com/containerd/plugin"
+	"github.com/containerd/plugin/registry"
 	"github.com/pkg/errors"
 
 	"github.com/containerd/nydus-snapshotter/config"
@@ -10,8 +12,8 @@ import (
 )
 
 func init() {
-	plugin.Register(&plugin.Registration{
-		Type:   plugin.SnapshotPlugin,
+	registry.Register(&plugin.Registration{
+		Type:   plugins.SnapshotPlugin,
 		ID:     "nydus",
 		Config: &config.SnapshotterConfig{},
 		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
@@ -22,8 +24,9 @@ func init() {
 				return nil, errors.New("invalid nydus snapshotter configuration")
 			}
 
-			if cfg.Root == "" {
-				cfg.Root = ic.Root
+			root := ic.Properties[plugins.PropertyRootDir]
+			if root == "" {
+				cfg.Root = root
 			}
 
 			if err := cfg.FillUpWithDefaults(); err != nil {

go.mod

@@ -1,149 +1,165 @@
 module github.com/containerd/nydus-snapshotter
 
-go 1.19
+go 1.22.0
 
 require (
-	github.com/KarpelesLab/reflink v0.0.2
-	github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible
-	github.com/aws/aws-sdk-go-v2 v1.17.5
-	github.com/aws/aws-sdk-go-v2/config v1.18.15
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.15
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55
-	github.com/containerd/containerd v1.7.0-rc.1
-	github.com/containerd/continuity v0.3.0
+	dario.cat/mergo v1.0.1
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6
+	github.com/KarpelesLab/reflink v1.0.1
+	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
+	github.com/aws/aws-sdk-go-v2 v1.30.3
+	github.com/aws/aws-sdk-go-v2/config v1.27.27
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.27
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.8
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2
+	github.com/containerd/cgroups/v3 v3.0.3
+	github.com/containerd/containerd/api v1.8.0
+	github.com/containerd/containerd/v2 v2.0.2
+	github.com/containerd/continuity v0.4.4
+	github.com/containerd/errdefs v1.0.0
 	github.com/containerd/fifo v1.1.0
-	github.com/containerd/nri v0.3.0
-	github.com/containerd/stargz-snapshotter v0.14.2
-	github.com/containerd/stargz-snapshotter/estargz v0.14.2
-	github.com/docker/cli v23.0.1+incompatible
+	github.com/containerd/log v0.1.0
+	github.com/containerd/nri v0.8.0
+	github.com/containerd/platforms v1.0.0-rc.1
+	github.com/containerd/plugin v1.0.0
+	github.com/containerd/stargz-snapshotter v0.15.2-0.20240709063920-1dac5ef89319
+	github.com/containerd/stargz-snapshotter/estargz v0.15.2-0.20240709063920-1dac5ef89319
+	github.com/containers/ocicrypt v1.2.0
+	github.com/distribution/reference v0.6.0
+	github.com/docker/cli v27.1.0+incompatible
+	github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
-	github.com/google/go-containerregistry v0.13.0
-	github.com/gorilla/mux v1.8.0
-	github.com/hashicorp/go-retryablehttp v0.7.2
-	github.com/klauspost/compress v1.16.0
+	github.com/google/go-containerregistry v0.20.1
+	github.com/gorilla/mux v1.8.1
+	github.com/hashicorp/go-retryablehttp v0.7.7
+	github.com/klauspost/compress v1.17.11
+	github.com/moby/locker v1.0.1
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
+	github.com/opencontainers/image-spec v1.1.0
+	github.com/opencontainers/runtime-spec v1.2.0
 	github.com/pelletier/go-toml v1.9.5
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.14.0
-	github.com/prometheus/client_model v0.3.0
-	github.com/rs/xid v1.4.0
-	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.8.2
-	github.com/urfave/cli/v2 v2.25.0
-	go.etcd.io/bbolt v1.3.7
-	golang.org/x/net v0.8.0
-	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.6.0
-	google.golang.org/grpc v1.53.0
+	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/client_model v0.6.1
+	github.com/rs/xid v1.5.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.10.0
+	github.com/urfave/cli/v2 v2.27.5
+	go.etcd.io/bbolt v1.3.11
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	golang.org/x/net v0.30.0
+	golang.org/x/sync v0.8.0
+	golang.org/x/sys v0.26.0
+	google.golang.org/grpc v1.67.1
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.26.2
-	k8s.io/apimachinery v0.26.2
-	k8s.io/client-go v0.26.2
-	k8s.io/cri-api v0.27.0-alpha.3
+	k8s.io/api v0.31.2
+	k8s.io/apimachinery v0.31.2
+	k8s.io/client-go v0.31.2
+	k8s.io/cri-api v0.31.2
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 )
 
 require (
-	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
-	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect
-	github.com/containerd/typeurl/v2 v2.1.0 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
-	github.com/fatih/color v1.14.1 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-hclog v1.2.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/moby/sys/sequential v0.5.0 // indirect
-	github.com/prometheus/common v0.41.0 // indirect
-	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	go.opentelemetry.io/otel v1.14.0 // indirect
-	go.opentelemetry.io/otel/trace v1.14.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-)
-
-require (
-	github.com/Microsoft/go-winio v0.6.0 // indirect
-	github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.24 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.23 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5
-	github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 // indirect
-	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.12.9 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
+	github.com/aws/smithy-go v1.20.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/containerd/cgroups v1.1.0 // indirect
-	github.com/containerd/ttrpc v1.1.1-0.20230127163717-32fab2374638 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cilium/ebpf v0.11.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
+	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/distribution v2.8.1+incompatible // indirect
-	github.com/docker/docker v23.0.1+incompatible // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
-	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
-	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/gnostic v0.6.9 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/imdario/mergo v0.3.13
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/moby/locker v1.0.1 // indirect
-	github.com/moby/sys/mountinfo v0.6.2 // indirect
-	github.com/moby/sys/signal v0.7.0 // indirect
+	github.com/miekg/pkcs11 v1.1.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/opencontainers/runc v1.1.4 // indirect
-	github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
-	github.com/opencontainers/selinux v1.10.2 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opencontainers/selinux v1.11.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/vbatts/tar-split v0.11.2 // indirect
+	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/vbatts/tar-split v0.11.5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.mozilla.org/pkcs7 v0.9.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/mod v0.9.0
-	golang.org/x/oauth2 v0.6.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
+	go.opentelemetry.io/otel v1.31.0 // indirect
+	go.opentelemetry.io/otel/metric v1.31.0 // indirect
+	go.opentelemetry.io/otel/trace v1.31.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/oauth2 v0.22.0 // indirect
+	golang.org/x/term v0.25.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.90.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect
-	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+	tags.cncf.io/container-device-interface v0.8.0 // indirect
+	tags.cncf.io/container-device-interface/specs-go v0.8.0 // indirect
 )
 
 retract (
@@ -153,3 +169,18 @@ retract (
 	v0.11.0
 	v0.3.0 // retagged: cd604c1b597558ea045a79c4f80a8c780909801b -> 85653575c7dafb6b06548478ee1dc61ac5905d00
 )
+
+exclude (
+	// These dependencies were updated to "master" in some modules we depend on,
+	// but have no code-changes since their last release. Unfortunately, this also
+	// causes a ripple effect, forcing all users of the containerd module to also
+	// update these dependencies to an unrelease / un-tagged version.
+	//
+	// Both these dependencies will unlikely do a new release in the near future,
+	// so exclude these versions so that we can downgrade to the current release.
+	//
+	// For additional details, see this PR and links mentioned in that PR:
+	// https://github.com/kubernetes-sigs/kustomize/pull/5830#issuecomment-2569960859
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
+)

go.sum

@@ -1,140 +1,155 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
-github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
-github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 h1:+vTEFqeoeur6XSq06bs+roX3YiT49gUniJK7Zky7Xjg=
-github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU=
+dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
+dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 h1:dIScnXFlF784X79oi7MzVT6GWqr/W1uUt0pB5CsDs9M=
+github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/KarpelesLab/reflink v0.0.2 h1:PK0nDUsk2eEdIvMOUzh2TI35GY5KYmTdqdCCwd3BIz0=
-github.com/KarpelesLab/reflink v0.0.2/go.mod h1:mB+2afhyn+eZTMFSH1gXunrgArTsiUBzfoAy0X2fatA=
-github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg=
-github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE=
-github.com/Microsoft/hcsshim v0.10.0-rc.7 h1:HBytQPxcv8Oy4244zbQbe6hnOnx544eL5QPUqhJldz8=
-github.com/Microsoft/hcsshim v0.10.0-rc.7/go.mod h1:ILuwjA+kNW+MrN/w5un7n3mTqkwsFu4Bp05/okFUZlE=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible h1:KXeJoM1wo9I/6xPTyt6qCxoSZnmASiAjlrr0dyTUKt8=
-github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/aws/aws-sdk-go-v2 v1.17.5 h1:TzCUW1Nq4H8Xscph5M/skINUitxM5UBAyvm2s7XBzL4=
-github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
-github.com/aws/aws-sdk-go-v2/config v1.18.15 h1:509yMO0pJUGUugBP2H9FOFyV+7Mz7sRR+snfDN5W4NY=
-github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.15 h1:0rZQIi6deJFjOEgHI9HI2eZcLPPEGQPictX66oRFLL8=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23 h1:Kbiv9PGnQfG/imNI4L/heyUXvzKmcWSBeDvkrQz5pFc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55 h1:ClZKHmu2QIRQCEQ2Y2upfu4JPO0pG69Ce5eiq3PS2V4=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.55/go.mod h1:L/h5B6I7reig2QJXCGY0e0NVx4hYCcjETmsfR02hFng=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29 h1:9/aKwwus0TQxppPXFmf010DFrE+ssSbzroLVYINA+xE=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23 h1:b/Vn141DBuLVgXbhRWIrl9g+ww7G+ScV5SzniWR13jQ=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30 h1:IVx9L7YFhpPq0tTnGo8u8TpluFu7nAn9X3sUDMb11c0=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.21 h1:QdxdY43AiwsqG/VAqHA7bIVSm3rKr8/p9i05ydA0/RM=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.21/go.mod h1:QtIEat7ksHH8nFItljyvMI0dGj8lipK2XZ4PhNihTEU=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.24 h1:Qmm8klpAdkuN3/rPrIMa/hZQ1z93WMBPjOzdAsbSnlo=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.24/go.mod h1:QelGeWBVRh9PbbXsfXKTFlU9FjT6W2yP+dW5jMQzOkg=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23 h1:QoOybhwRfciWUBbZ0gp9S7XaDnCuSTeK/fySB99V1ls=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.23 h1:qc+RW0WWZ2KApMnsu/EVCPqLTyIH55uc7YQq7mq4XqE=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.23/go.mod h1:FJhZWVWBCcgAF8jbep7pxQ1QUsjzTwa9tvEXGw2TDRo=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5 h1:kFfb+NMap4R7nDvBYyABa/nw7KFMtAfygD1Hyoxh4uE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.30.5/go.mod h1:Dze3kNt4T+Dgb8YCfuIFSBLmE6hadKNxqfdF0Xmqz1I=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.4 h1:qJdM48OOLl1FBSzI7ZrA1ZfLwOyCYqkXV5lko1hYDBw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 h1:YRkWXQveFb0tFC0TLktmmhGsOcCgLwvq88MC2al47AA=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc=
-github.com/aws/aws-sdk-go-v2/service/sts v1.18.5 h1:L1600eLr0YvTT7gNh3Ni24yGI7NSHkq9Gp62vijPRCs=
-github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s=
-github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
-github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/KarpelesLab/reflink v1.0.1 h1:d+tdjliwOCqvub9bl0Y02GxahWkNqejNb3TZTTUcQWA=
+github.com/KarpelesLab/reflink v1.0.1/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/Microsoft/hcsshim v0.12.9 h1:2zJy5KA+l0loz1HzEGqyNnjd3fyZA31ZBCGKacp6lLg=
+github.com/Microsoft/hcsshim v0.12.9/go.mod h1:fJ0gkFAna6ukt0bLdKB8djt4XIJhF/vEPuoIWYVvZ8Y=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
+github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
+github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
+github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM=
+github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
+github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.8 h1:u1KOU1S15ufyZqmH/rA3POkiRH6EcDANHj2xHRzq+zc=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.8/go.mod h1:WPv2FRnkIOoDv/8j2gSUsI4qDc7392w5anFB/I89GZ8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 h1:sZXIzO38GZOU+O0C+INqbH7C2yALwfMWpd64tONS/NE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
+github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
+github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
-github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y=
+github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
-github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
-github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/containerd/containerd v1.7.0-rc.1 h1:Picl2mIO3fwX92iptM308gfLQ2LlxMBtulG/CrH0YC0=
-github.com/containerd/containerd v1.7.0-rc.1/go.mod h1:7MbZqMXy9721b34Q0h8E8prpPN55Zu3SKs6K4jTxYFQ=
-github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg=
-github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM=
+github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=
+github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0=
+github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0=
+github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc=
+github.com/containerd/containerd/v2 v2.0.2 h1:GmH/tRBlTvrXOLwSpWE2vNAm8+MqI6nmxKpKBNKY8Wc=
+github.com/containerd/containerd/v2 v2.0.2/go.mod h1:wIqEvQ/6cyPFUGJ5yMFanspPabMLor+bF865OHvNTTI=
+github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII=
+github.com/containerd/continuity v0.4.4/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
 github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY=
 github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o=
-github.com/containerd/nri v0.3.0 h1:2ZM4WImye1ypSnE7COjOvPAiLv84kaPILBDvb1tbDK8=
-github.com/containerd/nri v0.3.0/go.mod h1:Zw9q2lP16sdg0zYybemZ9yTDy8g7fPCIB3KXOGlggXI=
-github.com/containerd/stargz-snapshotter v0.14.2 h1:xEBHkd8Q6l/fuBRYinPSe+KQ/UjWQw83KyXXlL43nLc=
-github.com/containerd/stargz-snapshotter v0.14.2/go.mod h1:3mirLRiFYyLuWRgreICKSq3KAv0RiMgYjy6G3Xmfva0=
-github.com/containerd/stargz-snapshotter/estargz v0.14.2 h1:Zxgfb4mpH2l5nWsIF5fdlIUCVfnz660QfP9A2VnNZW0=
-github.com/containerd/stargz-snapshotter/estargz v0.14.2/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
-github.com/containerd/ttrpc v1.1.1-0.20230127163717-32fab2374638 h1:8eVDsirmav+3F0gKY9q/NboZuAqLg++AzW5/HZKKUY4=
-github.com/containerd/ttrpc v1.1.1-0.20230127163717-32fab2374638/go.mod h1:YYyNVhZrTMiaf51Vj6WhAJqJw+vl/nzABhj8pWrzle4=
-github.com/containerd/typeurl/v2 v2.1.0 h1:yNAhJvbNEANt7ck48IlEGOxP7YAp6LLpGn5jZACDNIE=
-github.com/containerd/typeurl/v2 v2.1.0/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/nri v0.8.0 h1:n1S753B9lX8RFrHYeSgwVvS1yaUcHjxbB+f+xzEncRI=
+github.com/containerd/nri v0.8.0/go.mod h1:uSkgBrCdEtAiEz4vnrq8gmAC4EnVAM5Klt0OuK5rZYQ=
+github.com/containerd/platforms v1.0.0-rc.1 h1:83KIq4yy1erSRgOVHNk1HYdPvzdJ5CnsWaRoJX4C41E=
+github.com/containerd/platforms v1.0.0-rc.1/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4=
+github.com/containerd/plugin v1.0.0 h1:c8Kf1TNl6+e2TtMHZt+39yAPDbouRH9WAToRjex483Y=
+github.com/containerd/plugin v1.0.0/go.mod h1:hQfJe5nmWfImiqT1q8Si3jLv3ynMUIBB47bQ+KexvO8=
+github.com/containerd/stargz-snapshotter v0.15.2-0.20240709063920-1dac5ef89319 h1:Td/dlhRp/kIk9W1rjXHSL87zZZiBQaKPV18OnoEREUA=
+github.com/containerd/stargz-snapshotter v0.15.2-0.20240709063920-1dac5ef89319/go.mod h1:dgo5lVziOOnWX8SxxHqYuc8ShsQou54eKLdahxFlHVc=
+github.com/containerd/stargz-snapshotter/estargz v0.15.2-0.20240709063920-1dac5ef89319 h1:BRxgmkGWi5vAvajiCwEK+xit4FeFU3GRjbiX4DKTLtM=
+github.com/containerd/stargz-snapshotter/estargz v0.15.2-0.20240709063920-1dac5ef89319/go.mod h1:9WSor0wu2swhtYoFkrjy3GHt7aNgKR2A7FhnpP+CH5o=
+github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ=
+github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
+github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40=
+github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk=
+github.com/containers/ocicrypt v1.2.0 h1:X14EgRK3xNFvJEfI5O4Qn4T3E25ANudSOZz/sirVuPM=
+github.com/containers/ocicrypt v1.2.0/go.mod h1:ZNviigQajtdlxIZGibvblVuIFBKIuUI2M0QM12SD31U=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
+github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
-github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM=
-github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
-github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY=
-github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/cli v27.1.0+incompatible h1:P0KSYmPtNbmx59wHZvG6+rjivhKDRA1BvvWM0f5DgHc=
+github.com/docker/cli v27.1.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
+github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
 github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
-github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
-github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
-github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ=
-github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
-github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
-github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
-github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=
+github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db h1:StM6A9LvaVrFS2chAGcfRVDoBB6rHYPIGJ3GknpB25c=
+github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db/go.mod h1:pwuQfHWn6j2Fpl2AWw/bPLlKfojHxIIEa5TeKIgDFW4=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
+github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
 github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
 github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
 github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
-github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
 github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
+github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -143,101 +158,93 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0=
-github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.13.0 h1:y1C7Z3e149OJbOPDBxLYR8ITPz8dTKqQwjErKVHJC8k=
-github.com/google/go-containerregistry v0.13.0/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-containerregistry v0.20.1 h1:eTgx9QNYugV4DN5mz4U8hiAGTi1ybXn0TPi4Smd8du0=
+github.com/google/go-containerregistry v0.20.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM=
+github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM=
-github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
-github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
-github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
+github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
-github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
-github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
-github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
-github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
-github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
-github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI=
-github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
+github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg=
+github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
+github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0=
+github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8=
+github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo=
+github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
+github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -248,164 +255,150 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwd
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/onsi/ginkgo/v2 v2.5.0 h1:TRtrvv2vdQqzkwrQ1ke6vtXf7IK34RBUJafIy1wMwls=
-github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg=
+github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0=
+github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA=
+github.com/onsi/gomega v1.34.0 h1:eSSPsPNp6ZpsG8X1OVmOTxig+CblTc4AxpPBykhe2Os=
+github.com/onsi/gomega v1.34.0/go.mod h1:MIKI8c+f+QLWk+hxbePD4i0LMJSExPaZOVfkoex4cAo=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8=
-github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
-github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg=
-github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
-github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.1.0-rc.1 h1:wHa9jroFfKGQqFHj0I1fMRKLl0pfj+ynAqBxo3v6u9w=
-github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/opencontainers/selinux v1.10.2 h1:NFy2xCsjn7+WspbfZkUd5zyVeisV7VFbPSP96+8/ha4=
-github.com/opencontainers/selinux v1.10.2/go.mod h1:cARutUbaUrlRClyvxOICCgKixCs6L05aUsohzA3EkHQ=
+github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
+github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk=
+github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
+github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jDMcgULaH8=
+github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
-github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/common v0.41.0 h1:npo01n6vUlRViIj5fgwiK8vlNIh8bnoxqh3gypKsyAw=
-github.com/prometheus/common v0.41.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
-github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw=
+github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8=
-github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
-github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
-github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
+github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
+github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts=
+github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
-github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
-go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
+go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
+go.mozilla.org/pkcs7 v0.9.0 h1:yM4/HS9dYv7ri2biPtxt8ikvB37a980dg69/pKmS+eI=
+go.mozilla.org/pkcs7 v0.9.0/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
-go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
-go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
-go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM=
+go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
+go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
+go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
+go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
+go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
+go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
-golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
-golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
+golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
+golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
+golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -416,35 +409,26 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
+golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 h1:QQF+HdiI4iocoxUjjpLgvTYDHKm99C/VtTBFnfiCJos=
-google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
-google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
+google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
+google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -454,51 +438,49 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
-gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
+gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY=
+gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.26.2 h1:dM3cinp3PGB6asOySalOZxEG4CZ0IAdJsrYZXE/ovGQ=
-k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU=
-k8s.io/apimachinery v0.26.2 h1:da1u3D5wfR5u2RpLhE/ZtZS2P7QvDgLZTi9wrNZl/tQ=
-k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I=
-k8s.io/client-go v0.26.2 h1:s1WkVujHX3kTp4Zn4yGNFK+dlDXy1bAAkIl+cFAiuYI=
-k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU=
-k8s.io/cri-api v0.27.0-alpha.3 h1:RT3W8OnLFpk9MUod99GwYeNPnzhEqVH0aQE8zBKZ9ds=
-k8s.io/cri-api v0.27.0-alpha.3/go.mod h1:dwMiSnrLiMmrAsTo/fObV8+efaoI9hHa1IlL++hdDIs=
-k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw=
-k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU=
-k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY=
-k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk=
-k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0=
+k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk=
+k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
+k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc=
+k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs=
+k8s.io/cri-api v0.31.2 h1:O/weUnSHvM59nTio0unxIUFyRHMRKkYn96YDILSQKmo=
+k8s.io/cri-api v0.31.2/go.mod h1:Po3TMAYH/+KrZabi7QiwQI4a692oZcUOUThd/rqwxrI=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+tags.cncf.io/container-device-interface v0.8.0 h1:8bCFo/g9WODjWx3m6EYl3GfUG31eKJbaggyBDxEldRc=
+tags.cncf.io/container-device-interface v0.8.0/go.mod h1:Apb7N4VdILW0EVdEMRYXIDVRZfNJZ+kmEUss2kRRQ6Y=
+tags.cncf.io/container-device-interface/specs-go v0.8.0 h1:QYGFzGxvYK/ZLMrjhvY0RjpUavIn4KcmRmVP/JjdBTA=
+tags.cncf.io/container-device-interface/specs-go v0.8.0/go.mod h1:BhJIkjjPh4qpys+qm4DAYtUyryaTDg9zris+AczXyws=

integration/Dockerfile

@@ -1,21 +1,23 @@
-
-ARG CONTAINERD_VER=1.6.15
+ARG CONTAINERD_VER=2.0.0-rc.3
 ARG CONTAINERD_PROJECT=/containerd
-ARG RUNC_VERSION=1.1.4
+ARG RUNC_VER=1.1.4
 ARG NYDUS_SNAPSHOTTER_PROJECT=/nydus-snapshotter
 ARG DOWNLOADS_MIRROR="https://github.com"
-ARG NYDUS_VER=2.1.4
-ARG NERDCTL_VER=1.0.0
-
-FROM golang:1.19.6-bullseye AS golang-base
+ARG NYDUS_VER=2.3.0
+ARG NERDCTL_VER=1.7.6
+ARG DELVE_VER=1.23.0
+ARG GO_VER=1.22.5-bookworm
 
+FROM golang:$GO_VER AS golang-base
+ARG TARGETARCH
 ARG CONTAINERD_VER
 ARG CONTAINERD_PROJECT
-ARG RUNC_VERSION
+ARG RUNC_VER
 ARG NYDUS_SNAPSHOTTER_PROJECT
 ARG DOWNLOADS_MIRROR
 ARG NYDUS_VER
 ARG NERDCTL_VER
+ARG DELVE_VER
 
 # RUN echo '\
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye main contrib non-free\n\
@@ -23,33 +25,31 @@ ARG NERDCTL_VER
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-backports main contrib non-free\n\
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bullseye-security main contrib non-free\n' > /etc/apt/sources.list
 
-ENV GOPROXY=https://goproxy.cn
+RUN apt-get update -qq && apt-get install -qq libbtrfs-dev libseccomp-dev sudo psmisc jq lsof net-tools
 
-RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev sudo psmisc jq lsof net-tools
-
-RUN go install github.com/go-delve/delve/cmd/dlv@latest
+RUN go install github.com/go-delve/delve/cmd/dlv@v"$DELVE_VER"
 
 # Install containerd
-RUN wget ${DOWNLOADS_MIRROR}/containerd/containerd/releases/download/v${CONTAINERD_VER}/containerd-${CONTAINERD_VER}-linux-amd64.tar.gz && \
-tar xzf containerd-${CONTAINERD_VER}-linux-amd64.tar.gz && \
-install -D -m 755 bin/* /usr/local/bin/
+RUN wget -q "$DOWNLOADS_MIRROR"/containerd/containerd/releases/download/v"$CONTAINERD_VER"/containerd-"$CONTAINERD_VER"-linux-"$TARGETARCH".tar.gz && \
+  tar xzf containerd-"$CONTAINERD_VER"-linux-"$TARGETARCH".tar.gz && \
+  install -D -m 755 bin/* /usr/local/bin/
 COPY misc/example/containerd-config.toml /etc/containerd/config.toml
 
 # Install runc
-RUN wget ${DOWNLOADS_MIRROR}/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.amd64 && \
-install -D -m 755 runc.amd64 /usr/local/bin/runc
+RUN wget -q "$DOWNLOADS_MIRROR"/opencontainers/runc/releases/download/v"$RUNC_VER"/runc."$TARGETARCH" && \
+  install -D -m 755 runc."$TARGETARCH" /usr/local/bin/runc
 
 # Install nydusd nydus-image
-RUN  wget ${DOWNLOADS_MIRROR}/dragonflyoss/image-service/releases/download/v${NYDUS_VER}/nydus-static-v${NYDUS_VER}-linux-amd64.tgz && \
-tar xzf nydus-static-v${NYDUS_VER}-linux-amd64.tgz && \
-install -D -m 755 nydus-static/nydusd /usr/local/bin/nydusd && \
-install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image && \
-install -D -m 755 nydus-static/nydusctl /usr/local/bin/nydusctl
+RUN  wget -q "$DOWNLOADS_MIRROR"/dragonflyoss/nydus/releases/download/v"$NYDUS_VER"/nydus-static-v"$NYDUS_VER"-linux-"$TARGETARCH".tgz && \
+  tar xzf nydus-static-v"$NYDUS_VER"-linux-"$TARGETARCH".tgz && \
+  install -D -m 755 nydus-static/nydusd /usr/local/bin/nydusd && \
+  install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image && \
+  install -D -m 755 nydus-static/nydusctl /usr/local/bin/nydusctl
 
 # Install nerdctl
-RUN wget ${DOWNLOADS_MIRROR}/containerd/nerdctl/releases/download/v${NERDCTL_VER}/nerdctl-${NERDCTL_VER}-linux-amd64.tar.gz && \
-tar xzf nerdctl-${NERDCTL_VER}-linux-amd64.tar.gz && \
-install -D -m 755 nerdctl /usr/local/bin/nerdctl
+RUN wget -q "$DOWNLOADS_MIRROR"/containerd/nerdctl/releases/download/v"$NERDCTL_VER"/nerdctl-"$NERDCTL_VER"-linux-"$TARGETARCH".tar.gz && \
+  tar xzf nerdctl-"$NERDCTL_VER"-linux-"$TARGETARCH".tar.gz && \
+  install -D -m 755 nerdctl /usr/local/bin/nerdctl
 
 # Install fscache driver configuration file
 COPY misc/snapshotter/nydusd-config.fscache.json /etc/nydus/nydusd-config.fscache.json
@@ -63,6 +63,8 @@ COPY integration/entrypoint.sh /
 WORKDIR /nydus-snapshotter
 
 ENV PATH="${PATH}:/usr/local/bin/"
-ENV GO111MODULE=on
 
-ENTRYPOINT [ "/bin/bash",  "-c",  "make install && /entrypoint.sh"]
+# Prevent git from complaining about ownership
+RUN git config --global --add safe.directory /nydus-snapshotter
+
+ENTRYPOINT [ "/bin/bash",  "-c",  "make install && /entrypoint.sh"]

integration/entrypoint.sh

@@ -13,7 +13,7 @@ SNAPSHOTTER_CONFIG=/etc/nydus/config.toml
 CONTAINERD_ROOT=/var/lib/containerd/
 CONTAINERD_STATUS=/run/containerd/
 REMOTE_SNAPSHOTTER_SOCKET=/run/containerd-nydus/containerd-nydus-grpc.sock
-REMOTE_SNAPSHOTTER_ROOT=/var/lib/containerd-nydus
+REMOTE_SNAPSHOTTER_ROOT=/var/lib/containerd/io.containerd.snapshotter.v1.nydus
 CONTAINERD_SOCKET=/run/containerd/containerd.sock
 SNAPSHOTTER_SHARED_MNT=${REMOTE_SNAPSHOTTER_ROOT}/mnt
 SNAPSHOTTER_CACHE_DIR=${REMOTE_SNAPSHOTTER_ROOT}/cache
@@ -23,7 +23,7 @@ WORDPRESS_IMAGE=${WORDPRESS_IMAGE:-ghcr.io/dragonflyoss/image-service/wordpress:
 TOMCAT_IMAGE=${TOMCAT_IMAGE:-ghcr.io/dragonflyoss/image-service/tomcat:nydus-nightly-v5}
 STARGZ_IMAGE=${STARGZ_IMAGE:-ghcr.io/stargz-containers/wordpress:5.9.2-esgz}
 REDIS_OCI_IMAGE=${REDIS_OCI_IMAGE:-ghcr.io/stargz-containers/redis:6.2.6-org}
-WORDPRESS_OCI_IMAGE=${WORDPRESS_OCI_IMAGE:-ghcr.io/stargz-containers/wordpress:5.9.2-org}
+WORDPRESS_OCI_IMAGE=${WORDPRESS_OCI_IMAGE:-ghcr.io/dragonflyoss/image-service/wordpress:latest}
 
 PLUGIN=nydus
 
@@ -102,8 +102,9 @@ function retry {
 }
 
 function can_erofs_ondemand_read {
-    grep 'CONFIG_EROFS_FS_ONDEMAND=[ym]' /usr/src/linux-headers-"$(uname -r)"/.config 1>/dev/null
-    echo $?
+    return 1
+    # grep 'CONFIG_EROFS_FS_ONDEMAND=[ym]' /usr/src/linux-headers-"$(uname -r)"/.config 1>/dev/null
+    # echo $?
 }
 
 function validate_mnt_number {
@@ -117,13 +118,21 @@ function validate_mnt_number {
     fi
 }
 
+function set_config_option {
+    KEY="${1}"
+    VALUE="${2}"
+
+    sed -i "s/\($KEY *= *\).*/\1$VALUE/" "${SNAPSHOTTER_CONFIG}"
+}
+
 function set_recover_policy {
     policy="${1}"
 
-    TARGET_KEY="recover_policy"
-    REPLACEMENT_VALUE=\"${policy}\"
+    set_config_option "recover_policy" \"${policy}\"
+}
 
-    sed -i "s/\($TARGET_KEY *= *\).*/\1$REPLACEMENT_VALUE/" "${SNAPSHOTTER_CONFIG}"
+function set_enable_referrer_detect {
+    set_config_option "enable_referrer_detect" "true"
 }
 
 function reboot_containerd {
@@ -133,7 +142,7 @@ function reboot_containerd {
     killall "nydusd" || true
 
     # Let snapshotter shutdown all its services.
-    sleep 0.5
+    sleep 2
 
     # FIXME
     echo "umount globally shared mountpoint"
@@ -258,7 +267,7 @@ function start_single_container_on_stargz {
     reboot_containerd multiple
 
     killall "containerd-nydus-grpc" || true
-    sleep 0.5
+    sleep 2
 
     containerd-nydus-grpc --enable-stargz --daemon-mode multiple --fs-driver fusedev \
         --recover-policy none --log-to-stdout --config-path /etc/nydus/nydusd-config.json &
@@ -283,6 +292,17 @@ function start_container_on_oci {
     nerdctl image rm --force "${WORDPRESS_OCI_IMAGE}"
 }
 
+function start_container_with_referrer_detect {
+    echo "testing $FUNCNAME"
+    nerdctl_prune_images
+    reboot_containerd multiple
+
+    set_enable_referrer_detect
+    nerdctl --snapshotter nydus run -d --net none "${WORDPRESS_OCI_IMAGE}"
+
+    detect_go_race
+}
+
 function pull_remove_one_image {
     echo "testing $FUNCNAME"
     nerdctl_prune_images
@@ -412,7 +432,6 @@ function fscache_kill_nydusd_failover() {
     c1=$(nerdctl --snapshotter nydus create --net none "${JAVA_IMAGE}")
     c2=$(nerdctl --snapshotter nydus create --net none "${WORDPRESS_IMAGE}")
 
-
     killall -9 nydusd
 
     echo "start new containers"
@@ -544,39 +563,18 @@ function kill_multiple_nydusd_recover_failover {
     detect_go_race
 }
 
-function blob_manager_pull_preheat {
-    local daemon_mode=$1
-    echo "testing $FUNCNAME"
-    nerdctl_prune_images
-
-    # Don't forget unset me.
-    NYDUS_CONFIG_PATH=${FUSE_NYDUSD_LOCALFS_CONFIG}
-
-    reboot_containerd "${daemon_mode}" fusedev failover
-
-    nerdctl --snapshotter nydus image pull "${JAVA_IMAGE}"
-    nerdctl --snapshotter nydus image pull "${WORDPRESS_IMAGE}"
-    nerdctl --snapshotter nydus image pull "${TOMCAT_IMAGE}"
-
-    pause 1
-
-    is_cache_cleared && true
-    validate_mnt_number 0
-
-    nerdctl --snapshotter nydus image rm --force "${JAVA_IMAGE}"
-    nerdctl --snapshotter nydus image rm --force "${WORDPRESS_IMAGE}"
-
-    # Deleteing with flag --async as a fuzzer
-    nerdctl --snapshotter nydus image rm --force --async "${TOMCAT_IMAGE}"
-    nerdctl --snapshotter nydus image pull "${TOMCAT_IMAGE}"
-
-    is_cache_cleared || true
-
-    detect_go_race
-
-    unset NYDUS_CONFIG_PATH
+# Refer to https://github.com/moby/moby/blob/088afc99e4bf8adb78e29733396182417d67ada2/hack/dind#L28-L38
+function enable_nesting_for_cgroup_v2() {
+    if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+        mkdir -p /sys/fs/cgroup/init
+        xargs -rn1 </sys/fs/cgroup/cgroup.procs >/sys/fs/cgroup/init/cgroup.procs || :
+        sed -e 's/ / +/g' -e 's/^/-/' </sys/fs/cgroup/cgroup.controllers \
+            >/sys/fs/cgroup/cgroup.subtree_control
+    fi
 }
 
+enable_nesting_for_cgroup_v2
+
 reboot_containerd multiple
 
 start_single_container_multiple_daemons
@@ -596,12 +594,9 @@ only_restart_snapshotter multiple
 kill_snapshotter_and_nydusd_recover shared
 kill_snapshotter_and_nydusd_recover multiple
 
-
 ctr_snapshot_usage multiple
 ctr_snapshot_usage shared
 
-blob_manager_pull_preheat multiple fusedev
-
 if [[ $(can_erofs_ondemand_read) == 0 ]]; then
     kill_multiple_nydusd_recover_failover multiple
     kill_multiple_nydusd_recover_failover shared
@@ -612,3 +607,5 @@ if [[ $(can_erofs_ondemand_read) == 0 ]]; then
 fi
 
 start_container_on_oci
+
+start_container_with_referrer_detect

internal/constant/values.go

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// constants of nydus snapshotter CLI config
+
+package constant
+
+const (
+	DaemonModeMultiple  string = "multiple"
+	DaemonModeDedicated string = "dedicated"
+	DaemonModeShared    string = "shared"
+	DaemonModeNone      string = "none"
+	DaemonModeInvalid   string = ""
+)
+
+const (
+	// Mount RAFS filesystem by using EROFS over block devices.
+	FsDriverBlockdev string = "blockdev"
+	// Mount RAFS filesystem by using FUSE subsystem
+	FsDriverFusedev string = "fusedev"
+	// Mount RAFS filesystem by using fscache/EROFS.
+	FsDriverFscache string = "fscache"
+	// Only prepare/supply meta/data blobs, do not mount RAFS filesystem.
+	FsDriverNodev string = "nodev"
+	// Relay layer content download operation to other agents.
+	FsDriverProxy string = "proxy"
+)
+
+const (
+	DefaultDaemonMode string = DaemonModeMultiple
+
+	DefaultFsDriver string = FsDriverFusedev
+
+	DefaultLogLevel string = "info"
+	DefaultGCPeriod string = "24h"
+
+	DefaultNydusDaemonConfigPath string = "/etc/nydus/nydusd-config.json"
+	NydusdBinaryName             string = "nydusd"
+	NydusImageBinaryName         string = "nydus-image"
+
+	DefaultRootDir                 = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+	DefaultAddress                 = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+	DefaultSystemControllerAddress = "/run/containerd-nydus/system.sock"
+
+	// Log rotation
+	DefaultDaemonRotateLogMaxSize = 100 // 100 megabytes
+	DefaultRotateLogMaxSize       = 200 // 200 megabytes
+	DefaultRotateLogMaxBackups    = 5
+	DefaultRotateLogMaxAge        = 0 // days
+	DefaultRotateLogLocalTime     = true
+	DefaultRotateLogCompress      = true
+)

internal/flags/flags.go

@@ -8,19 +8,21 @@
 package flags
 
 import (
+	"github.com/containerd/nydus-snapshotter/internal/constant"
 	"github.com/urfave/cli/v2"
 )
 
 type Args struct {
 	Address               string
-	LogLevel              string
 	NydusdConfigPath      string
 	SnapshotterConfigPath string
 	RootDir               string
 	NydusdPath            string
 	NydusImagePath        string
+	NydusOverlayFSPath    string
 	DaemonMode            string
 	FsDriver              string
+	LogLevel              string
 	LogToStdout           bool
 	LogToStdoutCount      int
 	PrintVersion          bool
@@ -33,63 +35,74 @@ type Flags struct {
 
 func buildFlags(args *Args) []cli.Flag {
 	return []cli.Flag{
-		&cli.BoolFlag{
-			Name:        "version",
-			Usage:       "print version and build information",
-			Destination: &args.PrintVersion,
-		},
 		&cli.StringFlag{
 			Name:        "root",
-			Usage:       "the directory storing snapshotter working states",
+			Usage:       "directory to store snapshotter data and working states",
 			Destination: &args.RootDir,
+			DefaultText: constant.DefaultRootDir,
 		},
 		&cli.StringFlag{
 			Name:        "address",
-			Usage:       "gRPC socket path",
+			Usage:       "remote snapshotter gRPC socket path",
 			Destination: &args.Address,
+			DefaultText: constant.DefaultAddress,
 		},
 		&cli.StringFlag{
 			Name:        "config",
-			Usage:       "path to the nydus-snapshotter configuration",
+			Usage:       "path to nydus-snapshotter configuration (such as: config.toml)",
 			Destination: &args.SnapshotterConfigPath,
 		},
-		&cli.StringFlag{
-			Name:        "nydusd-config",
-			Aliases:     []string{"config-path"},
-			Usage:       "path to the nydusd configuration",
-			Destination: &args.NydusdConfigPath,
-		},
-		&cli.StringFlag{
-			Name:        "daemon-mode",
-			Usage:       "spawning nydusd daemon mode, legal values include \"multiple\", \"shared\" or \"none\"",
-			Destination: &args.DaemonMode,
-		},
-		&cli.StringFlag{
-			Name:        "fs-driver",
-			Usage:       "fulfill image service based on what fs driver, possible values include \"fusedev\", \"fscache\"",
-			Destination: &args.FsDriver,
-		},
-		&cli.StringFlag{
-			Name:        "log-level",
-			Usage:       "logging level, possible values \"trace\", \"debug\", \"info\", \"warn\", \"error\"",
-			Destination: &args.LogLevel,
-		},
-		&cli.BoolFlag{
-			Name:        "log-to-stdout",
-			Usage:       "print log messages to STDOUT",
-			Destination: &args.LogToStdout,
-			Count:       &args.LogToStdoutCount,
-		},
 		&cli.StringFlag{
 			Name:        "nydus-image",
-			Usage:       "`nydus-image` binary path, it will be searched from $PATH if this option is not provided",
+			Usage:       "path to `nydus-image` binary, default to search in $PATH (such as: /usr/local/bin/nydus-image)",
 			Destination: &args.NydusImagePath,
 		},
 		&cli.StringFlag{
 			Name:        "nydusd",
-			Usage:       "`nydusd` binary path, it will be searched if this option is not provided",
+			Usage:       "path to `nydusd` binary, default to search in $PATH (such as: /usr/local/bin/nydusd)",
 			Destination: &args.NydusdPath,
 		},
+		&cli.StringFlag{
+			Name:        "nydusd-config",
+			Aliases:     []string{"config-path"},
+			Usage:       "path to nydusd configuration (such as: nydusd-config.json or nydusd-config-v2.toml)",
+			Destination: &args.NydusdConfigPath,
+			DefaultText: constant.DefaultNydusDaemonConfigPath,
+		},
+		&cli.StringFlag{
+			Name:        "nydus-overlayfs-path",
+			Usage:       "path of nydus-overlayfs or name of binary from $PATH, defaults to 'nydus-overlayfs'",
+			Destination: &args.NydusOverlayFSPath,
+		},
+		&cli.StringFlag{
+			Name:        "daemon-mode",
+			Usage:       "nydusd daemon working mode, possible values: \"dedicated\", \"multiple\", \"shared\" or \"none\". \"multiple\" is an alias of \"dedicated\" and will be deprecated in v1.0",
+			Destination: &args.DaemonMode,
+			DefaultText: constant.DaemonModeMultiple,
+		},
+		&cli.StringFlag{
+			Name:        "fs-driver",
+			Usage:       "driver to mount RAFS filesystem, possible values: \"fusedev\", \"fscache\"",
+			Destination: &args.FsDriver,
+			DefaultText: constant.FsDriverFusedev,
+		},
+		&cli.StringFlag{
+			Name:        "log-level",
+			Usage:       "logging level, possible values: \"trace\", \"debug\", \"info\", \"warn\", \"error\"",
+			Destination: &args.LogLevel,
+			DefaultText: constant.DefaultLogLevel,
+		},
+		&cli.BoolFlag{
+			Name:        "log-to-stdout",
+			Usage:       "print log messages to standard output",
+			Destination: &args.LogToStdout,
+			Count:       &args.LogToStdoutCount,
+		},
+		&cli.BoolFlag{
+			Name:        "version",
+			Usage:       "print version and build information",
+			Destination: &args.PrintVersion,
+		},
 	}
 }
 

internal/logging/setup.go

@@ -12,7 +12,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	lumberjack "gopkg.in/natefinch/lumberjack.v2"

internal/logging/setup_test.go

@@ -13,7 +13,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/sirupsen/logrus"
 	"gotest.tools/assert"
 )
@@ -25,7 +25,7 @@ const (
 
 func GetRotateLogFileNumbers(testLogDir string, suffix string) int {
 	i := 0
-	err := filepath.Walk(testLogDir, func(fname string, fi os.FileInfo, err error) error {
+	err := filepath.Walk(testLogDir, func(fname string, fi os.FileInfo, _ error) error {
 		if !fi.IsDir() && strings.HasSuffix(fname, suffix) {
 			i++
 		}

misc/example/README.md

@@ -26,8 +26,8 @@ sudo tar xzf cni-plugins-linux-amd64-$CNI_VERSION.tgz -C /opt/cni/bin/
 wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz
 tar xzf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin/
 # install nydus-overlayfs
-NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//')
-wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
+NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//')
+wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
 tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz
 sudo cp nydus-static/nydus-overlayfs /usr/local/sbin/
 ```

misc/example/containerd-config-v1.toml

@@ -1,30 +0,0 @@
-root = "/var/lib/containerd"
-state = "/run/containerd"
-oom_score = 0
-
-[grpc]
-  address = "/run/containerd/containerd.sock"
-
-[debug]
-  level = "debug"
-
-[plugins]
-  [plugins.cgroups]
-    no_prometheus = false
-  [plugins.cri]
-    [plugins.cri.containerd]
-      snapshotter = "nydus"
-      disable_snapshot_annotations = false
-    [plugins.cri.containerd.runtimes]
-      [plugins.cri.containerd.runtimes.kata]
-         runtime_type = "io.containerd.kata.v2"
-	 privileged_without_host_devices = true
-    [plugins.cri.cni]
-      bin_dir = "/opt/cni/bin"
-      conf_dir = "/etc/cni/net.d"
-
-[proxy_plugins]
-  [proxy_plugins.nydus]
-      type = "snapshot"
-      address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
-

misc/example/optimizer-nri-plugin.conf

@@ -1,5 +1,7 @@
 # The directory to persist accessed files list for container.
 persist_dir = "/opt/nri/optimizer/results"
+# Whether to make the csv file human readable.
+readable = false
 # The path of optimizer server binary.
 server_path = "/usr/local/bin/optimizer-server"
 # The timeout to kill optimizer server, 0 to disable it.

misc/nri-prefetch/prefetchConfig.toml

@@ -0,0 +1,3 @@
+[file_prefetch]
+# This is used to configure the socket address for the file prefetch.
+socket_address = "/run/containerd-nydus/system.sock"

misc/optimizer/containerd-config.toml

@@ -0,0 +1,26 @@
+#   Copyright 2018-2022 Docker Inc.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+version = 2
+
+[proxy_plugins]
+  [proxy_plugins.nydus]
+    type = "snapshot"
+    address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+
+[plugins."io.containerd.nri.v1.nri"]
+  config_file = "/etc/nri/nri.conf"
+  disable = false
+  plugin_path = "/opt/nri/plugins"
+  socket_path = "/var/run/nri.sock"

misc/optimizer/crictl.yaml

@@ -0,0 +1,5 @@
+runtime-endpoint: unix:///run/containerd/containerd.sock
+image-endpoint: unix:////run/containerd/containerd.sock
+timeout: 2
+debug: false
+pull-image-on-create: false

misc/optimizer/nginx.yaml

@@ -0,0 +1,18 @@
+metadata:
+  name: nginx
+
+image:
+  image: nginx:1.23.3
+
+mounts:
+  - host_path: script
+    container_path: /script
+
+command:
+  - /script/entrypoint.sh
+
+args:
+ - /script/file_list.txt
+
+log_path: nginx.0.log
+linux: {}

misc/optimizer/sandbox.yaml

@@ -0,0 +1,7 @@
+metadata:
+  name: nginx-sandbox
+  namespace: default
+  attempt: 1
+  uid: hdishd83djaidwnduwk28bcsb
+log_directory: /tmp
+linux: {}

misc/optimizer/script/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+path=$1
+
+default_path=file_list_path.txt
+if [[ $# -eq 0 ]]; then
+    path=${default_path}
+fi
+
+files=($(cat ${path} | tr "\n" " "))
+files_number=${#files[@]}
+echo "file number: $files_number"
+
+for file in "${files[@]}"; do
+    file_size=$(stat -c%s "${file}")
+    echo "file: ${file} size: ${file_size}"
+    cat ${file} >/dev/null
+done
+
+echo "Read file list done."

misc/optimizer/script/file_list.txt

@@ -0,0 +1,12 @@
+/lib/x86_64-linux-gnu/ld-2.31.so
+/lib/x86_64-linux-gnu/libc-2.31.so
+/lib/x86_64-linux-gnu/libtinfo.so.6.2
+/lib/x86_64-linux-gnu/libdl-2.31.so
+/lib/x86_64-linux-gnu/libnss_files-2.31.so
+/lib/x86_64-linux-gnu/libselinux.so.1
+/usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.10.1
+/lib/x86_64-linux-gnu/libpthread-2.31.so
+/docker-entrypoint.sh
+/docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
+/docker-entrypoint.d/20-envsubst-on-templates.sh
+/docker-entrypoint.d/30-tune-worker-processes.sh

misc/snapshotter/Dockerfile

@@ -1,31 +1,46 @@
-FROM alpine:3.17.0 AS sourcer
+FROM alpine:3.17.0 AS base
 
-ARG NYDUS_VER=v2.1.2
+FROM base AS sourcer
+ARG TARGETARCH
+ARG NYDUS_VER=v2.3.0
 
-RUN apk add --no-cache curl
-RUN apk add --no-cache --upgrade grep
-RUN wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz && \
+RUN apk add -q --no-cache curl && \
+    apk add -q --no-cache --upgrade grep && \
+    curl -fsSL -O https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz && \
     echo $NYDUS_VER > /.nydus_version && \
     tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz && \
-    rm nydus-static-$NYDUS_VER-linux-amd64.tgz
-RUN mv nydus-static/* /
+    rm nydus-static-$NYDUS_VER-linux-amd64.tgz && \
+    mv nydus-static/* / \
+    && rm -rf /nydus-overlayfs
 
-FROM alpine:3.17.0
+FROM base AS kubectl-sourcer
+ARG TARGETARCH
+
+RUN apk add -q --no-cache curl && \
+    curl -fsSL -o /usr/bin/kubectl https://dl.k8s.io/release/"$(curl -L -s https://dl.k8s.io/release/stable.txt)"/bin/linux/"$TARGETARCH"/kubectl && \
+    chmod +x /usr/bin/kubectl
+
+FROM base
+ARG DESTINATION=/opt/nydus-artifacts
+ARG CONFIG_DESTINATION=${DESTINATION}/etc/nydus
+ARG BINARY_DESTINATION=${DESTINATION}/usr/local/bin
+ARG SCRIPT_DESTINATION=${DESTINATION}/opt/nydus
 
 WORKDIR /root/
+RUN apk add -q --no-cache libc6-compat bash
 
-RUN apk add --no-cache libc6-compat
-
-VOLUME /var/lib/containerd-nydus /run/containerd-nydus
+VOLUME /var/lib/containerd/io.containerd.snapshotter.v1.nydus
+VOLUME /run/containerd-nydus
 
 COPY --from=sourcer /.nydus_version /.nydus_version
+COPY --from=kubectl-sourcer /usr/bin/kubectl /usr/bin/kubectl
 
-RUN mkdir -p /usr/local/bin/ /etc/nydus/ /var/lib/containerd-nydus/cache /tmp/blobs/
-COPY --from=sourcer /nydus* /usr/local/bin/
-COPY containerd-nydus-grpc /usr/local/bin/
-RUN chmod +x /usr/local/bin/containerd-nydus-grpc
-COPY nydusd-config.fusedev.json /etc/nydus/config.json
-COPY nydusd-config-localfs.json /etc/nydus/localfs.json
-COPY entrypoint.sh /
-
-ENTRYPOINT ["/entrypoint.sh"]
+RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache /tmp/blobs/
+COPY --from=sourcer /nydus* ${BINARY_DESTINATION}/
+COPY --chmod=755 containerd-nydus-grpc nydus-overlayfs ${BINARY_DESTINATION}/
+COPY --chmod=755 snapshotter.sh ${SCRIPT_DESTINATION}/snapshotter.sh
+COPY nydusd-config.fusedev.json ${CONFIG_DESTINATION}/nydusd-fusedev.json
+COPY nydusd-config-localfs.json ${CONFIG_DESTINATION}/nydusd-localfs.json
+COPY nydusd-config.fscache.json ${CONFIG_DESTINATION}/nydusd-fscache.json
+COPY config.toml ${CONFIG_DESTINATION}/config.toml
+COPY nydus-snapshotter.service ${DESTINATION}/etc/systemd/system/nydus-snapshotter.service

misc/snapshotter/base/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+- nydus-snapshotter.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+

misc/snapshotter/base/nydus-snapshotter.yaml

@@ -0,0 +1,140 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
+  namespace: nydus-system
+data:
+  FS_DRIVER: "fusedev"
+  ENABLE_CONFIG_FROM_VOLUME: "false"
+  ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false"
+  ENABLE_SYSTEMD_SERVICE: "true"
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+  labels:
+    app: nydus-snapshotter
+spec:
+  selector:
+    matchLabels:
+      app: nydus-snapshotter
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        app: nydus-snapshotter
+
+    spec:
+      serviceAccountName: nydus-snapshotter-sa
+      hostNetwork: true
+      hostPID: true
+      containers:
+      - name: nydus-snapshotter
+        image: "ghcr.io/containerd/nydus-snapshotter:latest"
+        imagePullPolicy: Always
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: FS_DRIVER
+            valueFrom:
+              configMapKeyRef:
+                name: nydus-snapshotter-configs
+                key: FS_DRIVER
+                optional: true
+          - name: ENABLE_CONFIG_FROM_VOLUME
+            valueFrom:
+              configMapKeyRef:
+                name: nydus-snapshotter-configs
+                key: ENABLE_CONFIG_FROM_VOLUME
+                optional: true 
+          - name: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER
+            valueFrom:
+              configMapKeyRef:
+                name: nydus-snapshotter-configs
+                key: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER
+                optional: true
+          - name: ENABLE_SYSTEMD_SERVICE
+            valueFrom:
+              configMapKeyRef:
+                name: nydus-snapshotter-configs
+                key: ENABLE_SYSTEMD_SERVICE
+                optional: true                    
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - "bash"
+              - "-c"
+              - |
+                /opt/nydus-artifacts/opt/nydus/snapshotter.sh cleanup
+        command: 
+         - bash
+         - -c
+         - |-
+           /opt/nydus-artifacts/opt/nydus/snapshotter.sh deploy
+        volumeMounts:
+        - name: config-volume
+          mountPath: "/etc/nydus-snapshotter"
+        - name: nydus-lib
+          mountPath: "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+          mountPropagation: Bidirectional
+        - name: nydus-run
+          mountPath: "/run/containerd-nydus"
+          mountPropagation: Bidirectional
+        - name: nydus-opt
+          mountPath: "/opt/nydus"
+          mountPropagation: Bidirectional
+        - name: nydus-etc
+          mountPath: "/etc/nydus"
+          mountPropagation: Bidirectional     
+        - name: containerd-conf
+          mountPath: "/etc/containerd/"
+        - name: local-bin
+          mountPath: "/usr/local/bin/"
+        - name: etc-systemd-system
+          mountPath: "/etc/systemd/system/"
+        securityContext:
+          privileged: true
+
+      volumes:
+      - name: config-volume
+        configMap:
+          name: nydus-snapshotter-configs
+          optional: true
+      - name: nydus-run
+        hostPath:
+          path: /run/containerd-nydus
+          type: DirectoryOrCreate
+      - name: nydus-lib
+        hostPath:
+          path: /var/lib/containerd/io.containerd.snapshotter.v1.nydus
+          type: DirectoryOrCreate
+      - name: nydus-opt
+        hostPath:
+          path: /opt/nydus
+          type: DirectoryOrCreate
+      - name: nydus-etc
+        hostPath:
+          path: /etc/nydus
+          type: DirectoryOrCreate
+      - name: containerd-conf
+        hostPath:
+          path: /etc/containerd/
+      - name: local-bin
+        hostPath:
+          path: /usr/local/bin/
+      - name: etc-systemd-system
+        hostPath:
+          path: /etc/systemd/system/
+      

misc/snapshotter/config-blockdev.toml

@@ -0,0 +1,39 @@
+version = 1
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+# No nydusd daemon needed
+daemon_mode = "none"
+
+[daemon]
+# Use `blockdev` for tarfs
+fs_driver = "blockdev"
+# Path to nydus-image binary
+nydusimage_path = "/usr/local/bin/nydus-image"
+
+[remote]
+skip_ssl_verify = true
+
+[snapshot]
+# Insert Kata volume information to `Mount.Options`
+enable_kata_volume = true
+
+[experimental.tarfs]
+# Whether to enable nydus tarfs mode. Tarfs is supported by:
+# - The EROFS filesystem driver since Linux 6.4
+# - Nydus Image Service release v2.3
+enable_tarfs = true
+
+# Mount rafs on host by loopdev and EROFS
+mount_tarfs_on_host = false
+
+# Mode to export tarfs images:
+# - "none" or "": do not export tarfs
+# - "layer_verity_only": only generate disk verity information for a layer blob
+# - "image_verity_only": only generate disk verity information for all blobs of an image
+# - "layer_block": generate a raw block disk image with tarfs for a layer
+# - "image_block": generate a raw block disk image with tarfs for an image
+# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
+# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
+export_mode = "image_block_with_verity"

misc/snapshotter/config-proxy.toml

@@ -0,0 +1,15 @@
+version = 1
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+# No nydusd daemon needed
+daemon_mode = "none"
+
+[daemon]
+# Enable proxy mode
+fs_driver = "proxy"
+
+[snapshot]
+# Insert Kata volume information to `Mount.Options`
+enable_kata_volume = true

misc/snapshotter/config.toml

@@ -1,9 +1,11 @@
 version = 1
 # Snapshotter's own home directory where it stores and creates necessary resources
-root = "/var/lib/containerd-nydus"
+root = "/var/lib/containerd/io.containerd.snapshotter.v1.nydus"
 # The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
 address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
-daemon_mode = "multiple"
+# The nydus daemon mode can be one of the following options: multiple, dedicated, shared, or none. 
+# If `daemon_mode` option is not specified, the default value is multiple.
+daemon_mode = "dedicated"
 # Whether snapshotter should try to clean up resources when it is closed
 cleanup_on_close = false
 
@@ -11,7 +13,7 @@ cleanup_on_close = false
 # Snapshotter's debug and trace HTTP server interface
 enable = true
 # Unix domain socket path where system controller is listening on
-address = "/var/run/containerd-nydus/system.sock"
+address = "/run/containerd-nydus/system.sock"
 
 [system.debug]
 # Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started.
@@ -21,18 +23,28 @@ daemon_cpu_profile_duration_secs = 5
 pprof_address = ""
 
 [daemon]
-nydusd_path = "/usr/local/bin/nydusd"
-nydusimage_path = "/usr/local/bin/nydus-image"
-# fusedev or fscache
-fs_driver = "fusedev"
-# Specify nydusd log level
-log_level = "info"
-# How to process when daemon dies: "none", "restart" or "failover"
-recover_policy = "restart"
 # Specify a configuration file for nydusd
 nydusd_config = "/etc/nydus/nydusd-config.fusedev.json"
-# The fuse or fscache IO working threads started by nydusd
+nydusd_path = "/usr/local/bin/nydusd"
+nydusimage_path = "/usr/local/bin/nydus-image"
+# The fs driver can be one of the following options: fusedev, fscache, blockdev, proxy, or nodev. 
+# If `fs_driver` option is not specified, the default value is fusedev.
+fs_driver = "fusedev"
+# How to process when daemon dies: "none", "restart" or "failover"
+recover_policy = "restart"
+# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024].
+# Setting to 0 will use the default configuration of nydusd.
 threads_number = 4
+# Log rotation size for nydusd, in unit MB(megabytes). (default 100MB)
+log_rotation_size = 100
+
+[cgroup]
+# Whether to use separate cgroup for nydusd.
+enable = true
+# The memory limit for nydusd cgroup, which contains all nydusd processes.
+# Percentage is supported as well, please ensure it is end with "%".
+# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%".
+memory_limit = ""
 
 [log]
 # Print logs to stdout rather than logging files
@@ -45,7 +57,7 @@ log_rotation_local_time = true
 log_rotation_max_age = 7
 log_rotation_max_backups = 5
 # In unit MB(megabytes)
-log_rotation_max_size = 1
+log_rotation_max_size = 100
 
 [metrics]
 # Enable by assigning an address, empty indicates metrics server is disabled
@@ -58,7 +70,7 @@ convert_vpc_registry = false
 # Snapshotter will overwrite daemon's mirrors configuration
 # if the values loaded from this driectory are not null before starting a daemon.
 # Set to "" or an empty directory to disable it.
-dir = "/etc/nydus/certs.d"
+#dir = "/etc/nydus/certs.d"
 
 [remote.auth]
 # Fetch the private registry auth by listening to K8s API server
@@ -68,17 +80,24 @@ kubeconfig_path = ""
 # Fetch the private registry auth as CRI image service proxy
 enable_cri_keychain = false
 # the target image service when using image proxy
-image_service_address = ""
+#image_service_address = "/run/containerd/containerd.sock"
 
 [snapshot]
 # Let containerd use nydus-overlayfs mount helper
 enable_nydus_overlayfs = false
+# Path to the nydus-overlayfs binary or name of the binary in $PATH
+nydus_overlayfs_path = "nydus-overlayfs"
+# Insert Kata Virtual Volume option to `Mount.Options`
+enable_kata_volume = false
 # Whether to remove resources when a snapshot is removed
 sync_remove = false
 
 [cache_manager]
+# Disable or enable recyclebin
 disable = false
+# How long to keep deleted files in recyclebin
 gc_period = "24h"
+# Directory to host cached files
 cache_dir = ""
 
 [image]
@@ -87,5 +106,32 @@ validate_signature = false
 
 # The configuraions for features that are not production ready
 [experimental]
-# Whether tp enable stargz support
+# Whether to enable stargz support
 enable_stargz = false
+# Whether to enable referrers support
+# The option enables trying to fetch the Nydus image associated with the OCI image and run it.
+# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers
+enable_referrer_detect = false
+# Whether to enable authentication support
+# The option enables nydus snapshot to provide backend information to nydusd.
+enable_backend_source = false
+[experimental.tarfs]
+# Whether to enable nydus tarfs mode. Tarfs is supported by:
+# - The EROFS filesystem driver since Linux 6.4
+# - Nydus Image Service release v2.3
+enable_tarfs = false
+# Mount rafs on host by loopdev and EROFS
+mount_tarfs_on_host = false
+# Only enable nydus tarfs mode for images with `tarfs hint` label when true
+tarfs_hint = false
+# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default
+max_concurrent_proc = 0
+# Mode to export tarfs images:
+# - "none" or "": do not export tarfs
+# - "layer_verity_only": only generate disk verity information for a layer blob
+# - "image_verity_only": only generate disk verity information for all blobs of an image
+# - "layer_block": generate a raw block disk image with tarfs for a layer
+# - "image_block": generate a raw block disk image with tarfs for an image
+# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
+# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
+export_mode = ""

misc/snapshotter/entrypoint.sh

@@ -1,22 +0,0 @@
-#! /bin/sh
-
-NYDUS_LIB="${NYDUS_LIB:-/var/lib/containerd-nydus}"
-NYDUS_RUN="${NYDUS_RUN:-/run/containerd-nydus}"
-LEVEL="${LEVEL:-info}"
-
-set -eu
-BACKEND_TYPE="${BACKEND_TYPE:-config}"
-NYDUSD_DAEMON_MODE="${NYDUSD_DAEMON_MODE:-multiple}"
-
-if [ "$#" -eq 0 ]; then
-	containerd-nydus-grpc \
-		--nydusd /usr/local/bin/nydusd \
-		--nydusd-config /etc/nydus/${BACKEND_TYPE}.json \
-		--root ${NYDUS_LIB} \
-		--address ${NYDUS_RUN}/containerd-nydus-grpc.sock \
-		--log-level ${LEVEL} \
-		--daemon-mode ${NYDUSD_DAEMON_MODE} \
-		--log-to-stdout
-fi
-
-exec $@

misc/snapshotter/nydus-snapshotter-rbac.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nydus-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nydus-snapshotter-sa
+  namespace: nydus-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nydus-snapshotter-role
+subjects:
+- kind: ServiceAccount
+  name: nydus-snapshotter-sa
+  namespace: nydus-system

misc/snapshotter/nydus-snapshotter.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=nydus snapshotter
+After=network.target
+Before=containerd.service
+
+[Service]
+Type=simple
+Environment=HOME=/root
+ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config-proxy.toml
+Restart=always
+RestartSec=1
+KillMode=process
+OOMScoreAdjust=-999
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

misc/snapshotter/nydusd-config-localfs.json

@@ -3,7 +3,7 @@
     "backend": {
       "type": "localfs",
       "config": {
-        "dir": "/var/lib/containerd-nydus/cache/"
+        "dir": "/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache/"
       }
     },
     "cache": {
@@ -14,6 +14,7 @@
   "digest_validate": false,
   "iostats_files": false,
   "enable_xattr": true,
+  "amplify_io": 1048576,
   "fs_prefetch": {
     "enable": true,
     "threads_count": 2

misc/snapshotter/nydusd-config.fusedev.json

@@ -16,6 +16,7 @@
   "digest_validate": false,
   "iostats_files": false,
   "enable_xattr": true,
+  "amplify_io": 1048576,
   "fs_prefetch": {
     "enable": true,
     "threads_count": 8,

misc/snapshotter/overlays/k3s/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+patches:
+- path: mount_k3s_conf.yaml

misc/snapshotter/overlays/k3s/mount_k3s_conf.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+spec:
+  template:
+    spec:
+      volumes:
+        - name: containerd-conf
+          hostPath:
+            path: /var/lib/rancher/k3s/agent/etc/containerd/

misc/snapshotter/overlays/rke2/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+patches:
+- path: mount_rke2_conf.yaml

misc/snapshotter/overlays/rke2/mount_rke2_conf.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+spec:
+  template:
+    spec:
+      volumes:
+        - name: containerd-conf
+          hostPath:
+            path: /var/lib/rancher/rke2/agent/etc/containerd/

misc/snapshotter/snapshotter.sh

@@ -0,0 +1,347 @@
+#!/usr/bin/env bash
+# Copyright (c) 2023. Nydus Developers. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+SNAPSHOTTER_ARTIFACTS_DIR="/opt/nydus-artifacts"
+
+# Container runtime config, the default container runtime is containerd
+CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}"
+CONTAINER_RUNTIME_CONFIG="/etc/containerd/config.toml"
+
+# Common nydus snapshotter config options
+FS_DRIVER="${FS_DRIVER:-fusedev}"
+SNAPSHOTTER_GRPC_SOCKET="${SNAPSHOTTER_GRPC_SOCKET:-/run/containerd-nydus/containerd-nydus-grpc.sock}"
+
+# The directory about nydus and nydus snapshotter
+NYDUS_CONFIG_DIR="${NYDUS_CONFIG_DIR:-/etc/nydus}"
+NYDUS_LIB_DIR="${NYDUS_LIB_DIR:-/var/lib/containerd/io.containerd.snapshotter.v1.nydus}"
+NYDUS_BINARY_DIR="${NYDUS_BINARY_DIR:-/usr/local/bin}"
+SNAPSHOTTER_SCRYPT_DIR="${SNAPSHOTTER_SCRYPT_DIR:-/opt/nydus}"
+
+# The binary about nydus-snapshotter
+SNAPSHOTTER_BINARY="${SNAPSHOTTER_BINARY:-${NYDUS_BINARY_DIR}/containerd-nydus-grpc}"
+
+# The config about nydus snapshotter
+SNAPSHOTTER_CONFIG="${SNAPSHOTTER_CONFIG:-${NYDUS_CONFIG_DIR}/config.toml}"
+# The systemd service config about nydus snapshotter
+SNAPSHOTTER_SERVICE="${SNAPSHOTTER_SERVICE:-/etc/systemd/system/nydus-snapshotter.service}"
+# If true, the script would read the config from env.
+ENABLE_CONFIG_FROM_VOLUME="${ENABLE_CONFIG_FROM_VOLUME:-false}"
+# If true, the script would enable the "runtime specific snapshotter" in containerd config.
+ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER="${ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER:-false}"
+# If true, the snapshotter would be running as a systemd service
+ENABLE_SYSTEMD_SERVICE="${ENABLE_SYSTEMD_SERVICE:-false}"
+
+COMMANDLINE=""
+
+# If we fail for any reason a message will be displayed
+die() {
+    msg="$*"
+    echo "ERROR: $msg" >&2
+    exit 1
+}
+
+print_usage() {
+    echo "Usage: $0 [deploy/cleanup]"
+}
+
+wait_service_active(){
+    local wait_time="$1"
+    local sleep_time="$2"
+    local service="$3"
+
+    nsenter -t 1 -m systemctl restart $service
+
+    # Wait for containerd to be running
+    while [ "$wait_time" -gt 0 ]; do
+        if nsenter -t 1 -m systemctl is-active --quiet $service; then
+            echo "$service is running"
+            return 0  
+        else
+            sleep "$sleep_time"
+            wait_time=$((wait_time-sleep_time))
+        fi
+    done
+
+    echo "Timeout reached. $service may not be running."
+    nsenter -t 1 -m systemctl status $service
+    return 1  
+}
+
+function fs_driver_handler() {
+    if [ "${ENABLE_CONFIG_FROM_VOLUME}" == "true" ]; then
+        SNAPSHOTTER_CONFIG="${NYDUS_CONFIG_DIR}/config.toml"
+    else
+        case "${FS_DRIVER}" in
+        fusedev) 
+            sed -i -e "s|nydusd_config = .*|nydusd_config = \"${NYDUS_CONFIG_DIR}/nydusd-fusedev.json\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|fs_driver = .*|fs_driver = \"fusedev\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|daemon_mode = .*|daemon_mode = \"multiple\"|" "${SNAPSHOTTER_CONFIG}" 
+            ;;
+        fscache) 
+            sed -i -e "s|nydusd_config = .*|nydusd_config = \"${NYDUS_CONFIG_DIR}/nydusd-fscache.json\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|fs_driver = .*|fs_driver = \"fscache\"|" "${SNAPSHOTTER_CONFIG}"
+            sed -i -e "s|daemon_mode = .*|daemon_mode = \"multiple\"|" "${SNAPSHOTTER_CONFIG}"  
+            ;;
+        blockdev) 
+            sed -i -e "s|fs_driver = .*|fs_driver = \"blockdev\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|enable_kata_volume = .*|enable_kata_volume = true|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|enable_tarfs = .*|enable_tarfs = true|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|daemon_mode = .*|daemon_mode = \"none\"|" "${SNAPSHOTTER_CONFIG}"  
+            sed -i -e "s|export_mode = .*|export_mode = \"layer_block_with_verity\"|" "${SNAPSHOTTER_CONFIG}"  
+            ;;
+        proxy) 
+            sed -i -e "s|fs_driver = .*|fs_driver = \"proxy\"|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|enable_kata_volume = .*|enable_kata_volume = true|" "${SNAPSHOTTER_CONFIG}" 
+            sed -i -e "s|daemon_mode = .*|daemon_mode = \"none\"|" "${SNAPSHOTTER_CONFIG}"  
+            ;;
+        *) die "invalid fs driver ${FS_DRIVER}" ;;
+        esac
+    fi
+    COMMANDLINE+=" --config ${SNAPSHOTTER_CONFIG}"
+}
+
+function configure_snapshotter() {
+
+    echo "configuring snapshotter"
+
+    # Copy the container runtime config to a backup
+    cp "$CONTAINER_RUNTIME_CONFIG" "$CONTAINER_RUNTIME_CONFIG".bak.nydus
+
+
+    # When trying to edit the config file that is mounted by docker with `sed -i`, the error would happend:
+    # sed: cannot rename /etc/containerd/config.tomlpmdkIP: Device or resource busy  
+    # The reason is that `sed`` with option `-i` creates new file, and then replaces the old file with the new one, 
+    # which definitely will change the file inode. But the file is mounted by docker, which means we are not allowed to 
+    # change its inode from within docker container.
+    # 
+    # So we copy the original file to a backup, make changes to the backup, and then overwrite the original file with the backup.
+    cp "$CONTAINER_RUNTIME_CONFIG" "$CONTAINER_RUNTIME_CONFIG".bak
+    # Check and add nydus proxy plugin in the config
+    if grep -q '\[proxy_plugins.nydus\]' "$CONTAINER_RUNTIME_CONFIG".bak; then
+        echo "the config has configured the nydus proxy plugin!"
+    else
+        echo "Not found nydus proxy plugin!"
+        cat <<EOF >>"$CONTAINER_RUNTIME_CONFIG".bak
+        
+    [proxy_plugins.nydus]
+        type = "snapshot"
+        address = "$SNAPSHOTTER_GRPC_SOCKET"
+EOF
+    fi
+
+    if grep -q 'disable_snapshot_annotations' "$CONTAINER_RUNTIME_CONFIG".bak; then
+        sed -i -e "s|disable_snapshot_annotations = .*|disable_snapshot_annotations = false|" \
+                "${CONTAINER_RUNTIME_CONFIG}".bak
+    else
+        sed -i '/\[plugins\..*\.containerd\]/a\disable_snapshot_annotations = false' \
+                "${CONTAINER_RUNTIME_CONFIG}".bak
+    fi
+    if grep -q 'discard_unpacked_layers' "$CONTAINER_RUNTIME_CONFIG".bak; then
+        sed -i -e "s|discard_unpacked_layers = .*|discard_unpacked_layers = false|" \
+                "${CONTAINER_RUNTIME_CONFIG}".bak
+    else
+        sed -i '/\[plugins\..*\.containerd\]/a\discard_unpacked_layers = false' \
+                "${CONTAINER_RUNTIME_CONFIG}".bak
+    fi
+
+    if [ "${ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER}" == "false" ]; then
+        sed -i -e '/\[plugins\..*\.containerd\]/,/snapshotter =/ s/snapshotter = "[^"]*"/snapshotter = "nydus"/' "${CONTAINER_RUNTIME_CONFIG}".bak
+    fi
+    
+    cat "${CONTAINER_RUNTIME_CONFIG}".bak >  "${CONTAINER_RUNTIME_CONFIG}"
+}
+
+function install_snapshotter() {
+    echo "install nydus snapshotter artifacts"
+    find "${SNAPSHOTTER_ARTIFACTS_DIR}${NYDUS_BINARY_DIR}" -type f -exec install -Dm 755 -t "${NYDUS_BINARY_DIR}" "{}"  \;
+    find "${SNAPSHOTTER_ARTIFACTS_DIR}${NYDUS_CONFIG_DIR}" -type f -exec install -Dm 644 -t "${NYDUS_CONFIG_DIR}" "{}"  \;
+    install -D -m 644 "${SNAPSHOTTER_ARTIFACTS_DIR}${SNAPSHOTTER_SCRYPT_DIR}/snapshotter.sh" "${SNAPSHOTTER_SCRYPT_DIR}/snapshotter.sh"
+    if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then
+        install -D -m 644 "${SNAPSHOTTER_ARTIFACTS_DIR}${SNAPSHOTTER_SERVICE}" "${SNAPSHOTTER_SERVICE}"
+    fi
+    if [ "${ENABLE_CONFIG_FROM_VOLUME}" == "true" ]; then
+        find "/etc/nydus-snapshotter" -type f -exec install -Dm 644 -t "${NYDUS_CONFIG_DIR}" "{}"  \;
+    fi
+}
+
+function deploy_snapshotter() {
+    echo "deploying snapshotter"
+    install_snapshotter
+
+    COMMANDLINE="${SNAPSHOTTER_BINARY}"
+    fs_driver_handler
+    configure_snapshotter
+    if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then
+        echo "running snapshotter as systemd service"
+        sed -i "s|^ExecStart=.*$|ExecStart=$COMMANDLINE|" "${SNAPSHOTTER_SERVICE}"
+        nsenter -t 1 -m systemctl daemon-reload
+        nsenter -t 1 -m systemctl enable nydus-snapshotter.service
+        wait_service_active 30 5 nydus-snapshotter
+    else
+        echo "running snapshotter as standalone process"
+        ${COMMANDLINE} &
+    fi
+    wait_service_active 30 5 ${CONTAINER_RUNTIME}
+
+}
+
+function remove_images() {
+    local SNAPSHOTTER="nydus"
+    local NAMESPACE="k8s.io"
+    local ctr_args="nsenter -t 1 -m ctr"
+
+    if [[ " k3s k3s-agent rke2-agent rke2-server " =~ " ${CONTAINER_RUNTIME} " ]]; then
+        ctr_args+=" --address /run/k3s/containerd/containerd.sock"
+    fi
+    ctr_args+=" --namespace $NAMESPACE"
+
+    # List all snapshots for nydus snapshotter
+    local SNAPSHOTS=$($ctr_args snapshot --snapshotter $SNAPSHOTTER ls | awk 'NR>1 {print $1}')
+    echo "Images associated with snapshotter $SNAPSHOTTER:"
+
+    # Loop through each snapshot and find associated contents
+    for SNAPSHOT in $SNAPSHOTS; do
+        local CONTENTS=$($ctr_args content ls | grep $SNAPSHOT | awk '{print $1}')
+        echo "Snapshot: $SNAPSHOT, Contents: $CONTENTS"
+        if [ -z "$CONTENTS" ]; then
+            continue
+        fi
+        # Loop through each content and find associated digests of images
+        for CONTENT in $CONTENTS; do
+            local DIGESTS=$($ctr_args image ls | grep $CONTENT | awk '{print $3}')
+            echo "Content: $CONTENT, Digests: $DIGESTS"
+            if [ -z "$DIGESTS" ]; then
+                continue
+            fi
+            # Loop through each digest and find associated image references
+            for DIGEST in $DIGESTS; do
+                local IMAGES=$($ctr_args image ls | grep $DIGEST | awk '{print $1}')
+                echo "Digest: $DIGEST, Images: $IMAGES"
+                if [ -z "$IMAGES" ]; then
+                    continue
+                fi
+                for IMAGE in $IMAGES; do
+                    # Delete the image
+                    $ctr_args images rm $IMAGE > /dev/null 2>&1 || true
+                    echo "Image $IMAGE removed"
+                done
+            done
+            # Delete the content
+            $ctr_args content rm $CONTENT > /dev/null 2>&1 || true
+            echo "content $CONTENT removed"
+        done
+        # Delete the snapshot
+        $ctr_args snapshot --snapshotter $SNAPSHOTTER rm $SNAPSHOT > /dev/null 2>&1 || true
+        echo "snapshot $SNAPSHOT removed"
+    done
+    echo "INFO: Images removed"
+}
+
+function cleanup_snapshotter() {
+    echo "cleaning up snapshotter"
+
+    pid=$(ps -ef | grep containerd-nydus-grpc | grep -v grep | awk '{print $1}' || true)
+    if [ ! -z "$pid" ]; then
+        remove_images
+    fi
+    echo "Recover containerd config"
+    cat "$CONTAINER_RUNTIME_CONFIG".bak.nydus >"$CONTAINER_RUNTIME_CONFIG"
+    if [ "${ENABLE_SYSTEMD_SERVICE}" == "true" ]; then
+        nsenter -t 1 -m systemctl stop nydus-snapshotter.service
+        nsenter -t 1 -m systemctl disable --now nydus-snapshotter.service
+        rm -f "${SNAPSHOTTER_SERVICE}" || true
+    else
+        kill -9 $pid || true
+    fi
+    wait_service_active 30 5 ${CONTAINER_RUNTIME}
+    echo "Removing nydus-snapshotter artifacts from host"
+    rm -f "${NYDUS_BINARY_DIR}"/nydus*
+    rm -rf "${NYDUS_CONFIG_DIR}"/*
+    rm -rf "${SNAPSHOTTER_SCRYPT_DIR}"/*
+    rm -rf "${NYDUS_LIB_DIR}"/*
+    echo "cleaned up snapshotter"
+}
+
+function get_container_runtime() {
+    local runtime=$(kubectl get node ${NODE_NAME} -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}')
+    if [ "$?" -ne 0 ]; then
+        die "\"$NODE_NAME\" is an invalid node name"
+    fi
+
+    if echo "$runtime" | grep -qE 'containerd.*-k3s'; then
+        if nsenter -t 1 -m systemctl is-active --quiet rke2-agent; then
+            echo "rke2-agent"
+        elif nsenter -t 1 -m systemctl is-active --quiet rke2-server; then
+            echo "rke2-server"
+        elif nsenter -t 1 -m systemctl is-active --quiet k3s-agent; then
+            echo "k3s-agent"
+        else
+            echo "k3s"
+        fi
+    elif nsenter -t 1 -m systemctl is-active --quiet k0scontroller; then
+        echo "k0s-controller"
+    elif nsenter -t 1 -m systemctl is-active --quiet k0sworker; then
+        echo "k0s-worker"
+    else
+        echo "$runtime" | awk -F '[:]' '{print $1}'
+    fi
+}
+
+function main() {
+    # script requires that user is root
+    euid=$(id -u)
+    if [[ $euid -ne 0 ]]; then
+        die "This script must be run as root"
+    fi
+
+    CONTAINER_RUNTIME=$(get_container_runtime)
+    if [[ " k3s k3s-agent rke2-agent rke2-server " =~ " ${CONTAINER_RUNTIME} " ]]; then
+        CONTAINER_RUNTIME_CONFIG_TMPL="${CONTAINER_RUNTIME_CONFIG}.tmpl"
+        if [ ! -f "${CONTAINER_RUNTIME_CONFIG_TMPL}" ]; then
+            cp "${CONTAINER_RUNTIME_CONFIG}" "${CONTAINER_RUNTIME_CONFIG_TMPL}"
+        fi
+
+        CONTAINER_RUNTIME_CONFIG="${CONTAINER_RUNTIME_CONFIG_TMPL}"
+    elif [ "${CONTAINER_RUNTIME}" == "containerd" ]; then
+        if [ ! -f "${CONTAINER_RUNTIME_CONFIG}" ]; then
+            mkdir -p $(dirname ${CONTAINER_RUNTIME_CONFIG}) || true
+            if [ -x $(command -v ${CONTAINER_RUNTIME}) ]; then
+                ${CONTAINER_RUNTIME} config default > ${CONTAINER_RUNTIME_CONFIG}
+            else
+                die "Not able to find an executable ${CONTAINER_RUNTIME} binary to create the default config"
+            fi
+        fi
+    else
+        die "${CONTAINER_RUNTIME} is a unsupported containe runtime"
+    fi
+
+    action=${1:-}
+    if [ -z "$action" ]; then
+        print_usage
+        die "invalid arguments"
+    fi
+
+    case "$action" in
+    deploy)
+        deploy_snapshotter
+        ;;
+    cleanup)
+        cleanup_snapshotter
+        ;;
+    *)
+        die "invalid arguments"
+        print_usage
+        ;;
+    esac
+
+    sleep infinity
+}
+
+main "$@"

pkg/auth/image_proxy.go

@@ -11,15 +11,13 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/containerd/containerd/defaults"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/pkg/dialer"
-	"github.com/containerd/containerd/reference"
-	distribution "github.com/containerd/containerd/reference/docker"
-	runtime_alpha "github.com/containerd/containerd/third_party/k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+	"github.com/containerd/containerd/v2/defaults"
+	"github.com/containerd/containerd/v2/pkg/dialer"
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/log"
 	"github.com/containerd/stargz-snapshotter/service/keychain/cri"
-	"github.com/containerd/stargz-snapshotter/service/keychain/crialpha"
 	"github.com/containerd/stargz-snapshotter/service/resolver"
+	distribution "github.com/distribution/reference"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/backoff"
@@ -47,7 +45,7 @@ func newCRIConn(criAddr string) (*grpc.ClientConn, error) {
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
 		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
 	}
-	return grpc.Dial(dialer.DialAddress(criAddr), gopts...)
+	return grpc.NewClient(dialer.DialAddress(criAddr), gopts...)
 }
 
 // from stargz-snapshotter/cmd/containerd-stargz-grpc/main.go#main
@@ -57,30 +55,18 @@ func AddImageProxy(ctx context.Context, rpc *grpc.Server, imageServiceAddress st
 		criAddr = imageServiceAddress
 	}
 
-	connectAlphaCRI := func() (runtime_alpha.ImageServiceClient, error) {
-		conn, err := newCRIConn(criAddr)
-		if err != nil {
-			return nil, err
-		}
-		return runtime_alpha.NewImageServiceClient(conn), nil
-	}
-
-	connectCri := func() (runtime.ImageServiceClient, error) {
+	criCred, criServer := cri.NewCRIKeychain(ctx, func() (runtime.ImageServiceClient, error) {
 		conn, err := newCRIConn(criAddr)
 		if err != nil {
 			return nil, err
 		}
 
 		return runtime.NewImageServiceClient(conn), nil
-	}
+	})
 
-	criAlphaCred, criAlphaServer := crialpha.NewCRIAlphaKeychain(ctx, connectAlphaCRI)
-	runtime_alpha.RegisterImageServiceServer(rpc, criAlphaServer)
-
-	criCred, criServer := cri.NewCRIKeychain(ctx, connectCri)
 	runtime.RegisterImageServiceServer(rpc, criServer)
 
-	Credentials = append(Credentials, criAlphaCred, criCred)
+	Credentials = append(Credentials, criCred)
 
 	log.G(ctx).WithField("target-image-service", criAddr).Info("setup image proxy keychain")
 }

pkg/auth/image_proxy_test.go

@@ -9,118 +9,105 @@ package auth
 import (
 	"context"
 	"net"
-	"os"
 	"path/filepath"
 	"testing"
 
-	"github.com/containerd/containerd/pkg/dialer"
-	runtime_alpha "github.com/containerd/containerd/third_party/k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+	"github.com/containerd/containerd/v2/pkg/dialer"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 )
 
-type MockAlphaImageService struct {
-	runtime_alpha.UnimplementedImageServiceServer
-}
-
-func (*MockAlphaImageService) PullImage(ctx context.Context, req *runtime_alpha.PullImageRequest) (*runtime_alpha.PullImageResponse, error) {
-	return &runtime_alpha.PullImageResponse{}, nil
-}
-
 type MockImageService struct {
 	runtime.UnimplementedImageServiceServer
 }
 
-func (*MockImageService) PullImage(ctx context.Context, req *runtime.PullImageRequest) (*runtime.PullImageResponse, error) {
+func (*MockImageService) PullImage(_ context.Context, _ *runtime.PullImageRequest) (*runtime.PullImageResponse, error) {
 	return &runtime.PullImageResponse{}, nil
 }
 
 func TestFromImagePull(t *testing.T) {
 	var err error
-	assert := assert.New(t)
+	assertions := assert.New(t)
 
 	ctx := context.TODO()
 	d := t.TempDir()
-	defer os.RemoveAll(d)
 
 	tagImage := "docker.io/library/busybox:latest"
+
 	// should return nil if no proxy
 	kc, err := FromCRI("docker.io", tagImage)
-	assert.Nil(kc)
-	assert.NoError(err)
+	assertions.Nil(kc)
+	assertions.NoError(err)
 
+	// Mocking the end CRI request consumer.
 	mockRPC := grpc.NewServer()
 	mockSocket := filepath.Join(d, "mock.sock")
-	lm, err := net.Listen("unix", mockSocket)
-	assert.NoError(err)
+	listenMock, err := net.Listen("unix", mockSocket)
+	assertions.NoError(err)
+
+	// Mocking the end CRI request consumer.
+	server := &MockImageService{}
+	runtime.RegisterImageServiceServer(mockRPC, server)
+
+	go func() {
+		err := mockRPC.Serve(listenMock)
+		assertions.NoError(err)
+	}()
+	defer mockRPC.Stop()
 
 	// The server of CRI image service proxy.
 	proxyRPC := grpc.NewServer()
 	proxySocket := filepath.Join(d, "proxy.sock")
-	lp, err := net.Listen("unix", proxySocket)
-	assert.NoError(err)
-
-	// Mocking the end CRI request consumer.
-	serverAlpha := &MockAlphaImageService{}
-	server := &MockImageService{}
-	runtime_alpha.RegisterImageServiceServer(mockRPC, serverAlpha)
-	runtime.RegisterImageServiceServer(mockRPC, server)
-
-	go func() {
-		err := mockRPC.Serve(lm)
-		assert.NoError(err)
-	}()
-	defer mockRPC.Stop()
-
+	listenProxy, err := net.Listen("unix", proxySocket)
+	assertions.NoError(err)
 	AddImageProxy(ctx, proxyRPC, mockSocket)
 	go func() {
-		err := proxyRPC.Serve(lp)
-		assert.NoError(err)
+		err := proxyRPC.Serve(listenProxy)
+		assertions.NoError(err)
 	}()
 	defer proxyRPC.Stop()
 
-	kc, err = FromCRI("docker.io", tagImage)
 	// should return empty kc before pulling
-	assert.Nil(kc)
-	assert.NoError(err)
+	kc, err = FromCRI("docker.io", tagImage)
+	assertions.Nil(kc)
+	assertions.NoError(err)
 
 	gopts := []grpc.DialOption{
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
 		grpc.WithContextDialer(dialer.ContextDialer),
 	}
-	conn, err := grpc.Dial(dialer.DialAddress(proxySocket), gopts...)
-	assert.NoError(err)
-	criAlphaClient := runtime_alpha.NewImageServiceClient(conn)
-	irAlpha := &runtime_alpha.PullImageRequest{
-		Image: &runtime_alpha.ImageSpec{
+
+	conn, err := grpc.NewClient(dialer.DialAddress(proxySocket), gopts...)
+	assertions.NoError(err)
+	criClient := runtime.NewImageServiceClient(conn)
+
+	_, err = criClient.PullImage(ctx, &runtime.PullImageRequest{
+		Image: &runtime.ImageSpec{
 			Image: tagImage,
 		},
-		Auth: &runtime_alpha.AuthConfig{
+		Auth: &runtime.AuthConfig{
 			Username: "test",
 			Password: "passwd",
 		},
-	}
-	_, err = criAlphaClient.PullImage(ctx, irAlpha)
-	assert.NoError(err)
+	})
+	assertions.NoError(err)
 
-	criClient := runtime.NewImageServiceClient(conn)
-
-	kc, err = FromCRI("docker.io", tagImage)
 	// get correct kc after pulling
-	assert.Equal("test", kc.Username)
-	assert.Equal("passwd", kc.Password)
-	assert.NoError(err)
+	kc, err = FromCRI("docker.io", tagImage)
+	assertions.Equal("test", kc.Username)
+	assertions.Equal("passwd", kc.Password)
+	assertions.NoError(err)
 
-	kc, err = FromCRI("docker.io", "docker.io/library/busybox:another")
 	// get empty kc with wrong tag
-	assert.Nil(kc)
-	assert.NoError(err)
+	kc, err = FromCRI("docker.io", "docker.io/library/busybox:another")
+	assertions.Nil(kc)
+	assertions.NoError(err)
 
 	image2 := "ghcr.io/busybox:latest"
 
-	ir := &runtime.PullImageRequest{
+	_, err = criClient.PullImage(ctx, &runtime.PullImageRequest{
 		Image: &runtime.ImageSpec{
 			Image: image2,
 		},
@@ -128,31 +115,31 @@ func TestFromImagePull(t *testing.T) {
 			Username: "test_1",
 			Password: "passwd_1",
 		},
-	}
-	_, err = criClient.PullImage(ctx, ir)
-	assert.NoError(err)
+	})
+	assertions.NoError(err)
 
+	// get correct kc after pulling
 	kc, err = FromCRI("ghcr.io", image2)
-	assert.Equal(kc.Username, "test_1")
-	assert.Equal(kc.Password, "passwd_1")
-	assert.NoError(err)
+	assertions.NoError(err)
+	assertions.Equal(kc.Username, "test_1")
+	assertions.Equal(kc.Password, "passwd_1")
 
 	// should work with digest
 	digestImage := "docker.io/library/busybox@sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa"
-	irAlpha = &runtime_alpha.PullImageRequest{
-		Image: &runtime_alpha.ImageSpec{
+	_, err = criClient.PullImage(ctx, &runtime.PullImageRequest{
+		Image: &runtime.ImageSpec{
 			Image: digestImage,
 		},
-		Auth: &runtime_alpha.AuthConfig{
+		Auth: &runtime.AuthConfig{
 			Username: "digest",
 			Password: "dpwd",
 		},
-	}
-	_, err = criAlphaClient.PullImage(ctx, irAlpha)
-	assert.NoError(err)
+	})
+	assertions.NoError(err)
 
+	// get correct kc after pulling
 	kc, err = FromCRI("docker.io", digestImage)
-	assert.Equal("digest", kc.Username)
-	assert.Equal("dpwd", kc.Password)
-	assert.NoError(err)
+	assertions.NoError(err)
+	assertions.Equal("digest", kc.Username)
+	assertions.Equal("dpwd", kc.Password)
 }

pkg/auth/keychain.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nydus-snapshotter/pkg/label"
+	distribution "github.com/distribution/reference"
 	"github.com/google/go-containerregistry/pkg/authn"
 )
 
@@ -101,22 +101,23 @@ func GetRegistryKeyChain(host, ref string, labels map[string]string) *PassKeyCha
 	if kc != nil {
 		return kc
 	}
+
 	return FromKubeSecretDockerConfig(host)
 }
 
 func GetKeyChainByRef(ref string, labels map[string]string) (*PassKeyChain, error) {
-	named, err := docker.ParseDockerRef(ref)
+	named, err := distribution.ParseDockerRef(ref)
 	if err != nil {
 		return nil, errors.Wrapf(err, "parse ref %s", ref)
 	}
 
-	host := docker.Domain(named)
+	host := distribution.Domain(named)
 	keychain := GetRegistryKeyChain(host, ref, labels)
 
 	return keychain, nil
 }
 
-func (kc PassKeyChain) Resolve(target authn.Resource) (authn.Authenticator, error) {
+func (kc PassKeyChain) Resolve(_ authn.Resource) (authn.Authenticator, error) {
 	return authn.FromConfig(kc.toAuthConfig()), nil
 }
 

pkg/auth/kubesecret.go

@@ -125,7 +125,7 @@ func (kubelistener *KubeSecretListener) SyncKubeSecrets(ctx context.Context, cli
 				return
 			}
 		},
-		UpdateFunc: func(old, new interface{}) {
+		UpdateFunc: func(_, new interface{}) {
 			key, err := cache.MetaNamespaceKeyFunc(new)
 			if err != nil {
 				logrus.WithError(err).Errorf("failed to get key for secret from cache")

pkg/backend/backend.go

@@ -10,7 +10,7 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/v2/core/content"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )

pkg/backend/localfs.go

@@ -14,8 +14,8 @@ import (
 	"os"
 	"path"
 
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/errdefs"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"

pkg/backend/oss.go

@@ -14,8 +14,8 @@ import (
 	"time"
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/errdefs"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -143,7 +143,7 @@ func (b *OSSBackend) push(ctx context.Context, cs content.Store, desc ocispec.De
 	}
 	close(partsChan)
 
-	var parts []oss.UploadPart
+	parts := make([]oss.UploadPart, 0, 16)
 	for p := range partsChan {
 		parts = append(parts, p)
 	}

pkg/backend/s3.go

@@ -19,8 +19,8 @@ import (
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/errdefs"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -85,7 +85,7 @@ func (b *S3Backend) client() (*s3.Client, error) {
 	}
 
 	client := s3.NewFromConfig(s3AWSConfig, func(o *s3.Options) {
-		o.EndpointResolver = s3.EndpointResolverFromURL(b.endpointWithScheme)
+		o.BaseEndpoint = &b.endpointWithScheme
 		o.Region = b.region
 		o.UsePathStyle = true
 		if len(b.accessKeySecret) > 0 && len(b.accessKeyID) > 0 {

pkg/blob/blob_manager.go

@@ -1,156 +0,0 @@
-/*
- * Copyright (c) 2022. Nydus Developers. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package blob
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
-
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/snapshots/storage"
-	"github.com/containerd/nydus-snapshotter/pkg/resolve"
-	"github.com/containerd/nydus-snapshotter/pkg/utils/registry"
-)
-
-type Manager struct {
-	blobDir   string
-	eventChan chan string
-	resolver  *resolve.Resolver
-}
-
-func NewBlobManager(blobDir string, resolver *resolve.Resolver) *Manager {
-	return &Manager{
-		blobDir: blobDir,
-		// TODO(tianqian.zyf): Remove hardcode chan buffer
-		eventChan: make(chan string, 8),
-		resolver:  resolver,
-	}
-}
-
-func getBlobPath(dir string, blobDigest string) (string, error) {
-	digest, err := digest.Parse(blobDigest)
-	if err != nil {
-		return "", errors.Wrapf(err, "invalid layer digest %s", blobDigest)
-	}
-	return filepath.Join(dir, digest.Encoded()), nil
-}
-
-func (b *Manager) Run(ctx context.Context) error {
-	log.G(ctx).Info("blob manager goroutine start...")
-	for {
-		select {
-		case id := <-b.eventChan:
-			err := b.cleanupBlob(id)
-			if err != nil {
-				log.G(ctx).WithError(err).Warnf("delete blob %s failed", id)
-			} else {
-				log.G(ctx).Infof("delete blob %s success", id)
-			}
-		case <-ctx.Done():
-			log.G(ctx).Infof("exit from BlobManger")
-			return ctx.Err()
-		}
-	}
-}
-
-func (b *Manager) GetBlobDir() string {
-	return b.blobDir
-}
-
-func (b *Manager) cleanupBlob(id string) error {
-	return os.Remove(filepath.Join(b.blobDir, id))
-}
-
-func (b *Manager) decodeID(id string) (string, error) {
-	digest, err := digest.Parse(id)
-	if err != nil {
-		return "", errors.Wrapf(err, "invalid blob layer digest %s", id)
-	}
-	return digest.Encoded(), nil
-}
-
-func (b *Manager) Remove(id string, async bool) error {
-	id, err := b.decodeID(id)
-	if err != nil {
-		return err
-	}
-	if async {
-		b.eventChan <- id
-		return nil
-	}
-	return b.cleanupBlob(id)
-}
-
-func (b *Manager) CleanupBlobLayer(ctx context.Context, blobDigest string, async bool) error {
-	return b.Remove(blobDigest, async)
-}
-
-// Download blobs and bootstrap in nydus-snapshotter for preheating container image usage. It has to
-// enable blobs manager when start nydus-snapshotter
-func (b *Manager) PrepareBlobLayer(snapshot storage.Snapshot, labels map[string]string) error {
-	start := time.Now()
-	defer func() {
-		duration := time.Since(start)
-		log.L.Infof("total nydus prepare data layer duration %s", duration)
-	}()
-
-	ref, layerDigest := registry.ParseLabels(labels)
-	if ref == "" || layerDigest == "" {
-		return fmt.Errorf("can not find ref and digest from label %+v", labels)
-	}
-	blobPath, err := getBlobPath(b.GetBlobDir(), layerDigest)
-	if err != nil {
-		return errors.Wrap(err, "failed to get blob path")
-	}
-	_, err = os.Stat(blobPath)
-	if err == nil {
-		log.L.Debugf("%s blob layer already exists", blobPath)
-		return nil
-	} else if !os.IsNotExist(err) {
-		return errors.Wrap(err, "Unexpected error, we can't handle it")
-	}
-
-	readerCloser, err := b.resolver.Resolve(ref, layerDigest, labels)
-	if err != nil {
-		return errors.Wrapf(err, "failed to resolve from ref %s, digest %s", ref, layerDigest)
-	}
-	defer readerCloser.Close()
-
-	blobFile, err := os.CreateTemp(b.GetBlobDir(), "downloading-")
-	if err != nil {
-		return errors.Wrap(err, "create temp file for downloading blob")
-	}
-	defer func() {
-		if err != nil {
-			os.Remove(blobFile.Name())
-		}
-		blobFile.Close()
-	}()
-
-	_, err = io.Copy(blobFile, readerCloser)
-	if err != nil {
-		return errors.Wrap(err, "write blob to local file")
-	}
-	err = os.Rename(blobFile.Name(), blobPath)
-	if err != nil {
-		return errors.Wrap(err, "rename temp file as blob file")
-	}
-
-	err = os.Chmod(blobPath, 0440)
-	if err != nil {
-		return errors.Wrapf(err, "chmod blob %s", blobPath)
-	}
-
-	return nil
-}

pkg/blob/blob_manager_test.go

@@ -1,86 +0,0 @@
-/*
- * Copyright (c) 2022. Nydus Developers. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-package blob
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/containerd/nydus-snapshotter/pkg/resolve"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_Remove(t *testing.T) {
-	dir, err := os.MkdirTemp(os.TempDir(), "blb")
-	if err != nil {
-		t.Fatalf("failed to make dir %s", dir)
-	}
-	defer os.RemoveAll(dir)
-	id := "038f2b2815ae3c309b77bf34bf6ce988c922b7718773b4c98d5cd2b76c35a146"
-	file, err := os.Create(filepath.Join(dir, id))
-	if err != nil {
-		t.Fatalf("failed to create dir %s", dir)
-	}
-	file.Close()
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	blobMgr := NewBlobManager(dir, resolve.NewResolver())
-	go func() {
-		// This goroutine can be canceled, so ignore error
-		_ = blobMgr.Run(ctx)
-	}()
-	err = blobMgr.Remove("sha256:"+id, true)
-	assert.Nil(t, err)
-	// wait to deleted the file
-	time.Sleep(time.Millisecond * 100)
-	_, err = os.Stat(filepath.Join(dir, "sha256:"+id))
-	if err == nil || !os.IsNotExist(err) {
-		t.Fatalf("expect the blob file not exits, but actual exists")
-	}
-}
-
-func Test_decodeID(t *testing.T) {
-	blobMgr := NewBlobManager("dir", resolve.NewResolver())
-	tests := []struct {
-		name     string
-		id       string
-		decodeID string
-		hasError bool
-	}{
-		{
-			name:     "ok",
-			id:       "sha256:038f2b2815ae3c309b77bf34bf6ce988c922b7718773b4c98d5cd2b76c35a146",
-			decodeID: "038f2b2815ae3c309b77bf34bf6ce988c922b7718773b4c98d5cd2b76c35a146",
-			hasError: false,
-		},
-		{
-			name:     "not ok",
-			id:       "038f2b2815ae3c309b77bf34bf6ce988c922b7718773b4c98d5cd2b76c35a146",
-			decodeID: "",
-			hasError: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			realID, err := blobMgr.decodeID(tt.id)
-			if tt.hasError {
-				if err == nil {
-					t.Fatal("expect has error, but actual is nil")
-				}
-			} else {
-				if err != nil {
-					t.Fatalf("expect doesn't have error, but actual is %s", err)
-				}
-				if realID != tt.decodeID {
-					t.Fatalf("expect doesn't have error, but actual is %s", err)
-				}
-			}
-		})
-	}
-}

pkg/cache/manager.go

@@ -14,31 +14,33 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/snapshots"
+	"github.com/containerd/containerd/v2/core/snapshots"
 	"github.com/containerd/continuity/fs"
+	"github.com/containerd/log"
 	"github.com/containerd/nydus-snapshotter/pkg/store"
 )
 
 const (
-	chunkMapFileSuffix = ".chunk_map"
-	metaFileSuffix     = ".blob.meta"
+	imageDiskFileSuffix = ".image.disk"
+	layerDiskFileSuffix = ".layer.disk"
+	chunkMapFileSuffix  = ".chunk_map"
+	metaFileSuffix      = ".blob.meta"
 	// Blob cache is suffixed after nydus v2.1
 	dataFileSuffix = ".blob.data"
 )
 
+// Disk cache manager for fusedev.
 type Manager struct {
 	cacheDir string
 	period   time.Duration
 	eventCh  chan struct{}
-	fsDriver string
 }
 
 type Opt struct {
+	Disabled bool
 	CacheDir string
 	Period   time.Duration
 	Database *store.Database
-	FsDriver string
 }
 
 func NewManager(opt Opt) (*Manager, error) {
@@ -52,7 +54,6 @@ func NewManager(opt Opt) (*Manager, error) {
 		cacheDir: opt.CacheDir,
 		period:   opt.Period,
 		eventCh:  eventCh,
-		fsDriver: opt.FsDriver,
 	}
 
 	return m, nil
@@ -70,12 +71,15 @@ func (m *Manager) CacheUsage(ctx context.Context, blobID string) (snapshots.Usag
 	var usage snapshots.Usage
 
 	blobCachePath := path.Join(m.cacheDir, blobID)
+	blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix)
 	// For backward compatibility
 	blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix)
-	blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix)
+	blobChunkMapSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix+chunkMapFileSuffix)
 	blobMeta := path.Join(m.cacheDir, blobID+metaFileSuffix)
+	imageDisk := path.Join(m.cacheDir, blobID+imageDiskFileSuffix)
+	layerDisk := path.Join(m.cacheDir, blobID+layerDiskFileSuffix)
 
-	stuffs := []string{blobCachePath, blobCacheSuffixedPath, blobChunkMap, blobMeta}
+	stuffs := []string{blobCachePath, blobChunkMap, blobCacheSuffixedPath, blobChunkMapSuffixedPath, blobMeta, imageDisk, layerDisk}
 
 	for _, f := range stuffs {
 		du, err := fs.DiskUsage(ctx, f)
@@ -94,12 +98,15 @@ func (m *Manager) CacheUsage(ctx context.Context, blobID string) (snapshots.Usag
 
 func (m *Manager) RemoveBlobCache(blobID string) error {
 	blobCachePath := path.Join(m.cacheDir, blobID)
-	blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix)
 	blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix)
+	blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix)
+	blobChunkMapSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix+chunkMapFileSuffix)
 	blobMeta := path.Join(m.cacheDir, blobID+metaFileSuffix)
+	imageDisk := path.Join(m.cacheDir, blobID+imageDiskFileSuffix)
+	layerDisk := path.Join(m.cacheDir, blobID+layerDiskFileSuffix)
 
 	// NOTE: Delete chunk bitmap file before data blob
-	stuffs := []string{blobChunkMap, blobMeta, blobCachePath, blobCacheSuffixedPath}
+	stuffs := []string{blobChunkMap, blobChunkMapSuffixedPath, blobMeta, blobCachePath, blobCacheSuffixedPath, imageDisk, layerDisk}
 
 	for _, f := range stuffs {
 		err := os.Remove(f)

pkg/cgroup/cgroup.go

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package cgroup
+
+import (
+	"errors"
+
+	"github.com/containerd/cgroups/v3"
+	v1 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v1"
+	v2 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v2"
+)
+
+const (
+	defaultSlice = "system.slice"
+)
+
+var (
+	ErrCgroupNotSupported = errors.New("cgroups: cgroup not supported")
+)
+
+type Config struct {
+	MemoryLimitInBytes int64
+}
+
+type DaemonCgroup interface {
+	// Delete the current cgroup.
+	Delete() error
+	// Add a process to current cgroup.
+	AddProc(pid int) error
+}
+
+func createCgroup(name string, config Config) (DaemonCgroup, error) {
+	if cgroups.Mode() == cgroups.Unified {
+		return v2.NewCgroup(defaultSlice, name, config.MemoryLimitInBytes)
+	}
+
+	return v1.NewCgroup(defaultSlice, name, config.MemoryLimitInBytes)
+}
+
+func supported() bool {
+	return cgroups.Mode() != cgroups.Unavailable
+}
+
+func displayMode() string {
+	switch cgroups.Mode() {
+	case cgroups.Legacy:
+		return "legacy"
+	case cgroups.Hybrid:
+		return "hybrid"
+	case cgroups.Unified:
+		return "unified"
+	case cgroups.Unavailable:
+		return "unavailable"
+	default:
+		return "unknown"
+	}
+}

pkg/cgroup/manager.go

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package cgroup
+
+import (
+	"github.com/containerd/log"
+)
+
+type Manager struct {
+	name   string
+	config Config
+	cgroup DaemonCgroup
+}
+
+type Opt struct {
+	Name   string
+	Config Config
+}
+
+func NewManager(opt Opt) (*Manager, error) {
+	if !supported() {
+		log.L.Warn("cgroup is not supported")
+		return nil, ErrCgroupNotSupported
+	}
+
+	log.L.Infof("cgroup mode: %s", displayMode())
+	cg, err := createCgroup(opt.Name, opt.Config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Manager{
+		name:   opt.Name,
+		config: opt.Config,
+		cgroup: cg,
+	}, nil
+}
+
+// Please make sure the *Manager is not null.
+func (m *Manager) AddProc(pid int) error {
+	return m.cgroup.AddProc(pid)
+}
+
+// Please make sure the *Manager is not null.
+func (m *Manager) Delete() error {
+	return m.cgroup.Delete()
+}

pkg/cgroup/v1/v1.go

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package v1
+
+import (
+	"github.com/containerd/cgroups/v3/cgroup1"
+	"github.com/containerd/log"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+)
+
+type Cgroup struct {
+	controller cgroup1.Cgroup
+}
+
+func generateHierarchy() cgroup1.Hierarchy {
+	return cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory)
+}
+
+func NewCgroup(slice, name string, memoryLimitInBytes int64) (Cgroup, error) {
+	hierarchy := generateHierarchy()
+	specResources := &specs.LinuxResources{
+		Memory: &specs.LinuxMemory{
+			Limit: &memoryLimitInBytes,
+		},
+	}
+
+	controller, err := cgroup1.Load(cgroup1.Slice(slice, name), cgroup1.WithHiearchy(hierarchy))
+	if err != nil && err != cgroup1.ErrCgroupDeleted {
+		return Cgroup{}, err
+	}
+
+	if controller != nil {
+		processes, err := controller.Processes(cgroup1.Memory, true)
+		if err != nil {
+			return Cgroup{}, err
+		}
+		if len(processes) > 0 {
+			log.L.Infof("target cgroup is existed with processes %v", processes)
+			if err := controller.Update(specResources); err != nil {
+				return Cgroup{}, err
+			}
+			return Cgroup{
+				controller: controller,
+			}, nil
+		}
+		if err := controller.Delete(); err != nil {
+			return Cgroup{}, err
+		}
+	}
+
+	controller, err = cgroup1.New(cgroup1.Slice(slice, name), specResources, cgroup1.WithHiearchy(hierarchy))
+	if err != nil {
+		return Cgroup{}, errors.Wrapf(err, "create cgroup")
+	}
+	log.L.Infof("create cgroup (v1) successful, state: %v", controller.State())
+
+	return Cgroup{
+		controller: controller,
+	}, nil
+}
+
+func (cg Cgroup) Delete() error {
+	processes, err := cg.controller.Processes(cgroup1.Memory, true)
+	if err != nil {
+		return err
+	}
+	if len(processes) > 0 {
+		log.L.Infof("skip destroy cgroup because of running daemon %v", processes)
+		return nil
+	}
+	return cg.controller.Delete()
+}
+func (cg Cgroup) AddProc(pid int) error {
+	err := cg.controller.AddProc(uint64(pid), cgroup1.Memory)
+	if err != nil {
+		return err
+	}
+
+	log.L.Infof("add process %d to daemon cgroup successful", pid)
+	return nil
+}

pkg/cgroup/v2/v2.go

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package v2
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/containerd/cgroups/v3/cgroup2"
+	"github.com/containerd/log"
+	"golang.org/x/exp/slices"
+)
+
+const (
+	defaultRoot = "/sys/fs/cgroup"
+)
+
+var (
+	ErrRootMemorySubtreeControllerDisabled = errors.New("cgroups v2: root subtree controller for memory is disabled")
+)
+
+type Cgroup struct {
+	manager *cgroup2.Manager
+}
+
+func readSubtreeControllers(dir string) ([]string, error) {
+	b, err := os.ReadFile(filepath.Join(dir, "cgroup.subtree_control"))
+	if err != nil {
+		return nil, err
+	}
+	return strings.Fields(string(b)), nil
+}
+
+func NewCgroup(slice, name string, memoryLimitInBytes int64) (Cgroup, error) {
+	resources := &cgroup2.Resources{
+		Memory: &cgroup2.Memory{},
+	}
+	if memoryLimitInBytes > -1 {
+		resources = &cgroup2.Resources{
+			Memory: &cgroup2.Memory{
+				Max: &memoryLimitInBytes,
+			},
+		}
+	}
+
+	rootSubtreeControllers, err := readSubtreeControllers(defaultRoot)
+	if err != nil {
+		return Cgroup{}, err
+	}
+	log.L.Infof("root subtree controllers: %s", rootSubtreeControllers)
+
+	if !slices.Contains(rootSubtreeControllers, "memory") {
+		return Cgroup{}, ErrRootMemorySubtreeControllerDisabled
+	}
+
+	m, err := cgroup2.NewManager(defaultRoot, fmt.Sprintf("/%s/%s", slice, name), resources)
+	if err != nil {
+		return Cgroup{}, err
+	}
+
+	controllers, err := m.Controllers()
+	if err != nil {
+		return Cgroup{}, err
+	}
+	log.L.Infof("create cgroup (v2) successful, controllers: %v", controllers)
+
+	return Cgroup{
+		manager: m,
+	}, nil
+}
+
+func (cg Cgroup) Delete() error {
+	if cg.manager != nil {
+		return cg.manager.Delete()
+	}
+	return nil
+}
+func (cg Cgroup) AddProc(pid int) error {
+	if cg.manager != nil {
+		err := cg.manager.AddProc(uint64(pid))
+		if err != nil {
+			return err
+		}
+		log.L.Infof("add process %d to daemon cgroup successful", pid)
+	}
+	return nil
+}

pkg/converter/constant.go

@@ -19,6 +19,9 @@ const (
 	LayerAnnotationNydusBlobSize      = "containerd.io/snapshot/nydus-blob-size"
 	LayerAnnotationNydusBootstrap     = "containerd.io/snapshot/nydus-bootstrap"
 	LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid"
+	LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob"
+	LayerAnnotationNydusSourceDigest  = "containerd.io/snapshot/nydus-source-digest"
+	LayerAnnotationNydusTargetDigest  = "containerd.io/snapshot/nydus-target-digest"
 
 	LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids"
 

pkg/converter/convert_unix.go

@@ -22,13 +22,14 @@ import (
 	"sync"
 	"syscall"
 
-	"github.com/containerd/containerd/archive"
-	"github.com/containerd/containerd/archive/compression"
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/images/converter"
-	"github.com/containerd/containerd/labels"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/containerd/containerd/v2/pkg/archive"
+	"github.com/containerd/containerd/v2/pkg/archive/compression"
+	"github.com/containerd/containerd/v2/pkg/labels"
+	"github.com/containerd/containerd/v2/plugins/content/local"
+	"github.com/containerd/errdefs"
 	"github.com/containerd/fifo"
 	"github.com/klauspost/compress/zstd"
 	"github.com/opencontainers/go-digest"
@@ -108,7 +109,7 @@ func unpackOciTar(ctx context.Context, dst string, reader io.Reader) error {
 		ctx,
 		dst,
 		ds,
-		archive.WithConvertWhiteout(func(hdr *tar.Header, file string) (bool, error) {
+		archive.WithConvertWhiteout(func(_ *tar.Header, _ string) (bool, error) {
 			// Keep to extract all whiteout files.
 			return true, nil
 		}),
@@ -122,7 +123,7 @@ func unpackOciTar(ctx context.Context, dst string, reader io.Reader) error {
 // unpackNydusBlob unpacks a Nydus formatted tar stream into a directory.
 // unpackBlob indicates whether to unpack blob data.
 func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bool) error {
-	boot, err := os.OpenFile(bootDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	boot, err := os.OpenFile(bootDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
 	if err != nil {
 		return errors.Wrapf(err, "write to bootstrap %s", bootDst)
 	}
@@ -133,13 +134,18 @@ func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bo
 	}
 
 	if unpackBlob {
-		blob, err := os.OpenFile(blobDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+		blob, err := os.OpenFile(blobDst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640)
 		if err != nil {
 			return errors.Wrapf(err, "write to blob %s", blobDst)
 		}
 		defer blob.Close()
 
 		if _, err = UnpackEntry(ra, EntryBlob, blob); err != nil {
+			if errors.Is(err, ErrNotFound) {
+				// The nydus layer may contain only bootstrap and no blob
+				// data, which should be ignored.
+				return nil
+			}
 			return errors.Wrap(err, "unpack blob from nydus")
 		}
 	}
@@ -147,7 +153,7 @@ func unpackNydusBlob(bootDst, blobDst string, ra content.ReaderAt, unpackBlob bo
 	return nil
 }
 
-func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.Reader, *tar.Header) error) error {
+func seekFileByTarHeader(ra content.ReaderAt, targetName string, maxSize *int64, handle func(io.Reader, *tar.Header) error) error {
 	const headerSize = 512
 
 	if headerSize > ra.Size() {
@@ -178,6 +184,10 @@ func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.
 		}
 
 		if hdr.Name == targetName {
+			if maxSize != nil && hdr.Size > *maxSize {
+				return fmt.Errorf("invalid nydus tar size %d", ra.Size())
+			}
+
 			// Try to seek the part of tar data.
 			_, err = reader.Seek(cur-hdr.Size, io.SeekStart)
 			if err != nil {
@@ -203,9 +213,10 @@ func seekFileByTarHeader(ra content.ReaderAt, targetName string, handle func(io.
 
 func seekFileByTOC(ra content.ReaderAt, targetName string, handle func(io.Reader, *tar.Header) error) (*TOCEntry, error) {
 	entrySize := 128
+	maxSize := int64(1 << 20)
 	var tocEntry *TOCEntry
 
-	err := seekFileByTarHeader(ra, EntryTOC, func(tocEntryDataReader io.Reader, _ *tar.Header) error {
+	err := seekFileByTarHeader(ra, EntryTOC, &maxSize, func(tocEntryDataReader io.Reader, _ *tar.Header) error {
 		entryData, err := io.ReadAll(tocEntryDataReader)
 		if err != nil {
 			return errors.Wrap(err, "read toc entries")
@@ -291,7 +302,7 @@ func seekFile(ra content.ReaderAt, targetName string, handle func(io.Reader, *ta
 	}
 
 	// Seek target data by tar header, ensure compatible with old rafs blob format.
-	return nil, seekFileByTarHeader(ra, targetName, handle)
+	return nil, seekFileByTarHeader(ra, targetName, nil, handle)
 }
 
 // Pack converts an OCI tar stream to nydus formatted stream with a tar-like
@@ -311,7 +322,20 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 	}
 
 	builderPath := getBuilder(opt.BuilderPath)
-	opt.features = tool.DetectFeatures(builderPath, []tool.Feature{tool.FeatureTar2Rafs})
+
+	requiredFeatures := tool.NewFeatures(tool.FeatureTar2Rafs)
+	if opt.BatchSize != "" && opt.BatchSize != "0" {
+		requiredFeatures.Add(tool.FeatureBatchSize)
+	}
+	if opt.Encrypt {
+		requiredFeatures.Add(tool.FeatureEncrypt)
+	}
+
+	detectedFeatures, err := tool.DetectFeatures(builderPath, requiredFeatures, tool.GetHelp)
+	if err != nil {
+		return nil, err
+	}
+	opt.features = detectedFeatures
 
 	if opt.OCIRef {
 		if opt.FsVersion == "6" {
@@ -320,10 +344,18 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 		return nil, fmt.Errorf("oci ref can only be supported by fs version 6")
 	}
 
+	if opt.features.Contains(tool.FeatureBatchSize) && opt.FsVersion != "6" {
+		return nil, fmt.Errorf("'--batch-size' can only be supported by fs version 6")
+	}
+
 	if opt.features.Contains(tool.FeatureTar2Rafs) {
 		return packFromTar(ctx, dest, opt)
 	}
 
+	return packFromDirectory(ctx, dest, opt, builderPath)
+}
+
+func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, builderPath string) (io.WriteCloser, error) {
 	workDir, err := ensureWorkDir(opt.WorkDir)
 	if err != nil {
 		return nil, errors.Wrap(err, "ensure work directory")
@@ -360,7 +392,7 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 		<-unpackDone
 
 		blobPath := filepath.Join(workDir, "blob")
-		blobFifo, err := fifo.OpenFifo(ctx, blobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
+		blobFifo, err := fifo.OpenFifo(ctx, blobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
 		if err != nil {
 			return errors.Wrapf(err, "create fifo file")
 		}
@@ -377,8 +409,10 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 				PrefetchPatterns: opt.PrefetchPatterns,
 				AlignedChunk:     opt.AlignedChunk,
 				ChunkSize:        opt.ChunkSize,
+				BatchSize:        opt.BatchSize,
 				Compressor:       opt.Compressor,
 				Timeout:          opt.Timeout,
+				Encrypt:          opt.Encrypt,
 
 				Features: opt.features,
 			})
@@ -412,13 +446,13 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
 	}()
 
 	rafsBlobPath := filepath.Join(workDir, "blob.rafs")
-	rafsBlobFifo, err := fifo.OpenFifo(ctx, rafsBlobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
+	rafsBlobFifo, err := fifo.OpenFifo(ctx, rafsBlobPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
 	if err != nil {
 		return nil, errors.Wrapf(err, "create fifo file")
 	}
 
 	tarBlobPath := filepath.Join(workDir, "blob.targz")
-	tarBlobFifo, err := fifo.OpenFifo(ctx, tarBlobPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_NONBLOCK, 0644)
+	tarBlobFifo, err := fifo.OpenFifo(ctx, tarBlobPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_NONBLOCK, 0640)
 	if err != nil {
 		defer rafsBlobFifo.Close()
 		return nil, errors.Wrapf(err, "create fifo file")
@@ -479,8 +513,10 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
 				PrefetchPatterns: opt.PrefetchPatterns,
 				AlignedChunk:     opt.AlignedChunk,
 				ChunkSize:        opt.ChunkSize,
+				BatchSize:        opt.BatchSize,
 				Compressor:       opt.Compressor,
 				Timeout:          opt.Timeout,
+				Encrypt:          opt.Encrypt,
 
 				Features: opt.features,
 			})
@@ -497,8 +533,9 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
 }
 
 func calcBlobTOCDigest(ra content.ReaderAt) (*digest.Digest, error) {
+	maxSize := int64(1 << 20)
 	digester := digest.Canonical.Digester()
-	if err := seekFileByTarHeader(ra, EntryTOC, func(tocData io.Reader, _ *tar.Header) error {
+	if err := seekFileByTarHeader(ra, EntryTOC, &maxSize, func(tocData io.Reader, _ *tar.Header) error {
 		if _, err := io.Copy(digester.Hash(), tocData); err != nil {
 			return errors.Wrap(err, "calc toc data and header digest")
 		}
@@ -537,13 +574,13 @@ func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption)
 	for idx := range layers {
 		sourceBootstrapPaths = append(sourceBootstrapPaths, getBootstrapPath(idx))
 		if layers[idx].OriginalDigest != nil {
-			rafsBlobDigests = append(rafsBlobDigests, layers[idx].Digest.Hex())
-			rafsBlobSizes = append(rafsBlobSizes, layers[idx].ReaderAt.Size())
 			rafsBlobTOCDigest, err := calcBlobTOCDigest(layers[idx].ReaderAt)
 			if err != nil {
 				return nil, errors.Wrapf(err, "calc blob toc digest for layer %s", layers[idx].Digest)
 			}
 			rafsBlobTOCDigests = append(rafsBlobTOCDigests, rafsBlobTOCDigest.Hex())
+			rafsBlobDigests = append(rafsBlobDigests, layers[idx].Digest.Hex())
+			rafsBlobSizes = append(rafsBlobSizes, layers[idx].ReaderAt.Size())
 		}
 		eg.Go(func(idx int) func() error {
 			return func() error {
@@ -588,13 +625,23 @@ func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption)
 		return nil, errors.Wrap(err, "merge bootstrap")
 	}
 
+	bootstrapRa, err := local.OpenReader(targetBootstrapPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "open bootstrap reader")
+	}
+	defer bootstrapRa.Close()
+
+	files := append([]File{
+		{
+			Name:   EntryBootstrap,
+			Reader: content.NewReader(bootstrapRa),
+			Size:   bootstrapRa.Size(),
+		},
+	}, opt.AppendFiles...)
 	var rc io.ReadCloser
 
 	if opt.WithTar {
-		rc, err = packToTar(targetBootstrapPath, fmt.Sprintf("image/%s", EntryBootstrap), false)
-		if err != nil {
-			return nil, errors.Wrap(err, "pack bootstrap to tar")
-		}
+		rc = packToTar(files, false)
 	} else {
 		rc, err = os.Open(targetBootstrapPath)
 		if err != nil {
@@ -626,7 +673,7 @@ func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt Unpack
 	}
 
 	tarPath := filepath.Join(workDir, "oci.tar")
-	blobFifo, err := fifo.OpenFifo(ctx, tarPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0644)
+	blobFifo, err := fifo.OpenFifo(ctx, tarPath, syscall.O_CREAT|syscall.O_RDONLY|syscall.O_NONBLOCK, 0640)
 	if err != nil {
 		return errors.Wrapf(err, "create fifo file")
 	}
@@ -650,7 +697,7 @@ func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt Unpack
 		// generate backend config file
 		backendConfigStr := fmt.Sprintf(`{"version":2,"backend":{"type":"http-proxy","http-proxy":{"addr":"%s"}}}`, proxy.socketPath)
 		backendConfigPath := filepath.Join(workDir, "backend-config.json")
-		if err := os.WriteFile(backendConfigPath, []byte(backendConfigStr), 0644); err != nil {
+		if err := os.WriteFile(backendConfigPath, []byte(backendConfigStr), 0640); err != nil {
 			return errors.Wrap(err, "write backend config")
 		}
 		unpackOpt.BlobPath = ""
@@ -687,11 +734,11 @@ func IsNydusBlobAndExists(ctx context.Context, cs content.Store, desc ocispec.De
 		return false
 	}
 
-	return IsNydusBlob(ctx, desc)
+	return IsNydusBlob(desc)
 }
 
-// IsNydusBlob returns true when the specified descriptor is nydus blob format.
-func IsNydusBlob(ctx context.Context, desc ocispec.Descriptor) bool {
+// IsNydusBlob returns true when the specified descriptor is nydus blob layer.
+func IsNydusBlob(desc ocispec.Descriptor) bool {
 	if desc.Annotations == nil {
 		return false
 	}
@@ -700,6 +747,70 @@ func IsNydusBlob(ctx context.Context, desc ocispec.Descriptor) bool {
 	return hasAnno
 }
 
+// IsNydusBootstrap returns true when the specified descriptor is nydus bootstrap layer.
+func IsNydusBootstrap(desc ocispec.Descriptor) bool {
+	if desc.Annotations == nil {
+		return false
+	}
+
+	_, hasAnno := desc.Annotations[LayerAnnotationNydusBootstrap]
+	return hasAnno
+}
+
+// isNydusImage checks if the last layer is nydus bootstrap,
+// so that we can ensure it is a nydus image.
+func isNydusImage(manifest *ocispec.Manifest) bool {
+	layers := manifest.Layers
+	if len(layers) != 0 {
+		desc := layers[len(layers)-1]
+		if IsNydusBootstrap(desc) {
+			return true
+		}
+	}
+	return false
+}
+
+// makeBlobDesc returns a ocispec.Descriptor by the given information.
+func makeBlobDesc(ctx context.Context, cs content.Store, opt PackOption, sourceDigest, targetDigest digest.Digest) (*ocispec.Descriptor, error) {
+	targetInfo, err := cs.Info(ctx, targetDigest)
+	if err != nil {
+		return nil, errors.Wrapf(err, "get target blob info %s", targetDigest)
+	}
+	if targetInfo.Labels == nil {
+		targetInfo.Labels = map[string]string{}
+	}
+	// Write a diff id label of layer in content store for simplifying
+	// diff id calculation to speed up the conversion.
+	// See: https://github.com/containerd/containerd/blob/e4fefea5544d259177abb85b64e428702ac49c97/images/diffid.go#L49
+	targetInfo.Labels[labels.LabelUncompressed] = targetDigest.String()
+	_, err = cs.Update(ctx, targetInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "update layer label")
+	}
+
+	targetDesc := ocispec.Descriptor{
+		Digest:    targetDigest,
+		Size:      targetInfo.Size,
+		MediaType: MediaTypeNydusBlob,
+		Annotations: map[string]string{
+			// Use `containerd.io/uncompressed` to generate DiffID of
+			// layer defined in OCI spec.
+			LayerAnnotationUncompressed: targetDigest.String(),
+			LayerAnnotationNydusBlob:    "true",
+		},
+	}
+
+	if opt.OCIRef {
+		targetDesc.Annotations[label.NydusRefLayer] = sourceDigest.String()
+	}
+
+	if opt.Encrypt {
+		targetDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
+	}
+
+	return &targetDesc, nil
+}
+
 // LayerConvertFunc returns a function which converts an OCI image layer to
 // a nydus blob layer, and set the media type to "application/vnd.oci.image.layer.nydus.blob.v1".
 func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
@@ -708,6 +819,20 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
 			return nil, nil
 		}
 
+		// Skip the conversion of nydus layer.
+		if IsNydusBlob(desc) || IsNydusBootstrap(desc) {
+			return nil, nil
+		}
+
+		// Use remote cache to avoid unnecessary conversion
+		info, err := cs.Info(ctx, desc.Digest)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get blob info %s", desc.Digest)
+		}
+		if targetDigest := digest.Digest(info.Labels[LayerAnnotationNydusTargetDigest]); targetDigest.Validate() == nil {
+			return makeBlobDesc(ctx, cs, opt, desc.Digest, targetDigest)
+		}
+
 		ra, err := cs.ReaderAt(ctx, desc)
 		if err != nil {
 			return nil, errors.Wrap(err, "get source blob reader")
@@ -762,45 +887,18 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
 		}
 
 		blobDigest := digester.Digest()
-		info, err := cs.Info(ctx, blobDigest)
+		newDesc, err := makeBlobDesc(ctx, cs, opt, desc.Digest, blobDigest)
 		if err != nil {
-			return nil, errors.Wrapf(err, "get blob info %s", blobDigest)
-		}
-		if info.Labels == nil {
-			info.Labels = map[string]string{}
-		}
-		// Write a diff id label of layer in content store for simplifying
-		// diff id calculation to speed up the conversion.
-		// See: https://github.com/containerd/containerd/blob/e4fefea5544d259177abb85b64e428702ac49c97/images/diffid.go#L49
-		info.Labels[labels.LabelUncompressed] = blobDigest.String()
-		_, err = cs.Update(ctx, info)
-		if err != nil {
-			return nil, errors.Wrap(err, "update layer label")
-		}
-
-		newDesc := ocispec.Descriptor{
-			Digest:    blobDigest,
-			Size:      info.Size,
-			MediaType: MediaTypeNydusBlob,
-			Annotations: map[string]string{
-				// Use `containerd.io/uncompressed` to generate DiffID of
-				// layer defined in OCI spec.
-				LayerAnnotationUncompressed: blobDigest.String(),
-				LayerAnnotationNydusBlob:    "true",
-			},
-		}
-
-		if opt.OCIRef {
-			newDesc.Annotations[label.NydusRefLayer] = desc.Digest.String()
+			return nil, err
 		}
 
 		if opt.Backend != nil {
-			if err := opt.Backend.Push(ctx, cs, newDesc); err != nil {
+			if err := opt.Backend.Push(ctx, cs, *newDesc); err != nil {
 				return nil, errors.Wrap(err, "push to storage backend")
 			}
 		}
 
-		return &newDesc, nil
+		return newDesc, nil
 	}
 }
 
@@ -809,59 +907,41 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
 // the index conversion and the manifest conversion.
 func ConvertHookFunc(opt MergeOption) converter.ConvertHookFunc {
 	return func(ctx context.Context, cs content.Store, orgDesc ocispec.Descriptor, newDesc *ocispec.Descriptor) (*ocispec.Descriptor, error) {
+		// If the previous conversion did not occur, the `newDesc` may be nil.
+		if newDesc == nil {
+			return &orgDesc, nil
+		}
 		switch {
 		case images.IsIndexType(newDesc.MediaType):
-			return convertIndex(ctx, cs, orgDesc, newDesc)
+			return convertIndex(ctx, cs, newDesc)
 		case images.IsManifestType(newDesc.MediaType):
-			return convertManifest(ctx, cs, newDesc, opt)
+			return convertManifest(ctx, cs, orgDesc, newDesc, opt)
 		default:
 			return newDesc, nil
 		}
 	}
 }
 
-// convertIndex modifies the original index by appending "nydus.remoteimage.v1"
-// to the Platform.OSFeatures of each modified manifest descriptors.
-func convertIndex(ctx context.Context, cs content.Store, orgDesc ocispec.Descriptor, newDesc *ocispec.Descriptor) (*ocispec.Descriptor, error) {
-	var orgIndex ocispec.Index
-	if _, err := readJSON(ctx, cs, &orgIndex, orgDesc); err != nil {
-		return nil, errors.Wrap(err, "read target image index json")
-	}
-	// isManifestModified is a function to check whether the manifest is modified.
-	isManifestModified := func(manifest ocispec.Descriptor) bool {
-		for _, oldManifest := range orgIndex.Manifests {
-			if manifest.Digest == oldManifest.Digest {
-				return false
-			}
-		}
-		return true
-	}
-
+// convertIndex modifies the original index converting it to manifest directly if it contains only one manifest.
+func convertIndex(ctx context.Context, cs content.Store, newDesc *ocispec.Descriptor) (*ocispec.Descriptor, error) {
 	var index ocispec.Index
-	indexLabels, err := readJSON(ctx, cs, &index, *newDesc)
+	_, err := readJSON(ctx, cs, &index, *newDesc)
 	if err != nil {
 		return nil, errors.Wrap(err, "read index json")
 	}
-	for i, manifest := range index.Manifests {
-		if !isManifestModified(manifest) {
-			// Skip the manifest which is not modified.
-			continue
-		}
-		manifest.Platform.OSFeatures = append(manifest.Platform.OSFeatures, ManifestOSFeatureNydus)
-		index.Manifests[i] = manifest
+
+	// If the converted manifest list contains only one manifest,
+	// convert it directly to manifest.
+	if len(index.Manifests) == 1 {
+		return &index.Manifests[0], nil
 	}
-	// Update image index in content store.
-	newIndexDesc, err := writeJSON(ctx, cs, index, *newDesc, indexLabels)
-	if err != nil {
-		return nil, errors.Wrap(err, "write index json")
-	}
-	return newIndexDesc, nil
+	return newDesc, nil
 }
 
 // convertManifest merges all the nydus blob layers into a
 // nydus bootstrap layer, update the image config,
 // and modify the image manifest.
-func convertManifest(ctx context.Context, cs content.Store, newDesc *ocispec.Descriptor, opt MergeOption) (*ocispec.Descriptor, error) {
+func convertManifest(ctx context.Context, cs content.Store, oldDesc ocispec.Descriptor, newDesc *ocispec.Descriptor, opt MergeOption) (*ocispec.Descriptor, error) {
 	var manifest ocispec.Manifest
 	manifestDesc := *newDesc
 	manifestLabels, err := readJSON(ctx, cs, &manifest, manifestDesc)
@@ -869,10 +949,20 @@ func convertManifest(ctx context.Context, cs content.Store, newDesc *ocispec.Des
 		return nil, errors.Wrap(err, "read manifest json")
 	}
 
+	if isNydusImage(&manifest) {
+		return &manifestDesc, nil
+	}
+
 	// This option needs to be enabled for image scenario.
 	opt.WithTar = true
 
-	// Append bootstrap layer to manifest.
+	// If the original image is already an OCI type, we should forcibly set the
+	// bootstrap layer to the OCI type.
+	if !opt.OCI && oldDesc.MediaType == ocispec.MediaTypeImageManifest {
+		opt.OCI = true
+	}
+
+	// Append bootstrap layer to manifest, encrypt bootstrap layer if needed.
 	bootstrapDesc, blobDescs, err := MergeLayers(ctx, cs, manifest.Layers, opt)
 	if err != nil {
 		return nil, errors.Wrap(err, "merge nydus layers")
@@ -903,8 +993,13 @@ func convertManifest(ctx context.Context, cs content.Store, newDesc *ocispec.Des
 	if err != nil {
 		return nil, errors.Wrap(err, "read image config")
 	}
+	bootstrapHistory := ocispec.History{
+		CreatedBy: "Nydus Converter",
+		Comment:   "Nydus Bootstrap Layer",
+	}
 	if opt.Backend != nil {
 		config.RootFS.DiffIDs = []digest.Digest{digest.Digest(bootstrapDesc.Annotations[LayerAnnotationUncompressed])}
+		config.History = []ocispec.History{bootstrapHistory}
 	} else {
 		config.RootFS.DiffIDs = make([]digest.Digest, 0, len(manifest.Layers))
 		for i, layer := range manifest.Layers {
@@ -912,6 +1007,9 @@ func convertManifest(ctx context.Context, cs content.Store, newDesc *ocispec.Des
 			// Remove useless annotation.
 			delete(manifest.Layers[i].Annotations, LayerAnnotationUncompressed)
 		}
+		// Append history item for bootstrap layer, to ensure the history consistency.
+		// See https://github.com/distribution/distribution/blob/e5d5810851d1f17a5070e9b6f940d8af98ea3c29/manifest/schema1/config_builder.go#L136
+		config.History = append(config.History, bootstrapHistory)
 	}
 	// Update image config in content store.
 	newConfigDesc, err := writeJSON(ctx, cs, config, manifest.Config, configLabels)
@@ -922,6 +1020,15 @@ func convertManifest(ctx context.Context, cs content.Store, newDesc *ocispec.Des
 	// Update the config gc label
 	manifestLabels[configGCLabelKey] = newConfigDesc.Digest.String()
 
+	if opt.WithReferrer {
+		// Associate a reference to the original OCI manifest.
+		// See the `subject` field description in
+		// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
+		manifest.Subject = &oldDesc
+		// Remove the platform field as it is not supported by certain registries like ECR.
+		manifest.Subject.Platform = nil
+	}
+
 	// Update image manifest in content store.
 	newManifestDesc, err := writeJSON(ctx, cs, manifest, manifestDesc, manifestLabels)
 	if err != nil {
@@ -1042,17 +1149,26 @@ func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descript
 		if opt.OCIRef {
 			blobDesc.Annotations[label.NydusRefLayer] = layers[idx].OriginalDigest.String()
 		}
+
+		if opt.Encrypt != nil {
+			blobDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
+		}
+
 		blobDescs = append(blobDescs, blobDesc)
 	}
 
 	if opt.FsVersion == "" {
 		opt.FsVersion = "6"
 	}
+	mediaType := images.MediaTypeDockerSchema2LayerGzip
+	if opt.OCI {
+		mediaType = ocispec.MediaTypeImageLayerGzip
+	}
 
 	bootstrapDesc := ocispec.Descriptor{
 		Digest:    compressedDgst,
 		Size:      bootstrapInfo.Size,
-		MediaType: ocispec.MediaTypeImageLayerGzip,
+		MediaType: mediaType,
 		Annotations: map[string]string{
 			LayerAnnotationUncompressed: uncompressedDgst.Digest().String(),
 			LayerAnnotationFSVersion:    opt.FsVersion,
@@ -1061,5 +1177,12 @@ func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descript
 		},
 	}
 
+	if opt.Encrypt != nil {
+		// Encrypt the Nydus bootstrap layer.
+		bootstrapDesc, err = opt.Encrypt(ctx, cs, bootstrapDesc)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "encrypt bootstrap layer")
+		}
+	}
 	return &bootstrapDesc, blobDescs, nil
 }

pkg/converter/convert_windows.go

@@ -13,8 +13,9 @@ import (
 	"context"
 	"io"
 
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/images/converter"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -22,11 +23,11 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 	panic("not implemented")
 }
 
-func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) error {
+func Merge(ctx context.Context, layers []Layer, dest io.Writer, opt MergeOption) ([]digest.Digest, error) {
 	panic("not implemented")
 }
 
-func Unpack(ctx context.Context, ia content.ReaderAt, dest io.Writer, opt UnpackOption) error {
+func Unpack(ctx context.Context, ra content.ReaderAt, dest io.Writer, opt UnpackOption) error {
 	panic("not implemented")
 }
 
@@ -34,7 +35,11 @@ func IsNydusBlobAndExists(ctx context.Context, cs content.Store, desc ocispec.De
 	panic("not implemented")
 }
 
-func IsNydusBlob(ctx context.Context, desc ocispec.Descriptor) bool {
+func IsNydusBlob(desc ocispec.Descriptor) bool {
+	panic("not implemented")
+}
+
+func IsNydusBootstrap(desc ocispec.Descriptor) bool {
 	panic("not implemented")
 }
 
@@ -46,6 +51,6 @@ func ConvertHookFunc(opt MergeOption) converter.ConvertHookFunc {
 	panic("not implemented")
 }
 
-func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descriptor, opt MergeOption) (*ocispec.Descriptor, error) {
+func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descriptor, opt MergeOption) (*ocispec.Descriptor, []ocispec.Descriptor, error) {
 	panic("not implemented")
 }

pkg/converter/cs_proxy_unix.go

@@ -20,7 +20,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/v2/core/content"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )

pkg/converter/tool/builder.go

@@ -39,6 +39,8 @@ type PackOption struct {
 	OCIRef           bool
 	AlignedChunk     bool
 	ChunkSize        string
+	BatchSize        string
+	Encrypt          bool
 	Timeout          *time.Duration
 
 	Features Features
@@ -132,6 +134,12 @@ func buildPackArgs(option PackOption) []string {
 	if option.ChunkSize != "" {
 		args = append(args, "--chunk-size", option.ChunkSize)
 	}
+	if option.Features.Contains(FeatureBatchSize) {
+		args = append(args, "--batch-size", option.BatchSize)
+	}
+	if option.Encrypt {
+		args = append(args, "--encrypt")
+	}
 	args = append(args, option.SourcePath)
 
 	return args
@@ -297,7 +305,33 @@ func Unpack(option UnpackOption) error {
 	}
 
 	if option.BackendConfigPath != "" {
-		args = append(args, "--backend-config", option.BackendConfigPath)
+		configBytes, err := os.ReadFile(option.BackendConfigPath)
+		if err != nil {
+			return errors.Wrapf(err, "fail to read backend config file %s", option.BackendConfigPath)
+		}
+
+		var config map[string]interface{}
+		if err := json.Unmarshal(configBytes, &config); err != nil {
+			return errors.Wrapf(err, "fail to unmarshal backend config file %s", option.BackendConfigPath)
+		}
+
+		backendConfigType, ok := config["backend"].(map[string]interface{})["type"]
+		if !ok {
+			return errors.New("backend config file should contain a valid backend type")
+		}
+
+		backendConfig, ok := config["backend"].(map[string]interface{})[backendConfigType.(string)]
+		if !ok {
+			return errors.New("failed to get backend config with type " + backendConfigType.(string))
+		}
+
+		backendConfigBytes, err := json.Marshal(backendConfig)
+		if err != nil {
+			return errors.Wrapf(err, "fail to marshal backend config %v", backendConfig)
+		}
+
+		args = append(args, "--backend-type", backendConfigType.(string))
+		args = append(args, "--backend-config", string(backendConfigBytes))
 	} else if option.BlobPath != "" {
 		args = append(args, "--blob", option.BlobPath)
 	}

pkg/converter/tool/feature.go

@@ -8,20 +8,19 @@ package tool
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/exec"
-	"regexp"
+	"strings"
 	"sync"
 
 	"github.com/sirupsen/logrus"
-	"golang.org/x/mod/semver"
 )
 
-const envNydusDisableTar2Rafs = "NYDUS_DISABLE_TAR2RAFS"
+type Feature string
+type Features map[Feature]struct{}
 
-var currentVersion string
-var currentVersionDetectOnce sync.Once
-var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != ""
+const envNydusDisableTar2Rafs string = "NYDUS_DISABLE_TAR2RAFS"
 
 const (
 	// The option `--type tar-rafs` enables converting OCI tar blob
@@ -29,85 +28,119 @@ const (
 	// need to decompress it to a local directory first, thus greatly
 	// accelerating the pack process.
 	FeatureTar2Rafs Feature = "--type tar-rafs"
+	// The option `--batch-size` enables merging multiple small chunks
+	// into a big batch chunk, which can reduce the the size of the image
+	// and accelerate the runtime file loading.
+	FeatureBatchSize Feature = "--batch-size"
+	// The option `--encrypt` enables converting directories, tar files
+	// or OCI images into encrypted nydus blob.
+	FeatureEncrypt Feature = "--encrypt"
 )
 
-var featureMap = map[Feature]string{
-	FeatureTar2Rafs: "v2.2",
+var requiredFeatures Features
+var detectedFeatures Features
+var detectFeaturesOnce sync.Once
+var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != ""
+
+func NewFeatures(items ...Feature) Features {
+	features := Features{}
+	features.Add(items...)
+	return features
 }
 
-type Feature string
-type Features []Feature
+func (features *Features) Add(items ...Feature) {
+	for _, item := range items {
+		(*features)[item] = struct{}{}
+	}
+}
+
+func (features *Features) Remove(items ...Feature) {
+	for _, item := range items {
+		delete(*features, item)
+	}
+}
 
 func (features *Features) Contains(feature Feature) bool {
-	for _, feat := range *features {
-		if feat == feature {
+	_, ok := (*features)[feature]
+	return ok
+}
+
+func (features *Features) Equals(other Features) bool {
+	if len(*features) != len(other) {
+		return false
+	}
+
+	for f := range *features {
+		if !other.Contains(f) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// GetHelp returns the help message of `nydus-image create`.
+func GetHelp(builder string) []byte {
+	cmd := exec.CommandContext(context.Background(), builder, "create", "-h")
+	output, err := cmd.Output()
+	if err != nil {
+		return nil
+	}
+
+	return output
+}
+
+// detectFeature returns true if the feature is detected in the help message.
+func detectFeature(msg []byte, feature Feature) bool {
+	if feature == "" {
+		return false
+	}
+
+	if strings.Contains(string(msg), string(feature)) {
+		return true
+	}
+
+	if parts := strings.Split(string(feature), " "); len(parts) == 2 {
+		// Check each part of the feature.
+		// e.g., "--type tar-rafs" -> ["--type", "tar-rafs"]
+		if strings.Contains(string(msg), parts[0]) && strings.Contains(string(msg), parts[1]) {
 			return true
 		}
 	}
+
 	return false
 }
 
-func (features *Features) Remove(feature Feature) {
-	found := -1
-	for idx, feat := range *features {
-		if feat == feature {
-			found = idx
-			break
-		}
-	}
-	if found != -1 {
-		*features = append((*features)[:found], (*features)[found+1:]...)
-	}
-}
-
-func detectVersion(msg []byte) string {
-	re := regexp.MustCompile(`Version:\s*v*(\d+.\d+.\d+)`)
-	matches := re.FindSubmatch(msg)
-	if len(matches) > 1 {
-		return string(matches[1])
-	}
-	return ""
-}
-
 // DetectFeatures returns supported feature list from required feature list.
-func DetectFeatures(builder string, required Features) Features {
-	currentVersionDetectOnce.Do(func() {
-		if required.Contains(FeatureTar2Rafs) && disableTar2Rafs {
-			logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs)
-		}
+// The supported feature list is detected from the help message of `nydus-image create`.
+func DetectFeatures(builder string, required Features, getHelp func(string) []byte) (Features, error) {
+	detectFeaturesOnce.Do(func() {
+		requiredFeatures = required
+		detectedFeatures = Features{}
 
-		cmd := exec.CommandContext(context.Background(), builder, "--version")
-		output, err := cmd.Output()
-		if err != nil {
-			return
-		}
+		helpMsg := getHelp(builder)
 
-		currentVersion = detectVersion(output)
+		for feature := range required {
+			// The feature is supported by current version of nydus-image.
+			supported := detectFeature(helpMsg, feature)
+			if supported {
+				// It is an experimental feature, so we still provide an env
+				// variable to allow users to disable it.
+				if feature == FeatureTar2Rafs && disableTar2Rafs {
+					logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs)
+					continue
+				}
+				detectedFeatures.Add(feature)
+			} else {
+				logrus.Warnf("the feature '%s' is ignored, it requires higher version of nydus-image", feature)
+			}
+		}
 	})
 
-	if currentVersion == "" {
-		return required
+	// Return Error if required features changed in different calls.
+	if !requiredFeatures.Equals(required) {
+		return nil, fmt.Errorf("features changed: %v -> %v", requiredFeatures, required)
 	}
 
-	detectedFeatures := Features{}
-	for _, feature := range required {
-		requiredVersion := featureMap[feature]
-		if requiredVersion == "" {
-			detectedFeatures = append(detectedFeatures, feature)
-			continue
-		}
-
-		// The feature is supported by current version
-		supported := semver.Compare(requiredVersion, "v"+currentVersion) <= 0
-		if supported {
-			// It is an experimental feature, so we still provide an env
-			// variable to allow users to disable it.
-			if feature == FeatureTar2Rafs && disableTar2Rafs {
-				continue
-			}
-			detectedFeatures = append(detectedFeatures, feature)
-		}
-	}
-
-	return detectedFeatures
+	return detectedFeatures, nil
 }

pkg/converter/tool/feature_test.go

@@ -7,41 +7,830 @@
 package tool
 
 import (
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
 func TestFeature(t *testing.T) {
-	features := Features{FeatureTar2Rafs}
-	require.True(t, features.Contains(FeatureTar2Rafs))
+	testsAdd := []struct {
+		name     string
+		features Features
+		items    []Feature
+		expect   Features
+	}{
+		{
+			name:     "should successfully add items",
+			features: Features{FeatureBatchSize: {}},
+			items:    []Feature{FeatureTar2Rafs},
+			expect:   Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}},
+		},
+		{
+			name:     "should add nothing if duplicated",
+			features: Features{FeatureBatchSize: {}},
+			items:    []Feature{FeatureBatchSize},
+			expect:   Features{FeatureBatchSize: {}},
+		},
+		{
+			name:     "add should accept nil",
+			features: Features{FeatureBatchSize: {}},
+			items:    nil,
+			expect:   Features{FeatureBatchSize: {}},
+		},
+	}
+	for _, tt := range testsAdd {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.features.Add(tt.items...)
+			require.Equal(t, tt.expect, tt.features)
+		})
+	}
 
-	features.Remove(FeatureTar2Rafs)
-	require.False(t, features.Contains(FeatureTar2Rafs))
+	testsNew := []struct {
+		name   string
+		items  []Feature
+		expect Features
+	}{
+		{
+			name:   "should successfully new Features",
+			items:  []Feature{FeatureTar2Rafs, FeatureBatchSize},
+			expect: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}},
+		},
+		{
+			name:   "should duplicate same items",
+			items:  []Feature{FeatureBatchSize, FeatureBatchSize},
+			expect: Features{FeatureBatchSize: {}},
+		},
+		{
+			name:   "New should accept nil",
+			items:  nil,
+			expect: Features{},
+		},
+	}
+	for _, tt := range testsNew {
+		t.Run(tt.name, func(t *testing.T) {
+			features := NewFeatures(tt.items...)
+			require.Equal(t, tt.expect, features)
+		})
+	}
+
+	testsRemove := []struct {
+		name     string
+		features Features
+		items    []Feature
+		expect   Features
+	}{
+		{
+			name:     "should successfully remove items",
+			features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}},
+			items:    []Feature{FeatureTar2Rafs},
+			expect:   Features{FeatureBatchSize: {}},
+		},
+		{
+			name:     "should remove item iff exists",
+			features: Features{FeatureBatchSize: {}},
+			items:    []Feature{FeatureBatchSize, FeatureTar2Rafs},
+			expect:   Features{},
+		},
+		{
+			name:     "Remove should accept nil",
+			features: Features{FeatureBatchSize: {}},
+			items:    nil,
+			expect:   Features{FeatureBatchSize: {}},
+		},
+	}
+	for _, tt := range testsRemove {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.features.Remove(tt.items...)
+			require.Equal(t, tt.expect, tt.features)
+		})
+	}
+
+	testsContains := []struct {
+		name     string
+		features Features
+		item     Feature
+		expect   bool
+	}{
+		{
+			name:     "should return contains",
+			features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}},
+			item:     FeatureTar2Rafs,
+			expect:   true,
+		},
+		{
+			name:     "should return not contains",
+			features: Features{FeatureBatchSize: {}},
+			item:     FeatureTar2Rafs,
+			expect:   false,
+		},
+		{
+			name:     "Contains should accept empty string",
+			features: Features{FeatureBatchSize: {}},
+			item:     "",
+			expect:   false,
+		},
+	}
+	for _, tt := range testsContains {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.expect, tt.features.Contains(tt.item))
+		})
+	}
+
+	testsEquals := []struct {
+		name     string
+		features Features
+		other    Features
+		expect   bool
+	}{
+		{
+			name:     "should successfully check equality",
+			features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}},
+			other:    Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}},
+			expect:   true,
+		},
+		{
+			name:     "should successfully check inequality with different length",
+			features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}},
+			other:    Features{FeatureBatchSize: {}},
+			expect:   false,
+		},
+		{
+			name:     "should successfully check inequality with different items",
+			features: Features{FeatureTar2Rafs: {}},
+			other:    Features{FeatureBatchSize: {}},
+			expect:   false,
+		},
+		{
+			name:     "should ignore order",
+			features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}},
+			other:    Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}},
+			expect:   true,
+		},
+		{
+			name:     "Equals should accept nil",
+			features: Features{FeatureBatchSize: {}},
+			other:    nil,
+			expect:   false,
+		},
+	}
+	for _, tt := range testsEquals {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.expect, tt.features.Equals(tt.other))
+		})
+	}
 }
 
-func TestVersion(t *testing.T) {
-	require.Equal(t, "0.1.0", detectVersion([]byte(`
-	Version: 	0.1.0
-	Git Commit: 	57a5ae40e91f82eb9d1e9934dee98358bcf822eb
-	Build Time: 	Fri, 19 Mar 2021 10:45:00 +0000
-	Profile: 	release
-	Rustc: 		rustc 1.49.0 (e1884a8e3 2020-12-29)
-	`)))
+func TestDetectFeature(t *testing.T) {
+	tests := []struct {
+		name    string
+		feature Feature
+		helpMsg []byte
+		expect  bool
+	}{
+		{
+			name:    "'--type tar-rafs' is supported in v2.2.0-239-gf5c08fcf",
+			feature: FeatureTar2Rafs,
+			expect:  true,
+			helpMsg: []byte(`
+			Create RAFS filesystems from directories, tar files or OCI images
 
-	require.Equal(t, "2.1.3", detectVersion([]byte(`
-	Version: 	v2.1.3-rc1
-	Git Commit: 	24c3bb9ab213ab94dfbf9ba4106042b34034a390
-	Build Time: 	2023-01-19T02:26:07.782135583Z
-	Profile: 	release
-	Rustc: 		rustc 1.61.0 (fe5b13d68 2022-05-18)
-	`)))
+			Usage: nydus-image create [OPTIONS] <SOURCE>
+			
+			Arguments:
+			  <SOURCE>  source from which to build the RAFS filesystem
+			
+			Options:
+			  -L, --log-file <log-file>
+					  Log file path
+			  -t, --type <type>
+					  Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
+			  -B, --bootstrap <bootstrap>
+					  File path to save the generated RAFS metadata blob
+			  -l, --log-level <log-level>
+					  Log level: [default: info] [possible values: trace, debug, info, warn, error]
+			  -D, --blob-dir <blob-dir>
+					  Directory path to save generated RAFS metadata and data blobs
+			  -b, --blob <blob>
+					  File path to save the generated RAFS data blob
+				  --blob-inline-meta
+					  Inline RAFS metadata and blob metadata into the data blob
+				  --blob-id <blob-id>
+					  OSS object id for the generated RAFS data blob
+				  --blob-data-size <blob-data-size>
+					  Set data blob size for 'estargztoc-ref' conversion
+				  --chunk-size <chunk-size>
+					  Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
+				  --batch-size <batch-size>
+					  Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
+				  --compressor <compressor>
+					  Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
+				  --digester <digester>
+					  Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
+			  -C, --config <config>
+					  Configuration file for storage backend, cache and RAFS FUSE filesystem.
+			  -v, --fs-version <fs-version>
+					  Set RAFS format version number: [default: 6] [possible values: 5, 6]
+				  --features <features>
+					  Enable/disable features [possible values: blob-toc]
+				  --chunk-dict <chunk-dict>
+					  File path of chunk dictionary for data deduplication
+				  --parent-bootstrap <parent-bootstrap>
+					  File path of the parent/referenced RAFS metadata blob (optional)
+				  --aligned-chunk
+					  Align uncompressed data chunks to 4K, only for RAFS V5
+				  --repeatable
+					  Generate reproducible RAFS metadata
+				  --whiteout-spec <whiteout-spec>
+					  Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
+				  --prefetch-policy <prefetch-policy>
+					  Set data prefetch policy [default: none] [possible values: fs, blob, none]
+			  -J, --output-json <output-json>
+					  File path to save operation result in JSON format
+			  -h, --help
+					  Print help information
+			`),
+		},
+		{
+			name:    "'--batch-size' is supported in v2.2.0-239-gf5c08fcf",
+			feature: FeatureBatchSize,
+			expect:  true,
+			helpMsg: []byte(`
+			Create RAFS filesystems from directories, tar files or OCI images
 
-	require.Equal(t, "", detectVersion([]byte(`
-	Version: 	unknown
-	Git Commit: 	96efc2cf7e75174b49942fd41b84d672f921f9b4
-	Build Time: 	2023-02-16T13:20:59.102548977Z
-	Profile: 	release
-	Rustc: 		rustc 1.66.1 (90743e729 2023-01-10)
-	`)))
+			Usage: nydus-image create [OPTIONS] <SOURCE>
+			
+			Arguments:
+			  <SOURCE>  source from which to build the RAFS filesystem
+			
+			Options:
+			  -L, --log-file <log-file>
+					  Log file path
+			  -t, --type <type>
+					  Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
+			  -B, --bootstrap <bootstrap>
+					  File path to save the generated RAFS metadata blob
+			  -l, --log-level <log-level>
+					  Log level: [default: info] [possible values: trace, debug, info, warn, error]
+			  -D, --blob-dir <blob-dir>
+					  Directory path to save generated RAFS metadata and data blobs
+			  -b, --blob <blob>
+					  File path to save the generated RAFS data blob
+				  --blob-inline-meta
+					  Inline RAFS metadata and blob metadata into the data blob
+				  --blob-id <blob-id>
+					  OSS object id for the generated RAFS data blob
+				  --blob-data-size <blob-data-size>
+					  Set data blob size for 'estargztoc-ref' conversion
+				  --chunk-size <chunk-size>
+					  Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
+				  --batch-size <batch-size>
+					  Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
+				  --compressor <compressor>
+					  Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
+				  --digester <digester>
+					  Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
+			  -C, --config <config>
+					  Configuration file for storage backend, cache and RAFS FUSE filesystem.
+			  -v, --fs-version <fs-version>
+					  Set RAFS format version number: [default: 6] [possible values: 5, 6]
+				  --features <features>
+					  Enable/disable features [possible values: blob-toc]
+				  --chunk-dict <chunk-dict>
+					  File path of chunk dictionary for data deduplication
+				  --parent-bootstrap <parent-bootstrap>
+					  File path of the parent/referenced RAFS metadata blob (optional)
+				  --aligned-chunk
+					  Align uncompressed data chunks to 4K, only for RAFS V5
+				  --repeatable
+					  Generate reproducible RAFS metadata
+				  --whiteout-spec <whiteout-spec>
+					  Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
+				  --prefetch-policy <prefetch-policy>
+					  Set data prefetch policy [default: none] [possible values: fs, blob, none]
+			  -J, --output-json <output-json>
+					  File path to save operation result in JSON format
+			  -h, --help
+					  Print help information
+			`),
+		},
+		{
+			name:    "'--batch-size' is not supported in v2.2.0-163-g180f6d2c",
+			feature: FeatureBatchSize,
+			expect:  false,
+			helpMsg: []byte(`
+			Create RAFS filesystems from directories, tar files or OCI images
+
+			Usage: nydus-image create [OPTIONS] <SOURCE>
+			
+			Arguments:
+			  <SOURCE>  source from which to build the RAFS filesystem
+			
+			Options:
+			  -L, --log-file <log-file>
+					  Log file path
+			  -t, --type <type>
+					  Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
+			  -B, --bootstrap <bootstrap>
+					  File path to save the generated RAFS metadata blob
+			  -l, --log-level <log-level>
+					  Log level: [default: info] [possible values: trace, debug, info, warn, error]
+			  -D, --blob-dir <blob-dir>
+					  Directory path to save generated RAFS metadata and data blobs
+			  -b, --blob <blob>
+					  File path to save the generated RAFS data blob
+				  --blob-inline-meta
+					  Inline RAFS metadata and blob metadata into the data blob
+				  --blob-id <blob-id>
+					  OSS object id for the generated RAFS data blob
+				  --blob-data-size <blob-data-size>
+					  Set data blob size for 'estargztoc-ref' conversion
+				  --chunk-size <chunk-size>
+					  Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
+				  --compressor <compressor>
+					  Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
+				  --digester <digester>
+					  Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
+			  -C, --config <config>
+					  Configuration file for storage backend, cache and RAFS FUSE filesystem.
+			  -v, --fs-version <fs-version>
+					  Set RAFS format version number: [default: 6] [possible values: 5, 6]
+				  --features <features>
+					  Enable/disable features [possible values: blob-toc]
+				  --chunk-dict <chunk-dict>
+					  File path of chunk dictionary for data deduplication
+				  --parent-bootstrap <parent-bootstrap>
+					  File path of the parent/referenced RAFS metadata blob (optional)
+				  --aligned-chunk
+					  Align uncompressed data chunks to 4K, only for RAFS V5
+				  --repeatable
+					  Generate reproducible RAFS metadata
+				  --whiteout-spec <whiteout-spec>
+					  Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
+				  --prefetch-policy <prefetch-policy>
+					  Set data prefetch policy [default: none] [possible values: fs, blob, none]
+			  -J, --output-json <output-json>
+					  File path to save operation result in JSON format
+			  -h, --help
+					  Print help information
+			`),
+		},
+		{
+			name:    "'--encrypt' is supported in v2.2.0-261-g22ad0e2c",
+			feature: FeatureEncrypt,
+			expect:  true,
+			helpMsg: []byte(`
+			Create RAFS filesystems from directories, tar files or OCI images
+
+			Usage: nydus-image create [OPTIONS] <SOURCE>
+
+			Arguments:
+			<SOURCE>  source from which to build the RAFS filesystem
+
+			Options:
+			-L, --log-file <log-file>
+					Log file path
+			-t, --type <type>
+					Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
+			-B, --bootstrap <bootstrap>
+					File path to save the generated RAFS metadata blob
+			-l, --log-level <log-level>
+					Log level: [default: info] [possible values: trace, debug, info, warn, error]
+			-D, --blob-dir <blob-dir>
+					Directory path to save generated RAFS metadata and data blobs
+			-b, --blob <blob>
+					File path to save the generated RAFS data blob
+				--blob-inline-meta
+					Inline RAFS metadata and blob metadata into the data blob
+				--blob-id <blob-id>
+					OSS object id for the generated RAFS data blob
+				--blob-data-size <blob-data-size>
+					Set data blob size for 'estargztoc-ref' conversion
+				--chunk-size <chunk-size>
+					Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
+				--batch-size <batch-size>
+					Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
+				--compressor <compressor>
+					Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
+				--digester <digester>
+					Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
+			-C, --config <config>
+					Configuration file for storage backend, cache and RAFS FUSE filesystem.
+			-v, --fs-version <fs-version>
+					Set RAFS format version number: [default: 6] [possible values: 5, 6]
+				--features <features>
+					Enable/disable features [possible values: blob-toc]
+				--chunk-dict <chunk-dict>
+					File path of chunk dictionary for data deduplication
+				--parent-bootstrap <parent-bootstrap>
+					File path of the parent/referenced RAFS metadata blob (optional)
+				--aligned-chunk
+					Align uncompressed data chunks to 4K, only for RAFS V5
+				--repeatable
+					Generate reproducible RAFS metadata
+				--whiteout-spec <whiteout-spec>
+					Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
+				--prefetch-policy <prefetch-policy>
+					Set data prefetch policy [default: none] [possible values: fs, blob, none]
+			-J, --output-json <output-json>
+					File path to save operation result in JSON format
+			-E, --encrypt
+					Encrypt the generated RAFS metadata and data blobs
+			-h, --help
+					Print help information
+			`),
+		},
+
+		{
+			name:    "'--type tar-rafs' is not supported in v2.1.4",
+			feature: FeatureTar2Rafs,
+			expect:  false,
+			helpMsg: []byte(`
+			nydus-image-create 
+			Creates a nydus image from source
+			
+			USAGE:
+				nydus-image create [FLAGS] [OPTIONS] <SOURCE>... --blob <blob> --bootstrap <bootstrap> --fs-version <fs-version> --whiteout-spec <whiteout-spec>
+			
+			FLAGS:
+				-A, --aligned-chunk       Align data chunks to 4K
+					--disable-check       disable validation of metadata after building
+				-h, --help                Prints help information
+					--inline-bootstrap    append bootstrap data to blob
+				-R, --repeatable          generate reproducible nydus image
+				-V, --version             Prints version information
+			
+			OPTIONS:
+					--backend-config <backend-config>
+						[deprecated!] Blob storage backend config - JSON string, only support localfs for compatibility
+			
+					--backend-type <backend-type>
+						[deprecated!] Blob storage backend type, only support localfs for compatibility. Try use --blob instead.
+						[possible values: localfs]
+				-b, --blob <blob>                            path to store nydus image's data blob
+				-D, --blob-dir <blob-dir>                    directory to store nydus image's metadata and data blob
+					--blob-id <blob-id>                      blob id (as object id in backend/oss)
+					--blob-meta <blob-meta>                  path to store nydus blob metadata
+					--blob-offset <blob-offset>
+						add an offset for compressed blob (is only used to put the blob in the tarball) [default: 0]
+			
+				-B, --bootstrap <bootstrap>                  path to store the nydus image's metadata blob
+				-M, --chunk-dict <chunk-dict>                Specify a chunk dictionary for chunk deduplication
+				-S, --chunk-size <chunk-size>
+						size of nydus image data chunk, must be power of two and between 0x1000-0x100000: [default: 0x100000]
+			
+				-c, --compressor <compressor>
+						algorithm to compress image data blob: [default: lz4_block]  [possible values: none, lz4_block, gzip, zstd]
+			
+				-d, --digester <digester>
+						algorithm to digest inodes and data chunks: [default: blake3]  [possible values: blake3, sha256]
+			
+				-v, --fs-version <fs-version>
+						version number of nydus image format: [default: 5]  [possible values: 5, 6]
+			
+				-o, --log-file <log-file>                    Specify log file name
+				-l, --log-level <log-level>
+						Specify log level: [default: info]  [possible values: trace, debug, info, warn, error]
+			
+				-J, --output-json <output-json>              JSON file output path for result
+				-p, --parent-bootstrap <parent-bootstrap>    path to parent/referenced image's metadata blob (optional)
+				-P, --prefetch-policy <prefetch-policy>
+						blob data prefetch policy [default: none]  [possible values: fs, blob, none]
+			
+				-t, --source-type <source-type>
+						type of the source: [default: directory]  [possible values: directory, stargz_index]
+			
+				-W, --whiteout-spec <whiteout-spec>
+						type of whiteout specification: [default: oci]  [possible values: oci, overlayfs, none]
+			
+			
+			ARGS:
+				<SOURCE>...    source path to build the nydus image from
+			`),
+		},
+		{
+			name:    "'--batch-size' is not supported in v1.1.2",
+			feature: FeatureBatchSize,
+			expect:  false,
+			helpMsg: []byte(`
+			nydus-image-create 
+			Create a nydus format accelerated container image
+			
+			USAGE:
+				nydus-image create [FLAGS] [OPTIONS] <SOURCE> --blob <blob> --bootstrap <bootstrap> --whiteout-spec <whiteout-spec>
+			
+			FLAGS:
+					--aligned-chunk    Whether to align chunks into blobcache
+					--disable-check    Disable to validate bootstrap file after building
+				-h, --help             Prints help information
+					--repeatable       Produce environment independent image
+				-V, --version          Prints version information
+			
+			OPTIONS:
+					--backend-config <backend-config>
+						[deprecated!] Blob storage backend config - JSON string, only support localfs for compatibility
+			
+					--backend-type <backend-type>
+						[deprecated!] Blob storage backend type, only support localfs for compatibility. Try use --blob instead.
+						[possible values: localfs]
+					--blob <blob>                            A path to blob file which stores nydus image data portion
+					--blob-dir <blob-dir>
+						A directory where blob files are saved named as their sha256 digest. It's very useful when multiple layers
+						are built at the same time.
+					--blob-id <blob-id>                      blob id (as object id in backend/oss)
+					--bootstrap <bootstrap>                  A path to bootstrap file which stores nydus image metadata portion
+					--chunk-dict <chunk-dict>
+						specify a chunk dictionary file in bootstrap/db format for chunk deduplication.
+			
+					--compressor <compressor>
+						how blob will be compressed: none, lz4_block (default) [default: lz4_block]
+			
+					--digester <digester>
+						how inode and blob chunk will be digested: blake3 (default), sha256 [default: blake3]
+			
+					--log-level <log-level>
+						Specify log level: trace, debug, info, warn, error [default: info]  [possible values: trace, debug, info,
+						warn, error]
+					--output-json <output-json>              JSON output path for build result
+					--parent-bootstrap <parent-bootstrap>    bootstrap file path of parent (optional)
+					--prefetch-policy <prefetch-policy>
+						Prefetch policy: fs(issued from Fs layer), blob(issued from backend/blob layer), none(no readahead is
+						needed) [default: none]
+					--source-type <source-type>
+						source type [default: directory]  [possible values: directory, stargz_index]
+			
+					--whiteout-spec <whiteout-spec>
+						decide which whiteout spec to follow: "oci" or "overlayfs" [default: oci]  [possible values: oci, overlayfs]
+			
+			
+			ARGS:
+				<SOURCE>    source path
+			`),
+		},
+		{
+			name:    "'--type tar-rafs' is not supported in v0.1.0",
+			feature: FeatureTar2Rafs,
+			expect:  false,
+			helpMsg: []byte(`
+			nydus-image-create 
+			dump image bootstrap and upload blob to storage backend
+			
+			USAGE:
+				nydus-image create [FLAGS] [OPTIONS] <SOURCE> --bootstrap <bootstrap> --whiteout-spec <whiteout-spec>
+			
+			FLAGS:
+					--aligned-chunk    Whether to align chunks into blobcache
+					--disable-check    Disable to validate bootstrap file after building
+				-h, --help             Prints help information
+					--repeatable       Produce environment independent image
+				-V, --version          Prints version information
+			
+			OPTIONS:
+					--backend-config <backend-config>              blob storage backend config (JSON string)
+					--backend-config-file <backend-config-file>    blob storage backend config (JSON file)
+					--backend-type <backend-type>                  blob storage backend type (enable blob upload if specified)
+					--blob <blob>                                  blob file path
+					--blob-id <blob-id>                            blob id (as object id in backend)
+					--bootstrap <bootstrap>                        bootstrap file path (required)
+					--compressor <compressor>
+						how blob will be compressed: none, lz4_block (default) [default: lz4_block]
+			
+					--digester <digester>
+						how inode and blob chunk will be digested: blake3 (default), sha256 [default: blake3]
+			
+					--log-level <log-level>
+						Specify log level: trace, debug, info, warn, error [default: info]  [possible values: trace, debug, info,
+						warn, error]
+					--output-json <output-json>                    JSON output path for build result
+					--parent-bootstrap <parent-bootstrap>          bootstrap file path of parent (optional)
+					--prefetch-policy <prefetch-policy>
+						Prefetch policy: fs(issued from Fs layer), blob(issued from backend/blob layer), none(no readahead is
+						needed) [default: none]
+					--source-type <source-type>
+						source type [default: directory]  [possible values: directory, stargz_index]
+			
+					--whiteout-spec <whiteout-spec>
+						decide which whiteout spec to follow: "oci" or "overlayfs" [default: oci]  [possible values: oci, overlayfs]
+			
+			
+			ARGS:
+				<SOURCE>    source path
+			`),
+		},
+		{
+			name:    "'--encrypt' is not supported in v2.2.0",
+			feature: FeatureEncrypt,
+			expect:  false,
+			helpMsg: []byte(`
+			Create RAFS filesystems from directories, tar files or OCI images
+
+			Usage: nydus-image create [OPTIONS] <SOURCE>
+
+			Arguments:
+			<SOURCE>  source from which to build the RAFS filesystem
+
+			Options:
+			-L, --log-file <log-file>
+					Log file path
+			-t, --type <type>
+					Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
+			-B, --bootstrap <bootstrap>
+					File path to save the generated RAFS metadata blob
+			-l, --log-level <log-level>
+					Log level: [default: info] [possible values: trace, debug, info, warn, error]
+			-D, --blob-dir <blob-dir>
+					Directory path to save generated RAFS metadata and data blobs
+			-b, --blob <blob>
+					File path to save the generated RAFS data blob
+				--blob-inline-meta
+					Inline RAFS metadata and blob metadata into the data blob
+				--blob-id <blob-id>
+					OSS object id for the generated RAFS data blob
+				--blob-data-size <blob-data-size>
+					Set data blob size for 'estargztoc-ref' conversion
+				--chunk-size <chunk-size>
+					Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
+				--batch-size <batch-size>
+					Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
+				--compressor <compressor>
+					Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
+				--digester <digester>
+					Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
+			-C, --config <config>
+					Configuration file for storage backend, cache and RAFS FUSE filesystem.
+			-v, --fs-version <fs-version>
+					Set RAFS format version number: [default: 6] [possible values: 5, 6]
+				--features <features>
+					Enable/disable features [possible values: blob-toc]
+				--chunk-dict <chunk-dict>
+					File path of chunk dictionary for data deduplication
+				--parent-bootstrap <parent-bootstrap>
+					File path of the parent/referenced RAFS metadata blob (optional)
+				--aligned-chunk
+					Align uncompressed data chunks to 4K, only for RAFS V5
+				--repeatable
+					Generate reproducible RAFS metadata
+				--whiteout-spec <whiteout-spec>
+					Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
+				--prefetch-policy <prefetch-policy>
+					Set data prefetch policy [default: none] [possible values: fs, blob, none]
+			-J, --output-json <output-json>
+					File path to save operation result in JSON format
+			-h, --help
+					Print help information
+			`),
+		},
+		{
+			name:    "detectFeature should support empty input",
+			feature: "",
+			expect:  false,
+			helpMsg: []byte(`
+			OPTIONS:
+					--type <type>
+						[deprecated!] Conversion type.
+						[possible values: tar-rafs]
+			`),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.expect, detectFeature(tt.helpMsg, tt.feature))
+		})
+	}
+}
+
+func TestDetectFeatures(t *testing.T) {
+	testsCompare := []struct {
+		name            string
+		resetGlobal     bool
+		disableTar2Rafs bool
+		helpText        []byte
+		required        Features
+		detected        Features
+		expectErr       bool
+	}{
+		{
+			name:            "should satisfy required features in v2.2.0-239-gf5c08fcf",
+			resetGlobal:     true,
+			disableTar2Rafs: false,
+			helpText: []byte(`
+			Options:
+			  -t, --type <type>
+					  Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
+				  --batch-size <batch-size>
+					  Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
+			`),
+			required:  Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}},
+			detected:  Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}},
+			expectErr: false,
+		},
+		{
+			name:            "should not support '--encrypt', '--batch-size' or '--type tar-rafs' in v2.1.4",
+			resetGlobal:     true,
+			disableTar2Rafs: true,
+			helpText: []byte(`
+			nydus-image-create 
+			Creates a nydus image from source
+			
+			USAGE:
+				nydus-image create [FLAGS] [OPTIONS] <SOURCE>... --blob <blob> --bootstrap <bootstrap> --fs-version <fs-version> --whiteout-spec <whiteout-spec>
+			
+			FLAGS:
+				-A, --aligned-chunk       Align data chunks to 4K
+					--disable-check       disable validation of metadata after building
+				-h, --help                Prints help information
+					--inline-bootstrap    append bootstrap data to blob
+				-R, --repeatable          generate reproducible nydus image
+				-V, --version             Prints version information
+			
+			OPTIONS:
+					--backend-config <backend-config>
+						[deprecated!] Blob storage backend config - JSON string, only support localfs for compatibility
+			
+					--backend-type <backend-type>
+						[deprecated!] Blob storage backend type, only support localfs for compatibility. Try use --blob instead.
+						[possible values: localfs]
+				-b, --blob <blob>                            path to store nydus image's data blob
+				-D, --blob-dir <blob-dir>                    directory to store nydus image's metadata and data blob
+					--blob-id <blob-id>                      blob id (as object id in backend/oss)
+					--blob-meta <blob-meta>                  path to store nydus blob metadata
+					--blob-offset <blob-offset>
+						add an offset for compressed blob (is only used to put the blob in the tarball) [default: 0]
+			
+				-B, --bootstrap <bootstrap>                  path to store the nydus image's metadata blob
+				-M, --chunk-dict <chunk-dict>                Specify a chunk dictionary for chunk deduplication
+				-S, --chunk-size <chunk-size>
+						size of nydus image data chunk, must be power of two and between 0x1000-0x100000: [default: 0x100000]
+			
+				-c, --compressor <compressor>
+						algorithm to compress image data blob: [default: lz4_block]  [possible values: none, lz4_block, gzip, zstd]
+			
+				-d, --digester <digester>
+						algorithm to digest inodes and data chunks: [default: blake3]  [possible values: blake3, sha256]
+			
+				-v, --fs-version <fs-version>
+						version number of nydus image format: [default: 5]  [possible values: 5, 6]
+			
+				-o, --log-file <log-file>                    Specify log file name
+				-l, --log-level <log-level>
+						Specify log level: [default: info]  [possible values: trace, debug, info, warn, error]
+			
+				-J, --output-json <output-json>              JSON file output path for result
+				-p, --parent-bootstrap <parent-bootstrap>    path to parent/referenced image's metadata blob (optional)
+				-P, --prefetch-policy <prefetch-policy>
+						blob data prefetch policy [default: none]  [possible values: fs, blob, none]
+			
+				-t, --source-type <source-type>
+						type of the source: [default: directory]  [possible values: directory, stargz_index]
+			
+				-W, --whiteout-spec <whiteout-spec>
+						type of whiteout specification: [default: oci]  [possible values: oci, overlayfs, none]
+			
+			
+			ARGS:
+				<SOURCE>...    source path to build the nydus image from
+			`),
+			required:  Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}, FeatureEncrypt: {}},
+			detected:  Features{},
+			expectErr: false,
+		},
+		{
+			name:            "should ignore '--type tar-rafs' if disabled",
+			resetGlobal:     true,
+			disableTar2Rafs: true,
+			helpText: []byte(`
+			Options:
+			  -t, --type <type>
+					  Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
+				  --batch-size <batch-size>
+					  Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
+			`),
+			required:  Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}},
+			detected:  Features{FeatureBatchSize: {}},
+			expectErr: false,
+		},
+		{
+			name:            "should return error if required features changed in different calls",
+			resetGlobal:     false,
+			disableTar2Rafs: false,
+			helpText:        nil,
+			required:        Features{},
+			detected:        nil,
+			expectErr:       true,
+		},
+	}
+	for _, tt := range testsCompare {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.resetGlobal {
+				// Reset global variables.
+				requiredFeatures = Features{}
+				detectedFeatures = Features{}
+				detectFeaturesOnce = sync.Once{}
+				disableTar2Rafs = tt.disableTar2Rafs
+			}
+			detected, err := DetectFeatures("", tt.required, func(_ string) []byte { return tt.helpText })
+			require.Equal(t, tt.expectErr, err != nil)
+			require.Equal(t, tt.detected, detected)
+		})
+	}
 }

pkg/converter/types.go

@@ -13,7 +13,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/v2/core/content"
 	"github.com/containerd/nydus-snapshotter/pkg/converter/tool"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -21,9 +21,13 @@ import (
 
 type Compressor = uint32
 
+type Encrypter = func(context.Context, content.Store, ocispec.Descriptor) (ocispec.Descriptor, error)
+
 const (
-	CompressorNone Compressor = 0x0001
-	CompressorZstd Compressor = 0x0002
+	CompressorNone     Compressor = 0x0000_0001
+	CompressorZstd     Compressor = 0x0000_0002
+	CompressorLz4Block Compressor = 0x0000_0004
+	CompressorMask     Compressor = 0x0000_000f
 )
 
 var (
@@ -71,10 +75,14 @@ type PackOption struct {
 	AlignedChunk bool
 	// ChunkSize sets the size of data chunks, must be power of two and between 0x1000-0x1000000.
 	ChunkSize string
+	// BacthSize sets the size of batch data chunks, must be power of two and between 0x1000-0x1000000 or zero.
+	BatchSize string
 	// Backend uploads blobs generated by nydus-image builder to a backend storage.
 	Backend Backend
 	// Timeout cancels execution once exceed the specified time.
 	Timeout *time.Duration
+	// Whether the generated Nydus blobs should be encrypted.
+	Encrypt bool
 
 	// Features keeps a feature list supported by newer version of builder,
 	// It is detected automatically, so don't export it.
@@ -97,12 +105,28 @@ type MergeOption struct {
 	PrefetchPatterns string
 	// WithTar puts bootstrap into a tar stream (no gzip).
 	WithTar bool
+	// OCI converts docker media types to OCI media types.
+	OCI bool
 	// OCIRef enables converting OCI tar(.gz) blob to nydus referenced blob.
 	OCIRef bool
+	// WithReferrer associates a reference to the original OCI manifest.
+	// See the `subject` field description in
+	// https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
+	//
+	// With this association, we can track all nydus images associated with
+	// an OCI image. For example, in Harbor we can cascade to show nydus
+	// images linked to an OCI image, deleting the OCI image can also delete
+	// the corresponding nydus images. At runtime, nydus snapshotter can also
+	// automatically upgrade an OCI image run to nydus image.
+	WithReferrer bool
 	// Backend uploads blobs generated by nydus-image builder to a backend storage.
 	Backend Backend
 	// Timeout cancels execution once exceed the specified time.
 	Timeout *time.Duration
+	// Encrypt encrypts the bootstrap layer if it's specified.
+	Encrypt Encrypter
+	// AppendFiles specifies the files that need to be appended to the bootstrap layer.
+	AppendFiles []File
 }
 
 type UnpackOption struct {
@@ -135,11 +159,13 @@ type TOCEntry struct {
 }
 
 func (entry *TOCEntry) GetCompressor() (Compressor, error) {
-	switch {
-	case entry.Flags&CompressorNone == CompressorNone:
+	switch entry.Flags & CompressorMask {
+	case CompressorNone:
 		return CompressorNone, nil
-	case entry.Flags&CompressorZstd == CompressorZstd:
+	case CompressorZstd:
 		return CompressorZstd, nil
+	case CompressorLz4Block:
+		return CompressorLz4Block, nil
 	}
 	return 0, fmt.Errorf("unsupported compressor, entry flags %x", entry.Flags)
 }

pkg/converter/utils.go

@@ -14,14 +14,19 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"path/filepath"
 
-	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/v2/core/content"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+type File struct {
+	Name   string
+	Reader io.Reader
+	Size   int64
+}
+
 type writeCloser struct {
 	closed bool
 	io.WriteCloser
@@ -59,7 +64,7 @@ type seekReader struct {
 
 func (ra *seekReader) Read(p []byte) (int, error) {
 	n, err := ra.ReaderAt.ReadAt(p, ra.pos)
-	ra.pos += int64(len(p))
+	ra.pos += int64(n)
 	return n, err
 }
 
@@ -83,39 +88,27 @@ func newSeekReader(ra io.ReaderAt) *seekReader {
 	}
 }
 
-// packToTar makes .tar(.gz) stream of file named `name` and return reader.
-func packToTar(src string, name string, compress bool) (io.ReadCloser, error) {
-	fi, err := os.Stat(src)
-	if err != nil {
-		return nil, err
-	}
-
+// packToTar packs files to .tar(.gz) stream then return reader.
+func packToTar(files []File, compress bool) io.ReadCloser {
 	dirHdr := &tar.Header{
-		Name:     filepath.Dir(name),
+		Name:     "image",
 		Mode:     0755,
 		Typeflag: tar.TypeDir,
 	}
 
-	hdr := &tar.Header{
-		Name: name,
-		Mode: 0444,
-		Size: fi.Size(),
-	}
-
-	reader, writer := io.Pipe()
+	pr, pw := io.Pipe()
 
 	go func() {
 		// Prepare targz writer
 		var tw *tar.Writer
 		var gw *gzip.Writer
 		var err error
-		var file *os.File
 
 		if compress {
-			gw = gzip.NewWriter(writer)
+			gw = gzip.NewWriter(pw)
 			tw = tar.NewWriter(gw)
 		} else {
-			tw = tar.NewWriter(writer)
+			tw = tar.NewWriter(pw)
 		}
 
 		defer func() {
@@ -137,30 +130,30 @@ func packToTar(src string, name string, compress bool) (io.ReadCloser, error) {
 				finalErr = err2
 			}
 
-			writer.CloseWithError(finalErr)
+			pw.CloseWithError(finalErr)
 		}()
 
-		file, err = os.Open(src)
-		if err != nil {
-			return
-		}
-		defer file.Close()
-
 		// Write targz stream
 		if err = tw.WriteHeader(dirHdr); err != nil {
 			return
 		}
 
-		if err = tw.WriteHeader(hdr); err != nil {
-			return
-		}
-
-		if _, err = io.Copy(tw, file); err != nil {
-			return
+		for _, file := range files {
+			hdr := tar.Header{
+				Name: filepath.Join("image", file.Name),
+				Mode: 0444,
+				Size: file.Size,
+			}
+			if err = tw.WriteHeader(&hdr); err != nil {
+				return
+			}
+			if _, err = io.Copy(tw, file.Reader); err != nil {
+				return
+			}
 		}
 	}()
 
-	return reader, nil
+	return pr
 }
 
 // Copied from containerd/containerd project, copyright The containerd Authors.

pkg/daemon/client.go

@@ -22,26 +22,34 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/containerd/log"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
+	"github.com/containerd/nydus-snapshotter/pkg/metrics/tool"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/retry"
 )
 
 const (
+	// Get information about nydus daemon
 	endpointDaemonInfo = "/api/v1/daemon"
-	endpointMount      = "/api/v1/mount"
-	endpointMetrics    = "/api/v1/metrics"
+	// Mount or umount filesystems.
+	endpointMount = "/api/v1/mount"
+	// Fetch generic filesystem metrics.
+	endpointMetrics = "/api/v1/metrics"
 	// Fetch metrics relevant to caches usage.
-	endpointCacheMetrics    = "/api/v1/metrics/blobcache"
+	endpointCacheMetrics = "/api/v1/metrics/blobcache"
+	// Fetch metrics about inflighting operations.
 	endpointInflightMetrics = "/api/v1/metrics/inflight"
-	// Command nydusd to retrieve its runtime states, which is used during failover
+	// Request nydus daemon to retrieve its runtime states from the supervisor, recovering states for failover.
 	endpointTakeOver = "/api/v1/daemon/fuse/takeover"
-	// Command nydusd to send out its runtime states, which prepares failover.
+	// Request nydus daemon to send its runtime states to the supervisor, preparing for failover.
 	endpointSendFd = "/api/v1/daemon/fuse/sendfd"
-	// Command nydusd to begin file system service.
+	// Request nydus daemon to start filesystem service.
 	endpointStart = "/api/v1/daemon/start"
-	endpointExit  = "/api/v1/daemon/exit"
+	// Request nydus daemon to exit
+	endpointExit = "/api/v1/daemon/exit"
 
 	// --- V2 API begins
+	// Add/remove blobs managed by the blob cache manager.
 	endpointBlobs = "/api/v2/blobs"
 
 	defaultHTTPClientTimeout = 30 * time.Second
@@ -160,7 +168,7 @@ func buildTransport(sock string) http.RoundTripper {
 	}
 }
 
-func WaitUntilSocketExisted(sock string) error {
+func WaitUntilSocketExisted(sock string, pid int) error {
 	return retry.Do(func() (err error) {
 		var st fs.FileInfo
 		if st, err = os.Stat(sock); err != nil {
@@ -175,7 +183,20 @@ func WaitUntilSocketExisted(sock string) error {
 	},
 		retry.Attempts(100), // totally wait for 10 seconds, should be enough
 		retry.LastErrorOnly(true),
-		retry.Delay(100*time.Millisecond))
+		retry.Delay(100*time.Millisecond),
+		retry.OnlyRetryIf(func(error) bool {
+			zombie, err := tool.IsZombieProcess(pid)
+			if err != nil {
+				return false
+			}
+			// Stop retry if nydus daemon process is already in Zombie state.
+			if zombie {
+				log.L.Errorf("Process %d has been a zombie", pid)
+				return true
+			}
+			return false
+		}),
+	)
 }
 
 func NewNydusClient(sock string) (NydusdClient, error) {
@@ -206,6 +227,19 @@ func (c *nydusdClient) GetDaemonInfo() (*types.DaemonInfo, error) {
 	return &info, nil
 }
 
+func (c *nydusdClient) Mount(mp, bootstrap, mountConfig string) error {
+	cmd, err := json.Marshal(types.NewMountRequest(bootstrap, mountConfig))
+	if err != nil {
+		return errors.Wrap(err, "construct mount request")
+	}
+
+	query := query{}
+	query.Add("mountpoint", mp)
+	url := c.url(endpointMount, query)
+
+	return c.request(http.MethodPost, url, bytes.NewBuffer(cmd), nil)
+}
+
 func (c *nydusdClient) Umount(mp string) error {
 	query := query{}
 	query.Add("mountpoint", mp)
@@ -213,6 +247,32 @@ func (c *nydusdClient) Umount(mp string) error {
 	return c.request(http.MethodDelete, url, nil, nil)
 }
 
+func (c *nydusdClient) BindBlob(daemonConfig string) error {
+	url := c.url(endpointBlobs, query{})
+	return c.request(http.MethodPut, url, bytes.NewBuffer([]byte(daemonConfig)), nil)
+}
+
+// Delete /api/v2/blobs implements different functions according to different parameters
+//  1. domainID , delete all blob entries in the domain.
+//  2. domainID + blobID, delete the blob entry, if the blob is bootstrap
+//     also delete blob entries belong to it.
+//  3. blobID, try to find and cull blob cache files by blobID in all domains.
+func (c *nydusdClient) UnbindBlob(domainID, blobID string) error {
+	query := query{}
+	if domainID != "" {
+		query.Add("domain_id", domainID)
+		if domainID != blobID {
+			query.Add("blob_id", blobID)
+		}
+	} else {
+		query.Add("blob_id", blobID)
+	}
+
+	url := c.url(endpointBlobs, query)
+
+	return c.request(http.MethodDelete, url, nil, nil)
+}
+
 func (c *nydusdClient) GetFsMetrics(sid string) (*types.FsMetrics, error) {
 	query := query{}
 	if sid != "" {
@@ -262,46 +322,6 @@ func (c *nydusdClient) GetCacheMetrics(sid string) (*types.CacheMetrics, error)
 	return &m, nil
 }
 
-// `daemonConfig` a json string represents daemon configuration.
-func (c *nydusdClient) Mount(mp, bootstrap, daemonConfig string) error {
-	cmd, err := json.Marshal(types.NewMountRequest(bootstrap, daemonConfig))
-	if err != nil {
-		return errors.Wrap(err, "construct mount request")
-	}
-
-	query := query{}
-	query.Add("mountpoint", mp)
-	url := c.url(endpointMount, query)
-
-	return c.request(http.MethodPost, url, bytes.NewBuffer(cmd), nil)
-}
-
-func (c *nydusdClient) BindBlob(daemonConfig string) error {
-	url := c.url(endpointBlobs, query{})
-	return c.request(http.MethodPut, url, bytes.NewBuffer([]byte(daemonConfig)), nil)
-}
-
-// Delete /api/v2/blobs implements different functions according to different parameters
-//  1. domainID , delete all blob entries in the domain.
-//  2. domainID + blobID, delete the blob entry, if the blob is bootstrap
-//     also delete blob entries belong to it.
-//  3. blobID, try to find and cull blob cache files by blobID in all domains.
-func (c *nydusdClient) UnbindBlob(domainID, blobID string) error {
-	query := query{}
-	if domainID != "" {
-		query.Add("domain_id", domainID)
-		if domainID != blobID {
-			query.Add("blob_id", blobID)
-		}
-	} else {
-		query.Add("blob_id", blobID)
-	}
-
-	url := c.url(endpointBlobs, query)
-
-	return c.request(http.MethodDelete, url, nil, nil)
-}
-
 func (c *nydusdClient) TakeOver() error {
 	url := c.url(endpointTakeOver, query{})
 	return c.request(http.MethodPut, url, nil, nil)

pkg/daemon/client_test.go

@@ -37,7 +37,7 @@ func prepareNydusServer(t *testing.T) (string, func()) {
 	if err == nil {
 		_ = os.Remove(mockSocket)
 	}
-	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(200)
 		info := types.DaemonInfo{
 			ID:      "testid",

pkg/daemon/command/command.go

@@ -26,14 +26,17 @@ type DaemonCommand struct {
 	Upgrade        bool   `type:"flag" name:"upgrade" default:""`
 	ThreadNum      string `type:"param" name:"thread-num"`
 	// `--id` is required by `--supervisor` when starting nydusd
-	ID         string `type:"param" name:"id"`
-	Config     string `type:"param" name:"config"`
-	Bootstrap  string `type:"param" name:"bootstrap"`
-	Mountpoint string `type:"param" name:"mountpoint"`
-	APISock    string `type:"param" name:"apisock"`
-	LogLevel   string `type:"param" name:"log-level"`
-	Supervisor string `type:"param" name:"supervisor"`
-	LogFile    string `type:"param" name:"log-file"`
+	ID              string `type:"param" name:"id"`
+	Config          string `type:"param" name:"config"`
+	Bootstrap       string `type:"param" name:"bootstrap"`
+	Mountpoint      string `type:"param" name:"mountpoint"`
+	APISock         string `type:"param" name:"apisock"`
+	LogLevel        string `type:"param" name:"log-level"`
+	LogRotationSize int    `type:"param" name:"log-rotation-size"`
+	Supervisor      string `type:"param" name:"supervisor"`
+	LogFile         string `type:"param" name:"log-file"`
+	PrefetchFiles   string `type:"param" name:"prefetch-files"`
+	BackendSource   string `type:"param" name:"backend-source"`
 }
 
 // Build exec style command line
@@ -62,7 +65,7 @@ func BuildCommand(opts []Opt) ([]string, error) {
 
 			value := v.Field(i).Interface()
 
-			pair := []string{fmt.Sprintf("--%s", tag.Get("name")), fmt.Sprintf("%s", value)}
+			pair := []string{fmt.Sprintf("--%s", tag.Get("name")), fmt.Sprintf("%v", value)}
 			args = append(args, pair...)
 		case "subcommand":
 			// Zero value will be skipped appending to command line
@@ -104,6 +107,12 @@ func WithMode(m string) Opt {
 	}
 }
 
+func WithPrefetchFiles(p string) Opt {
+	return func(cmd *DaemonCommand) {
+		cmd.PrefetchFiles = p
+	}
+}
+
 func WithFscacheDriver(w string) Opt {
 	return func(cmd *DaemonCommand) {
 		cmd.FscacheDriver = w
@@ -146,12 +155,24 @@ func WithAPISock(api string) Opt {
 	}
 }
 
+func WithLogFile(l string) Opt {
+	return func(cmd *DaemonCommand) {
+		cmd.LogFile = l
+	}
+}
+
 func WithLogLevel(l string) Opt {
 	return func(cmd *DaemonCommand) {
 		cmd.LogLevel = l
 	}
 }
 
+func WithLogRotationSize(l int) Opt {
+	return func(cmd *DaemonCommand) {
+		cmd.LogRotationSize = l
+	}
+}
+
 func WithSupervisor(s string) Opt {
 	return func(cmd *DaemonCommand) {
 		cmd.Supervisor = s
@@ -170,8 +191,8 @@ func WithUpgrade() Opt {
 	}
 }
 
-func WithLogFile(l string) Opt {
+func WithBackendSource(source string) Opt {
 	return func(cmd *DaemonCommand) {
-		cmd.LogFile = l
+		cmd.BackendSource = source
 	}
 }

pkg/daemon/config.go

@@ -12,8 +12,10 @@ import (
 	"path"
 	"path/filepath"
 
-	"github.com/containerd/nydus-snapshotter/config"
 	"github.com/pkg/errors"
+
+	"github.com/containerd/nydus-snapshotter/config"
+	"github.com/containerd/nydus-snapshotter/internal/constant"
 )
 
 // Build runtime nydusd daemon object, which might be persisted later
@@ -38,6 +40,9 @@ func WithRef(ref int32) NewDaemonOpt {
 
 func WithLogDir(dir string) NewDaemonOpt {
 	return func(d *Daemon) error {
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return errors.Wrapf(err, "create logging dir %s", dir)
+		}
 		d.States.LogDir = filepath.Join(dir, d.ID())
 		return nil
 	}
@@ -53,7 +58,7 @@ func WithLogToStdout(logToStdout bool) NewDaemonOpt {
 func WithLogLevel(logLevel string) NewDaemonOpt {
 	return func(d *Daemon) error {
 		if logLevel == "" {
-			d.States.LogLevel = config.DefaultLogLevel
+			d.States.LogLevel = constant.DefaultLogLevel
 		} else {
 			d.States.LogLevel = logLevel
 		}
@@ -61,6 +66,13 @@ func WithLogLevel(logLevel string) NewDaemonOpt {
 	}
 }
 
+func WithLogRotationSize(logRotationSize int) NewDaemonOpt {
+	return func(d *Daemon) error {
+		d.States.LogRotationSize = logRotationSize
+		return nil
+	}
+}
+
 func WithConfigDir(dir string) NewDaemonOpt {
 	return func(d *Daemon) error {
 		s := filepath.Join(dir, d.ID())
@@ -93,3 +105,10 @@ func WithFsDriver(fsDriver string) NewDaemonOpt {
 		return nil
 	}
 }
+
+func WithDaemonMode(daemonMode config.DaemonMode) NewDaemonOpt {
+	return func(d *Daemon) error {
+		d.States.DaemonMode = daemonMode
+		return nil
+	}
+}

pkg/daemon/daemon.go

@@ -10,6 +10,7 @@ package daemon
 import (
 	"os"
 	"path/filepath"
+	"sort"
 	"sync"
 	"sync/atomic"
 	"syscall"
@@ -17,12 +18,13 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 
 	"github.com/containerd/nydus-snapshotter/config"
 	"github.com/containerd/nydus-snapshotter/config/daemonconfig"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
+	"github.com/containerd/nydus-snapshotter/pkg/rafs"
 	"github.com/containerd/nydus-snapshotter/pkg/supervisor"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/erofs"
 	"github.com/containerd/nydus-snapshotter/pkg/utils/mount"
@@ -36,39 +38,41 @@ const (
 
 type NewDaemonOpt func(d *Daemon) error
 
-type States struct {
-	// Generated by daemons manager as a unique to identify a nydusd
-	ID          string
-	ProcessID   int
-	APISocket   string
-	LogDir      string
-	LogLevel    string
-	LogToStdout bool
-	FsDriver    string
-	// Host kernel mountpoint, only applies to fuse fs driver. The fscache fs driver
-	// doesn't need a host kernel mountpoint.
-	Mountpoint string
-	ThreadNum  int
+// Fields in this structure should be write-once, and caller should hold `Daemon.mu` when updating fields.
+type ConfigState struct {
+	// A unique ID generated by daemon manager to identify the nydusd instance.
+	ID              string
+	ProcessID       int
+	APISocket       string
+	DaemonMode      config.DaemonMode
+	FsDriver        string
+	LogDir          string
+	LogLevel        string
+	LogRotationSize int
+	LogToStdout     bool
+	Mountpoint      string
+	SupervisorPath  string
+	ThreadNum       int
 	// Where the configuration file resides, all rafs instances share the same configuration template
-	ConfigDir      string
-	SupervisorPath string
+	ConfigDir string
 }
 
 // TODO: Record queried nydusd state
 type Daemon struct {
-	States States
+	States ConfigState
 
 	mu sync.Mutex
-	// FsInstances map[int]*Rafs
-	// should be persisted to DB
-	// maps to at least one rafs instance.
-	// It is possible to be empty after the daemon object is created.
-	Instances rafsSet
+	// Host all RAFS filesystems managed by this daemon:
+	// fusedev dedicated mode: one and only one RAFS instance
+	// fusedev shared mode: zero, one or more RAFS instances
+	// fscache shared mode: zero, one or more RAFS instances
+	RafsCache rafs.Cache
 
-	// client will be rebuilt on Reconnect, skip marshal/unmarshal
-	client NydusdClient
 	// Protect nydusd http client
 	cmu sync.Mutex
+	// client will be rebuilt on Reconnect, skip marshal/unmarshal
+	client NydusdClient
+
 	// Nil means this daemon object has no supervisor
 	Supervisor *supervisor.Supervisor
 	Config     daemonconfig.DaemonConfig
@@ -80,7 +84,7 @@ type Daemon struct {
 
 	ref int32
 	// Cache the nydusd daemon state to avoid frequently querying nydusd by API.
-	State types.DaemonState
+	state types.DaemonState
 }
 
 func (d *Daemon) Lock() {
@@ -112,7 +116,6 @@ func (d *Daemon) GetRef() int32 {
 }
 
 func (d *Daemon) HostMountpoint() (mnt string) {
-	// Identify a shared nydusd for multiple rafs instances.
 	mnt = d.States.Mountpoint
 	return
 }
@@ -138,18 +141,18 @@ func (d *Daemon) LogFile() string {
 	return filepath.Join(d.States.LogDir, "nydusd.log")
 }
 
-func (d *Daemon) AddInstance(r *Rafs) {
-	d.Instances.Add(r)
+func (d *Daemon) AddRafsInstance(r *rafs.Rafs) {
+	d.RafsCache.Add(r)
 	d.IncRef()
 	r.DaemonID = d.ID()
 }
 
-func (d *Daemon) RemoveInstance(snapshotID string) {
-	d.Instances.Remove(snapshotID)
+func (d *Daemon) RemoveRafsInstance(snapshotID string) {
+	d.RafsCache.Remove(snapshotID)
 	d.DecRef()
 }
 
-// Nydusd daemon current working state by requesting to nydusd:
+// Get and cache daemon current working state by querying nydusd:
 // 1. INIT
 // 2. READY: All needed resources are ready.
 // 3. RUNNING
@@ -166,26 +169,33 @@ func (d *Daemon) GetState() (types.DaemonState, error) {
 	st := info.DaemonState()
 
 	d.Lock()
-	d.State = st
+	d.state = st
 	d.Version = info.DaemonVersion()
 	d.Unlock()
 
 	return st, nil
 }
 
-// Waits for some time until daemon reaches the expected state.
-// For example:
-//  1. INIT
-//  2. READY
-//  3. RUNNING
+// Return the cached nydusd working status, no API is invoked.
+func (d *Daemon) State() types.DaemonState {
+	d.Lock()
+	defer d.Unlock()
+	return d.state
+}
+
+// Reset the cached nydusd working status
+func (d *Daemon) ResetState() {
+	d.Lock()
+	defer d.Unlock()
+	d.state = types.DaemonStateUnknown
+}
+
+// Wait for the nydusd daemon to reach specified state with timeout.
 func (d *Daemon) WaitUntilState(expected types.DaemonState) error {
 	return retry.Do(func() error {
-		d.Lock()
-		if expected == d.State {
-			d.Unlock()
+		if expected == d.State() {
 			return nil
 		}
-		d.Unlock()
 
 		state, err := d.GetState()
 		if err != nil {
@@ -199,25 +209,40 @@ func (d *Daemon) WaitUntilState(expected types.DaemonState) error {
 
 		return nil
 	},
-		retry.Attempts(20), // totally wait for 2 seconds, should be enough
 		retry.LastErrorOnly(true),
+		retry.Attempts(20), // totally wait for 2 seconds, should be enough
 		retry.Delay(100*time.Millisecond),
 	)
 }
 
-func (d *Daemon) SharedMount(rafs *Rafs) error {
-	client, err := d.GetClient()
-	if err != nil {
-		return errors.Wrapf(err, "mount instance %s", rafs.SnapshotID)
+func (d *Daemon) IsSharedDaemon() bool {
+	if d.States.DaemonMode != "" {
+		return d.States.DaemonMode == config.DaemonModeShared
 	}
 
+	return d.HostMountpoint() == config.GetRootMountpoint()
+}
+
+func (d *Daemon) SharedMount(rafs *rafs.Rafs) error {
 	defer d.SendStates()
 
-	if d.States.FsDriver == config.FsDriverFscache {
+	switch d.States.FsDriver {
+	case config.FsDriverFscache:
 		if err := d.sharedErofsMount(rafs); err != nil {
 			return errors.Wrapf(err, "mount erofs")
 		}
 		return nil
+	case config.FsDriverFusedev:
+		return d.sharedFusedevMount(rafs)
+	default:
+		return errors.Errorf("unsupported fs driver %s", d.States.FsDriver)
+	}
+}
+
+func (d *Daemon) sharedFusedevMount(rafs *rafs.Rafs) error {
+	client, err := d.GetClient()
+	if err != nil {
+		return errors.Wrapf(err, "mount instance %s", rafs.SnapshotID)
 	}
 
 	bootstrap, err := rafs.BootstrapFile()
@@ -244,38 +269,20 @@ func (d *Daemon) SharedMount(rafs *Rafs) error {
 	return nil
 }
 
-func (d *Daemon) SharedUmount(rafs *Rafs) error {
-	c, err := d.GetClient()
-	if err != nil {
-		return errors.Wrapf(err, "umount instance %s", rafs.SnapshotID)
-	}
-
-	defer d.SendStates()
-
-	if d.States.FsDriver == config.FsDriverFscache {
-		if err := d.sharedErofsUmount(rafs); err != nil {
-			return errors.Wrapf(err, "failed to erofs mount")
-		}
-		return nil
-	}
-
-	return c.Umount(rafs.RelaMountpoint())
-}
-
-func (d *Daemon) sharedErofsMount(rafs *Rafs) error {
+func (d *Daemon) sharedErofsMount(ra *rafs.Rafs) error {
 	client, err := d.GetClient()
 	if err != nil {
 		return errors.Wrapf(err, "bind blob %s", d.ID())
 	}
 
 	// TODO: Why fs cache needing this work dir?
-	if err := os.MkdirAll(rafs.FscacheWorkDir(), 0755); err != nil {
-		return errors.Wrapf(err, "failed to create fscache work dir %s", rafs.FscacheWorkDir())
+	if err := os.MkdirAll(ra.FscacheWorkDir(), 0755); err != nil {
+		return errors.Wrapf(err, "failed to create fscache work dir %s", ra.FscacheWorkDir())
 	}
 
-	c, err := daemonconfig.NewDaemonConfig(d.States.FsDriver, d.ConfigFile(rafs.SnapshotID))
+	c, err := daemonconfig.NewDaemonConfig(d.States.FsDriver, d.ConfigFile(ra.SnapshotID))
 	if err != nil {
-		log.L.Errorf("Failed to reload daemon configuration %s, %s", d.ConfigFile(rafs.SnapshotID), err)
+		log.L.Errorf("Failed to reload daemon configuration %s, %s", d.ConfigFile(ra.SnapshotID), err)
 		return err
 	}
 
@@ -288,22 +295,18 @@ func (d *Daemon) sharedErofsMount(rafs *Rafs) error {
 		return errors.Wrapf(err, "request to bind fscache blob")
 	}
 
-	mountPoint := rafs.GetMountpoint()
+	mountPoint := ra.GetMountpoint()
 	if err := os.MkdirAll(mountPoint, 0755); err != nil {
 		return errors.Wrapf(err, "create mountpoint %s", mountPoint)
 	}
 
-	bootstrapPath, err := rafs.BootstrapFile()
-	if err != nil {
-		return err
-	}
-	fscacheID := erofs.FscacheID(rafs.SnapshotID)
+	fscacheID := erofs.FscacheID(ra.SnapshotID)
 
 	cfg := c.(*daemonconfig.FscacheDaemonConfig)
-	rafs.AddAnnotation(AnnoFsCacheDomainID, cfg.DomainID)
-	rafs.AddAnnotation(AnnoFsCacheID, fscacheID)
+	ra.AddAnnotation(rafs.AnnoFsCacheDomainID, cfg.DomainID)
+	ra.AddAnnotation(rafs.AnnoFsCacheID, fscacheID)
 
-	if err := erofs.Mount(bootstrapPath, cfg.DomainID, fscacheID, mountPoint); err != nil {
+	if err := erofs.Mount(cfg.DomainID, fscacheID, mountPoint); err != nil {
 		if !errdefs.IsErofsMounted(err) {
 			return errors.Wrapf(err, "mount erofs to %s", mountPoint)
 		}
@@ -317,19 +320,39 @@ func (d *Daemon) sharedErofsMount(rafs *Rafs) error {
 	return nil
 }
 
-func (d *Daemon) sharedErofsUmount(rafs *Rafs) error {
+func (d *Daemon) SharedUmount(rafs *rafs.Rafs) error {
+	defer d.SendStates()
+
+	switch d.States.FsDriver {
+	case config.FsDriverFscache:
+		if err := d.sharedErofsUmount(rafs); err != nil {
+			return errors.Wrapf(err, "failed to erofs mount")
+		}
+		return nil
+	case config.FsDriverFusedev:
+		c, err := d.GetClient()
+		if err != nil {
+			return errors.Wrapf(err, "umount instance %s", rafs.SnapshotID)
+		}
+		return c.Umount(rafs.RelaMountpoint())
+	default:
+		return errors.Errorf("unsupported fs driver %s", d.States.FsDriver)
+	}
+}
+
+func (d *Daemon) sharedErofsUmount(ra *rafs.Rafs) error {
 	c, err := d.GetClient()
 	if err != nil {
 		return errors.Wrapf(err, "unbind blob %s", d.ID())
 	}
-	domainID := rafs.Annotations[AnnoFsCacheDomainID]
-	fscacheID := rafs.Annotations[AnnoFsCacheID]
+	domainID := ra.Annotations[rafs.AnnoFsCacheDomainID]
+	fscacheID := ra.Annotations[rafs.AnnoFsCacheID]
 
 	if err := c.UnbindBlob(domainID, fscacheID); err != nil {
 		return errors.Wrapf(err, "request to unbind fscache blob, domain %s, fscache %s", domainID, fscacheID)
 	}
 
-	mountpoint := rafs.GetMountpoint()
+	mountpoint := ra.GetMountpoint()
 	if err := erofs.Umount(mountpoint); err != nil {
 		return errors.Wrapf(err, "umount erofs %s mountpoint, %s", err, mountpoint)
 	}
@@ -343,6 +366,33 @@ func (d *Daemon) sharedErofsUmount(rafs *Rafs) error {
 	return nil
 }
 
+func (d *Daemon) UmountRafsInstance(r *rafs.Rafs) error {
+	if d.IsSharedDaemon() {
+		if err := d.SharedUmount(r); err != nil {
+			return errors.Wrapf(err, "umount fs instance %s", r.SnapshotID)
+		}
+	}
+
+	return nil
+}
+
+func (d *Daemon) UmountRafsInstances() error {
+	if d.IsSharedDaemon() {
+		d.RafsCache.Lock()
+		defer d.RafsCache.Unlock()
+
+		instances := d.RafsCache.ListLocked()
+
+		for _, r := range instances {
+			if err := d.SharedUmount(r); err != nil {
+				return errors.Wrapf(err, "umount fs instance %s", r.SnapshotID)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (d *Daemon) SendStates() {
 	su := d.Supervisor
 	if su != nil {
@@ -413,6 +463,15 @@ func (d *Daemon) Exit() error {
 	return nil
 }
 
+func (d *Daemon) GetDaemonInfo() (*types.DaemonInfo, error) {
+	c, err := d.GetClient()
+	if err != nil {
+		return nil, errors.Wrapf(err, "get daemon information")
+	}
+
+	return c.GetDaemonInfo()
+}
+
 func (d *Daemon) GetFsMetrics(sid string) (*types.FsMetrics, error) {
 	c, err := d.GetClient()
 	if err != nil {
@@ -431,15 +490,6 @@ func (d *Daemon) GetInflightMetrics() (*types.InflightMetrics, error) {
 	return c.GetInflightMetrics()
 }
 
-func (d *Daemon) GetDaemonInfo() (*types.DaemonInfo, error) {
-	c, err := d.GetClient()
-	if err != nil {
-		return nil, errors.Wrapf(err, "get daemon information")
-	}
-
-	return c.GetDaemonInfo()
-}
-
 func (d *Daemon) GetCacheMetrics(sid string) (*types.CacheMetrics, error) {
 	c, err := d.GetClient()
 	if err != nil {
@@ -452,8 +502,18 @@ func (d *Daemon) GetClient() (NydusdClient, error) {
 	d.cmu.Lock()
 	defer d.cmu.Unlock()
 
-	if err := d.ensureClientUnlocked(); err != nil {
-		return nil, err
+	if d.client == nil {
+		sock := d.GetAPISock()
+		// The socket file may be residual from a dead nydusd
+		err := WaitUntilSocketExisted(sock, d.Pid())
+		if err != nil {
+			return nil, errors.Wrapf(errdefs.ErrNotFound, "daemon socket %s", sock)
+		}
+		client, err := NewNydusClient(sock)
+		if err != nil {
+			return nil, errors.Wrapf(err, "create daemon %s client", d.ID())
+		}
+		d.client = client
 	}
 
 	return d.client, nil
@@ -465,24 +525,6 @@ func (d *Daemon) ResetClient() {
 	d.cmu.Unlock()
 }
 
-// The client should be locked outside
-func (d *Daemon) ensureClientUnlocked() error {
-	if d.client == nil {
-		sock := d.GetAPISock()
-		// The socket file may be residual from a dead nydusd
-		err := WaitUntilSocketExisted(sock)
-		if err != nil {
-			return errors.Wrapf(errdefs.ErrNotFound, "daemon socket %s", sock)
-		}
-		client, err := NewNydusClient(sock)
-		if err != nil {
-			return errors.Wrapf(err, "create daemon %s client", d.ID())
-		}
-		d.client = client
-	}
-	return nil
-}
-
 func (d *Daemon) Terminate() error {
 	// if we found pid here, we need to kill and wait process to exit, Pid=0 means somehow we lost
 	// the daemon pid, so that we can't kill the process, just roughly umount the mountpoint
@@ -518,7 +560,7 @@ func (d *Daemon) Wait() error {
 		// nydus-snapshotter, p.Wait() will return err, so here should exclude this case
 		if _, err = p.Wait(); err != nil && !errors.Is(err, syscall.ECHILD) {
 			log.L.Errorf("failed to process wait, %v", err)
-		} else if d.HostMountpoint() != "" || config.GetFsDriver() != config.FsDriverFscache {
+		} else if d.HostMountpoint() != "" && config.GetFsDriver() == config.FsDriverFusedev {
 			// No need to umount if the nydusd never performs mount. In other word, it does not
 			// associate with a host mountpoint.
 			if err := mount.WaitUntilUnmounted(d.HostMountpoint()); err != nil {
@@ -534,7 +576,7 @@ func (d *Daemon) Wait() error {
 func (d *Daemon) ClearVestige() {
 	mounter := mount.Mounter{}
 	if d.States.FsDriver == config.FsDriverFscache {
-		instances := d.Instances.List()
+		instances := d.RafsCache.List()
 		for _, i := range instances {
 			if err := mounter.Umount(i.GetMountpoint()); err != nil {
 				log.L.Warnf("Can't umount %s, %v", d.States.Mountpoint, err)
@@ -559,11 +601,45 @@ func (d *Daemon) ClearVestige() {
 	d.ResetClient()
 }
 
+func (d *Daemon) CloneRafsInstances(src *Daemon) {
+	instances := src.RafsCache.List()
+	d.RafsCache.SetIntances(instances)
+}
+
+// Daemon must be started and reach RUNNING state before call this method
+func (d *Daemon) RecoverRafsInstances() error {
+	if d.IsSharedDaemon() {
+		d.RafsCache.Lock()
+		defer d.RafsCache.Unlock()
+
+		instances := make([]*rafs.Rafs, 0, 16)
+		for _, r := range d.RafsCache.ListLocked() {
+			instances = append(instances, r)
+		}
+
+		sort.Slice(instances, func(i, j int) bool {
+			return instances[i].Seq < instances[j].Seq
+		})
+
+		for _, i := range instances {
+			if d.HostMountpoint() != i.GetMountpoint() {
+				log.L.Infof("Recovered mount instance %s", i.SnapshotID)
+				if err := d.SharedMount(i); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 // Instantiate a daemon object
 func NewDaemon(opt ...NewDaemonOpt) (*Daemon, error) {
 	d := &Daemon{}
 	d.States.ID = newID()
-	d.Instances = rafsSet{instances: make(map[string]*Rafs)}
+	d.States.DaemonMode = config.DaemonModeDedicated
+	d.RafsCache = rafs.NewRafsCache()
 
 	for _, o := range opt {
 		err := o(d)

pkg/daemon/rafs.go

@@ -1,248 +0,0 @@
-/*
- * Copyright (c) 2022. Nydus Developers. All rights reserved.
- *
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package daemon
-
-import (
-	"os"
-	"path"
-	"path/filepath"
-	"sort"
-	"sync"
-
-	"github.com/mohae/deepcopy"
-	"github.com/pkg/errors"
-
-	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/log"
-
-	"github.com/containerd/nydus-snapshotter/config"
-)
-
-const (
-	AnnoFsCacheDomainID string = "fscache.domainid"
-	AnnoFsCacheID       string = "fscache.id"
-)
-
-type NewRafsOpt func(r *Rafs) error
-
-func init() {
-	// A set of rafs instances associates to a nydusd daemon or not
-	RafsSet = rafsSet{instances: make(map[string]*Rafs)}
-}
-
-var RafsSet rafsSet
-
-type rafsSet struct {
-	mu        sync.Mutex
-	instances map[string]*Rafs
-}
-
-func (rs *rafsSet) Lock() {
-	rs.mu.Lock()
-}
-
-func (rs *rafsSet) Unlock() {
-	rs.mu.Unlock()
-}
-
-func (rs *rafsSet) Add(r *Rafs) {
-	rs.mu.Lock()
-	rs.instances[r.SnapshotID] = r
-	rs.mu.Unlock()
-}
-
-func (rs *rafsSet) Remove(snapshotID string) {
-	rs.mu.Lock()
-	delete(rs.instances, snapshotID)
-	rs.mu.Unlock()
-}
-
-func (rs *rafsSet) Get(snapshotID string) *Rafs {
-	rs.mu.Lock()
-	defer rs.mu.Unlock()
-
-	return rs.instances[snapshotID]
-}
-
-func (rs *rafsSet) Len() int {
-	rs.mu.Lock()
-	len := len(rs.instances)
-	rs.mu.Unlock()
-
-	return len
-}
-
-func (rs *rafsSet) Head() *Rafs {
-	rs.mu.Lock()
-	defer rs.mu.Unlock()
-	for _, v := range rs.instances {
-		return v
-	}
-
-	return nil
-}
-
-func (rs *rafsSet) List() map[string]*Rafs {
-	rs.mu.Lock()
-	defer rs.mu.Unlock()
-
-	instances := deepcopy.Copy(rs.instances).(map[string]*Rafs)
-
-	return instances
-}
-
-func (rs *rafsSet) ListUnlocked() map[string]*Rafs {
-	return rs.instances
-}
-
-// The whole struct will be persisted
-type Rafs struct {
-	Seq uint64
-	// Given by containerd
-	SnapshotID string
-	// Usually is the image reference
-	ImageID     string
-	DaemonID    string
-	SnapshotDir string
-	// 1. A host kernel EROFS mountpoint
-	// 2. Absolute path to each rafs instance root directory.
-	Mountpoint  string
-	Annotations map[string]string
-}
-
-func NewRafs(snapshotID, imageID string) (*Rafs, error) {
-	snapshotDir := path.Join(config.GetSnapshotsRootDir(), snapshotID)
-	rafs := &Rafs{SnapshotID: snapshotID,
-		ImageID:     imageID,
-		SnapshotDir: snapshotDir,
-		Annotations: make(map[string]string),
-	}
-
-	if err := os.MkdirAll(snapshotDir, 0755); err != nil {
-		return nil, err
-	}
-
-	RafsSet.Add(rafs)
-
-	return rafs, nil
-}
-
-func (r *Rafs) AddAnnotation(k, v string) {
-	r.Annotations[k] = v
-}
-
-func (r *Rafs) SetMountpoint(mp string) {
-	r.Mountpoint = mp
-}
-
-func (r *Rafs) GetSnapshotDir() string {
-	return r.SnapshotDir
-}
-
-// Mountpoint for nydusd within single kernel mountpoint(FUSE mount). Each mountpoint
-// is create by API based pseudo mount. `RootMountPoint` is real mountpoint
-// where to perform the kernel mount.
-// Nydusd API based mountpoint must start with "/", otherwise nydusd API server returns error.
-func (r *Rafs) RelaMountpoint() string {
-	return filepath.Join("/", r.SnapshotID)
-}
-
-// Reflects the path where the a rafs instance root stays. The
-// root path could be a host kernel mountpoint when the instance
-// is attached by API `POST /api/v1/mount?mountpoint=/` or nydusd mounts an instance directly when starting.
-// Generally, we use this method to get the path as overlayfs lowerdir.
-// The path includes container image rootfs.
-func (r *Rafs) GetMountpoint() string {
-	return r.Mountpoint
-}
-
-func (r *Rafs) BootstrapFile() (string, error) {
-	// meta files are stored at <snapshot_id>/fs/image/image.boot
-	bootstrap := filepath.Join(r.SnapshotDir, "fs", "image", "image.boot")
-	_, err := os.Stat(bootstrap)
-	if err == nil {
-		return bootstrap, nil
-	}
-
-	if os.IsNotExist(err) {
-		// check legacy location for backward compatibility
-		bootstrap = filepath.Join(r.SnapshotDir, "fs", "image.boot")
-		_, err = os.Stat(bootstrap)
-		if err == nil {
-			return bootstrap, nil
-		}
-	}
-
-	return "", errors.Wrapf(errdefs.ErrNotFound, "bootstrap %s", bootstrap)
-}
-
-// Blob caches' chunk bitmap and meta headers are stored here.
-func (r *Rafs) FscacheWorkDir() string {
-	return filepath.Join(r.SnapshotDir, "fs")
-}
-
-func (d *Daemon) UmountAllInstances() error {
-	d.Instances.Lock()
-	defer d.Instances.Unlock()
-
-	instances := d.Instances.ListUnlocked()
-
-	for _, r := range instances {
-		if err := d.SharedUmount(r); err != nil {
-			return errors.Wrapf(err, "umount fs instance %s", r.SnapshotID)
-		}
-	}
-
-	return nil
-}
-
-func (d *Daemon) CloneInstances(src *Daemon) {
-	src.Instances.Lock()
-	defer src.Instances.Unlock()
-
-	instances := src.Instances.ListUnlocked()
-
-	d.Lock()
-	defer d.Unlock()
-	d.Instances.instances = instances
-}
-
-func (d *Daemon) UmountInstance(r *Rafs) error {
-	if r.Mountpoint != d.States.Mountpoint {
-		if err := d.SharedUmount(r); err != nil {
-			return errors.Wrapf(err, "umount fs instance %s", r.SnapshotID)
-		}
-	}
-
-	return nil
-}
-
-// Daemon must be started and reach RUNNING state before call this method
-func (d *Daemon) RecoveredMountInstances() error {
-	d.Instances.Lock()
-	defer d.Instances.Unlock()
-
-	var instances []*Rafs
-	for _, r := range d.Instances.ListUnlocked() {
-		instances = append(instances, r)
-	}
-
-	sort.Slice(instances, func(i, j int) bool {
-		return instances[i].Seq < instances[j].Seq
-	})
-
-	for _, i := range instances {
-		if d.HostMountpoint() != i.GetMountpoint() {
-			log.L.Infof("Recovered mount instance %s", i.SnapshotID)
-			if err := d.SharedMount(i); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}

pkg/daemon/types/types.go

@@ -17,12 +17,6 @@ type BuildTimeInfo struct {
 
 type DaemonState string
 
-type DaemonInfo struct {
-	ID      string        `json:"id"`
-	Version BuildTimeInfo `json:"version"`
-	State   DaemonState   `json:"state"`
-}
-
 const (
 	DaemonStateUnknown   DaemonState = "UNKNOWN"
 	DaemonStateInit      DaemonState = "INIT"
@@ -32,6 +26,12 @@ const (
 	DaemonStateDestroyed DaemonState = "DESTROYED"
 )
 
+type DaemonInfo struct {
+	ID      string        `json:"id"`
+	Version BuildTimeInfo `json:"version"`
+	State   DaemonState   `json:"state"`
+}
+
 func (info *DaemonInfo) DaemonState() DaemonState {
 	return info.State
 }

pkg/encryption/encryption.go

@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2023. Nydus Developers. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package encryption
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/errdefs"
+
+	"github.com/containers/ocicrypt"
+	encconfig "github.com/containers/ocicrypt/config"
+	enchelpers "github.com/containers/ocicrypt/helpers"
+	encocispec "github.com/containers/ocicrypt/spec"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Copied from containerd/imgcrypt project, copyright The imgcrypt Authors.
+// https://github.com/containerd/imgcrypt/blob/e7500301cabcc9f3cab3daee3f541079b509e95f/images/encryption/encryption.go#LL82C5-L82C5
+// encryptLayer encrypts the layer using the CryptoConfig and creates a new OCI Descriptor.
+// A call to this function may also only manipulate the wrapped keys list.
+// The caller is expected to store the returned encrypted data and OCI Descriptor
+func encryptLayer(cc *encconfig.CryptoConfig, dataReader content.ReaderAt, desc ocispec.Descriptor) (ocispec.Descriptor, io.Reader, ocicrypt.EncryptLayerFinalizer, error) {
+	var (
+		size int64
+		d    digest.Digest
+		err  error
+	)
+
+	encLayerReader, encLayerFinalizer, err := ocicrypt.EncryptLayer(cc.EncryptConfig, ocicrypt.ReaderFromReaderAt(dataReader), desc)
+	if err != nil {
+		return ocispec.Descriptor{}, nil, nil, err
+	}
+
+	// were data touched ?
+	if encLayerReader != nil {
+		size = 0
+		d = ""
+	} else {
+		size = desc.Size
+		d = desc.Digest
+	}
+
+	newDesc := ocispec.Descriptor{
+		Digest:   d,
+		Size:     size,
+		Platform: desc.Platform,
+	}
+
+	switch desc.MediaType {
+	case images.MediaTypeDockerSchema2LayerGzip:
+		newDesc.MediaType = encocispec.MediaTypeLayerGzipEnc
+	case images.MediaTypeDockerSchema2Layer:
+		newDesc.MediaType = encocispec.MediaTypeLayerEnc
+	case encocispec.MediaTypeLayerGzipEnc:
+		newDesc.MediaType = encocispec.MediaTypeLayerGzipEnc
+	case encocispec.MediaTypeLayerZstdEnc:
+		newDesc.MediaType = encocispec.MediaTypeLayerZstdEnc
+	case encocispec.MediaTypeLayerEnc:
+		newDesc.MediaType = encocispec.MediaTypeLayerEnc
+
+	// TODO: Mediatypes to be added in ocispec
+	case ocispec.MediaTypeImageLayerGzip:
+		newDesc.MediaType = encocispec.MediaTypeLayerGzipEnc
+	case ocispec.MediaTypeImageLayerZstd:
+		newDesc.MediaType = encocispec.MediaTypeLayerZstdEnc
+	case ocispec.MediaTypeImageLayer:
+		newDesc.MediaType = encocispec.MediaTypeLayerEnc
+
+	default:
+		return ocispec.Descriptor{}, nil, nil, fmt.Errorf("unsupporter layer MediaType: %s", desc.MediaType)
+	}
+
+	return newDesc, encLayerReader, encLayerFinalizer, nil
+}
+
+// Copied from containerd/imgcrypt project, copyright The imgcrypt Authors.
+// https://github.com/containerd/imgcrypt/blob/e7500301cabcc9f3cab3daee3f541079b509e95f/images/encryption/encryption.go#LL164C11-L164C11
+// decryptLayer decrypts the layer using the CryptoConfig and creates a new OCI Descriptor.
+// The caller is expected to store the returned plain data and OCI Descriptor
+func decryptLayer(cc *encconfig.CryptoConfig, dataReader content.ReaderAt, desc ocispec.Descriptor, unwrapOnly bool) (ocispec.Descriptor, io.Reader, error) {
+	resultReader, d, err := ocicrypt.DecryptLayer(cc.DecryptConfig, ocicrypt.ReaderFromReaderAt(dataReader), desc, unwrapOnly)
+	if err != nil || unwrapOnly {
+		return ocispec.Descriptor{}, nil, err
+	}
+
+	newDesc := ocispec.Descriptor{
+		Digest:   d,
+		Size:     0,
+		Platform: desc.Platform,
+	}
+
+	switch desc.MediaType {
+	case encocispec.MediaTypeLayerGzipEnc:
+		newDesc.MediaType = images.MediaTypeDockerSchema2LayerGzip
+	case encocispec.MediaTypeLayerZstdEnc:
+		newDesc.MediaType = ocispec.MediaTypeImageLayerZstd
+	case encocispec.MediaTypeLayerEnc:
+		newDesc.MediaType = images.MediaTypeDockerSchema2Layer
+	default:
+		return ocispec.Descriptor{}, nil, fmt.Errorf("unsupporter layer MediaType: %s", desc.MediaType)
+	}
+	return newDesc, resultReader, nil
+}
+
+// Copied from containerd/imgcrypt project, copyright The imgcrypt Authors.
+// https://github.com/containerd/imgcrypt/blob/e7500301cabcc9f3cab3daee3f541079b509e95f/images/encryption/encryption.go#LL250C5-L250C5
+func ingestReader(ctx context.Context, cs content.Ingester, ref string, r io.Reader) (digest.Digest, int64, error) {
+	cw, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
+	if err != nil {
+		return "", 0, fmt.Errorf("failed to open writer: %w", err)
+	}
+	defer cw.Close()
+
+	if _, err := content.CopyReader(cw, r); err != nil {
+		return "", 0, fmt.Errorf("copy failed: %w", err)
+	}
+
+	st, err := cw.Status()
+	if err != nil {
+		return "", 0, fmt.Errorf("failed to get state: %w", err)
+	}
+
+	if err := cw.Commit(ctx, st.Offset, ""); err != nil {
+		if !errdefs.IsAlreadyExists(err) {
+			return "", 0, fmt.Errorf("failed commit on ref %q: %w", ref, err)
+		}
+	}
+
+	return cw.Digest(), st.Offset, nil
+}
+
+// Encrypt Nydus bootstrap layer
+func EncryptNydusBootstrap(ctx context.Context, cs content.Store, desc ocispec.Descriptor, encryptRecipients []string) (ocispec.Descriptor, error) {
+	var (
+		resultReader      io.Reader
+		newDesc           ocispec.Descriptor
+		encLayerFinalizer ocicrypt.EncryptLayerFinalizer
+	)
+
+	dataReader, err := cs.ReaderAt(ctx, desc)
+	if err != nil {
+		return ocispec.Descriptor{}, err
+	}
+	defer dataReader.Close()
+
+	cc, err := enchelpers.CreateCryptoConfig(encryptRecipients, []string{})
+	if err != nil {
+		return ocispec.Descriptor{}, fmt.Errorf("create encrypt config failed: %w", err)
+	}
+	newDesc, resultReader, encLayerFinalizer, err = encryptLayer(&cc, dataReader, desc)
+	if err != nil {
+		return ocispec.Descriptor{}, fmt.Errorf("failed to encrypt bootstrap layer: %w", err)
+	}
+	newDesc.Annotations = ocicrypt.FilterOutAnnotations(desc.Annotations)
+
+	// some operations, such as changing recipients, may not touch the layer at all
+	if resultReader != nil {
+		var ref string
+		// If we have the digest, write blob with checks
+		haveDigest := newDesc.Digest.String() != ""
+		if haveDigest {
+			ref = fmt.Sprintf("encrypted-bootstrap-%s", newDesc.Digest.String())
+		} else {
+			ref = fmt.Sprintf("encrypted-bootstrap-%d-%d", rand.Int(), rand.Int())
+		}
+
+		if haveDigest {
+			// Write blob if digest is known beforehand
+			if err := content.WriteBlob(ctx, cs, ref, resultReader, newDesc); err != nil {
+				return ocispec.Descriptor{}, fmt.Errorf("failed to write config: %w", err)
+			}
+		} else {
+			newDesc.Digest, newDesc.Size, err = ingestReader(ctx, cs, ref, resultReader)
+			if err != nil {
+				return ocispec.Descriptor{}, err
+			}
+		}
+	}
+
+	// After performing encryption, call finalizer to get annotations
+	if encLayerFinalizer != nil {
+		annotations, err := encLayerFinalizer()
+		if err != nil {
+			return ocispec.Descriptor{}, fmt.Errorf("error getting annotations from encLayer finalizer: %w", err)
+		}
+		for k, v := range annotations {
+			newDesc.Annotations[k] = v
+		}
+	}
+	return newDesc, err
+}
+
+// Decrypt the Nydus boostrap layer.
+// If unwrapOnly is set we will only try to decrypt the layer encryption key and return,
+// the layer itself won't be decrypted actually.
+func DeryptNydusBootstrap(ctx context.Context, cs content.Store, desc ocispec.Descriptor, decryptKeys []string, unwrapOnly bool) (ocispec.Descriptor, error) {
+	var (
+		resultReader io.Reader
+		newDesc      ocispec.Descriptor
+	)
+
+	dataReader, err := cs.ReaderAt(ctx, desc)
+	if err != nil {
+		return ocispec.Descriptor{}, err
+	}
+	defer dataReader.Close()
+
+	cc, err := enchelpers.CreateCryptoConfig([]string{}, decryptKeys)
+	if err != nil {
+		return ocispec.Descriptor{}, fmt.Errorf("create decrypt config failed: %w", err)
+	}
+	newDesc, resultReader, err = decryptLayer(&cc, dataReader, desc, unwrapOnly)
+	if err != nil || unwrapOnly {
+		return ocispec.Descriptor{}, fmt.Errorf("failed to decrypt bootstrap layer: %w", err)
+	}
+
+	newDesc.Annotations = ocicrypt.FilterOutAnnotations(desc.Annotations)
+
+	// some operations, such as changing recipients, may not touch the layer at all
+	if resultReader != nil {
+		var ref string
+		// If we have the digest, write blob with checks
+		haveDigest := newDesc.Digest.String() != ""
+		if haveDigest {
+			ref = fmt.Sprintf("decrypted-bootstrap-%s", newDesc.Digest.String())
+		} else {
+			ref = fmt.Sprintf("decrypted-bootstrap-%d-%d", rand.Int(), rand.Int())
+		}
+
+		if haveDigest {
+			// Write blob if digest is known beforehand
+			if err := content.WriteBlob(ctx, cs, ref, resultReader, newDesc); err != nil {
+				return ocispec.Descriptor{}, fmt.Errorf("failed to write config: %w", err)
+			}
+		} else {
+			newDesc.Digest, newDesc.Size, err = ingestReader(ctx, cs, ref, resultReader)
+			if err != nil {
+				return ocispec.Descriptor{}, err
+			}
+		}
+	}
+	return newDesc, err
+}

pkg/errdefs/errors.go

@@ -11,12 +11,13 @@ import (
 	"net"
 	"syscall"
 
+	"github.com/containerd/errdefs"
 	"github.com/pkg/errors"
 )
 
 var (
-	ErrAlreadyExists   = errors.New("already exists")
-	ErrNotFound        = errors.New("not found")
+	ErrAlreadyExists   = errdefs.ErrAlreadyExists
+	ErrNotFound        = errdefs.ErrNotFound
 	ErrInvalidArgument = errors.New("invalid argument")
 	ErrUnavailable     = errors.New("unavailable")
 	ErrNotImplemented  = errors.New("not implemented") // represents not supported and unimplemented

pkg/fanotify/conn/conn.go

@@ -8,16 +8,29 @@ package conn
 
 import (
 	"bufio"
-	"io"
+	"encoding/json"
 )
 
 type Client struct {
-	Scanner *bufio.Scanner
+	Reader *bufio.Reader
 }
 
-func (c *Client) GetPath() (string, error) {
-	if !c.Scanner.Scan() { // NOTE: no timeout
-		return "", io.EOF
-	}
-	return c.Scanner.Text(), nil
+type EventInfo struct {
+	Path    string `json:"path"`
+	Size    uint32 `json:"size"`
+	Elapsed uint64 `json:"elapsed"`
+}
+
+func (c *Client) GetEventInfo() (*EventInfo, error) {
+	eventInfo := EventInfo{}
+
+	eventByte, err := c.Reader.ReadBytes('\n')
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(eventByte, &eventInfo); err != nil {
+		return nil, err
+	}
+
+	return &eventInfo, nil
 }