Merge pull request #2105 from containerd/dependabot/go_modules/containerd-2f3dabfc34

build(deps): bump the containerd group across 2 directories with 1 update

.github/dependabot.yml

@@ -3,20 +3,43 @@ updates:
 
   # Automatic upgrade for go modules.
   - package-ecosystem: "gomod"
-    directory: "/"
+    directories:
+      - "/estargz"
+      - "/ipfs"
+      - "/"
+      - "/cmd"
     schedule:
       interval: "daily"
     ignore:
       # We upgrade this manually on each release
       - dependency-name: "github.com/containerd/stargz-snapshotter/estargz"
-      # This forcefully points to v1.22.1. See go.mod.
-      - dependency-name: "github.com/urfave/cli"
-
-  # Automatic upgrade for go modules of estargz package.
-  - package-ecosystem: "gomod"
-    directory: "/estargz"
-    schedule:
-      interval: "daily"
+    groups:
+      golang-x:
+        patterns:
+          - "golang.org/x/*"
+      google-golang:
+        patterns:
+          - "google.golang.org/*"
+      containerd:
+        patterns:
+          - "github.com/containerd/*"
+      opencontainers:
+        patterns:
+          - "github.com/opencontainers/*"
+      k8s:
+        patterns:
+          - "k8s.io/*"
+      gomod:
+        # this pattern covers all go dependencies that are not in
+        # the above groups. dependabot doesn't seem to update sub-modules if
+        # a dependency doesn't belong to a group, so we define this group
+        # explicitly.
+        exclude-patterns:
+          - "golang.org/x/*"
+          - "google.golang.org/*"
+          - "github.com/containerd/*"
+          - "github.com/opencontainers/*"
+          - "k8s.io/*"
 
   # Automatic upgrade for base images used in the Dockerfile
   - package-ecosystem: "docker"

.github/workflows/benchmark.yml

@@ -1,19 +1,23 @@
 name: Benchmark
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 
 env:
   DOCKER_BUILDKIT: 1
 
 jobs:
   hello-bench:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: HelloBench
     env:
       BENCHMARK_LOG_DIR: ${{ github.workspace }}/log/
       BENCHMARK_RESULT_DIR: ${{ github.workspace }}/benchmark/
       BENCHMARK_REGISTRY: ghcr.io
       BENCHMARK_USER: stargz-containers
-      BENCHMARK_TARGETS: python:3.9 gcc:10.2.0 postgres:13.1 tomcat:10.0.0-jdk15-openjdk-buster
+      BENCHMARK_TARGETS: python:3.10 gcc:11.2.0 postgres:14.2 tomcat:10.1.0-jdk17-openjdk-bullseye
       BENCHMARK_SAMPLES_NUM: 5
       BENCHMARK_PERCENTILE: 95
       BENCHMARK_PERCENTILES_GRANULARITY: 25
@@ -25,9 +29,9 @@ jobs:
     steps:
     - name: Install tools
       run: |
-          sudo apt-get update && sudo apt-get --no-install-recommends install -y gnuplot
-          pip install numpy
-    - uses: actions/checkout@v2
+          sudo apt-get update && \
+          sudo apt-get install -y gnuplot python3-numpy
+    - uses: actions/checkout@v4
     - name: Prepare directories
       run: mkdir "${BENCHMARK_RESULT_DIR}" "${BENCHMARK_LOG_DIR}"
     - name: Get instance information
@@ -39,7 +43,7 @@ jobs:
       env:
         BENCHMARK_RUNTIME_MODE: ${{ matrix.runtime }}
       run: make benchmark
-    - uses: actions/upload-artifact@v2.2.4
+    - uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
         name: benchmarking-result-${{ matrix.runtime }}

.github/workflows/kind-image.yml

@@ -0,0 +1,51 @@
+name: Kind image
+on:
+  push:
+    tags:
+      - 'v*'
+  pull_request:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  kind-image:
+    runs-on: ubuntu-24.04
+    name: Kind image
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}-kind
+
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v7.0.0-28
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        uses: docker/build-push-action@v6.18.0
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64

.github/workflows/nightly.yml

@@ -21,77 +21,87 @@ env:
 
 jobs:
   integration:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Integration
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run integration test
       run: make integration
 
   test-optimize:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Optimize
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test for optimize subcommand of ctr-remote
       run: make test-optimize
 
   test-kind:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Kind
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test for pulling image from private registry on Kubernetes
       run: make test-kind
 
   test-criauth:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: CRIAuth
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test for pulling image from private registry on Kubernetes
       run: make test-criauth
 
   test-cri-containerd:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: CRIValidationContainerd
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Varidate the runtime through CRI with containerd
       run: make test-cri-containerd
 
   test-cri-o:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: CRIValidationCRIO
     steps:
-    - uses: actions/checkout@v2
+    - name: Install the latest docker
+      run: |
+        sudo apt-get remove moby-cli moby-engine
+        wget -O get-docker.sh https://get.docker.com
+        sh get-docker.sh
+    - uses: actions/checkout@v4
     - name: Varidate the runtime through CRI with CRI-O
-      run: make test-cri-o
+      env:
+        DOCKER_BUILD_ARGS: "--build-arg=RUNC_VERSION=v1.0.3"
+      run: |
+        # needed to pass "runtime should output OOMKilled reason" test
+        sudo swapoff -a
+        make test-cri-o
 
   test-k3s:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: K3S
     steps:
-    - uses: actions/setup-go@v2
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.16.x'
+        go-version: '1.24.x'
     - name: Install k3d
       run: |
-        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.0.0/install.sh | bash
+        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.6.3/install.sh | bash
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
     - name: Install yq
       run: |
         sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64
         sudo chmod +x /usr/local/bin/yq
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test with k3s
       run: make test-k3s

.github/workflows/release.yml

@@ -9,7 +9,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Build
     strategy:
       matrix:
@@ -17,7 +17,7 @@ jobs:
     env:
       OUTPUT_DIR: ${{ github.workspace }}/out
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build Binary
       env:
         DOCKER_BUILDKIT: 1
@@ -29,26 +29,28 @@ jobs:
         if [ "${ARCH_ID}" == "arm-v7" ] ; then
           BUILD_ARGS="--build-arg=TARGETARCH=arm --build-arg=GOARM=7"
         fi
+        # make binaries static
+        BUILD_ARGS="$BUILD_ARGS --build-arg=CGO_ENABLED=0"
         TAR_FILE_NAME="stargz-snapshotter-${RELEASE_TAG}-linux-${ARCH_ID}.tar.gz"
         SHA256SUM_FILE_NAME="${TAR_FILE_NAME}.sha256sum"
         docker build ${BUILD_ARGS} --target release-binaries -o - . | gzip > "${OUTPUT_DIR}/${TAR_FILE_NAME}"
         ( cd ${OUTPUT_DIR}; sha256sum ${TAR_FILE_NAME} ) > "${OUTPUT_DIR}/${SHA256SUM_FILE_NAME}"
     - name: Save Binary
-      uses: actions/upload-artifact@v2.2.4
+      uses: actions/upload-artifact@v4
       with:
         name: builds-${{ matrix.arch }}
         path: ${{ env.OUTPUT_DIR }}/*
 
   release:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Release
     needs: [build]
     env:
       OUTPUT_DIR: ${{ github.workspace }}/builds
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Download Builds
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v5
       with:
         path: ${{ env.OUTPUT_DIR }}
     - name: Create Release
@@ -57,15 +59,11 @@ jobs:
       run: |
         RELEASE_TAG="${GITHUB_REF##*/}"
         cat <<EOF > ${GITHUB_WORKSPACE}/release-note.txt
-        ${RELEASE_TAG}
-
         (TBD)
         EOF
-        ASSET_FLAGS=()
+        ASSET_ARGS=()
         ls -al ${OUTPUT_DIR}/
         for A in "amd64" "arm-v7" "arm64" "ppc64le" "s390x" ; do
-          for F in ${OUTPUT_DIR}/builds-${A}/* ; do
-            ASSET_FLAGS+=("-a" "$F")
-          done
+          ASSET_ARGS+=("${OUTPUT_DIR}/builds-${A}/*")
         done
-        hub release create "${ASSET_FLAGS[@]}" -F ${GITHUB_WORKSPACE}/release-note.txt --draft "${RELEASE_TAG}"
+        gh release create -F ${GITHUB_WORKSPACE}/release-note.txt --draft --title "${RELEASE_TAG}" "${RELEASE_TAG}" ${ASSET_ARGS[@]}

.github/workflows/tests.yml

@@ -1,38 +1,53 @@
 name: Tests
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 
 env:
   DOCKER_BUILDKIT: 1
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Build
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build all
       run: ./script/util/make.sh build -j2
 
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Test
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Test all
       run: ./script/util/make.sh test-all -j2
 
   linter:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Linter
+    strategy:
+      fail-fast: false
+      matrix:
+        targetdir: [".", "./estargz", "./cmd", "./ipfs"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: '0'
-    - name: Run Linter
-      run: ./script/util/make.sh install-check-tools check
+    - uses: actions/setup-go@v5
+      with:
+        go-version: '1.24.x'
+    - name: golangci-lint
+      uses: golangci/golangci-lint-action@v8.0.0
+      with:
+        version: v2.1
+        args: --verbose --timeout=10m
+        working-directory: ${{ matrix.targetdir }}
 
   integration:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Integration
     strategy:
       fail-fast: false
@@ -40,6 +55,9 @@ jobs:
         buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version
         builtin: ["true", "false"]
         metadata-store: ["memory", "db"]
+        fuse-passthrough: ["true", "false"]
+        fuse-manager: ["true", "false"]
+        transfer-service: ["true", "false"]
         exclude:
         - buildargs: ""
           builtin: "true"
@@ -47,19 +65,40 @@ jobs:
           builtin: "true"
         - metadata-store: "db"
           buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - fuse-passthrough: "true"
+          builtin: "true"
+        - fuse-passthrough: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - fuse-passthrough: "true"
+          metadata-store: "db"
+        - fuse-manager: "true"
+          builtin: "true"
+        - fuse-manager: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - transfer-service: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - transfer-service: "true"
+          builtin: "true"
+        - transfer-service: "true"
+          metadata-store: "db"
+        - transfer-service: "true"
+          fuse-passthrough: "true"
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run integration test
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
         BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
         METADATA_STORE: ${{ matrix.metadata-store }}
+        FUSE_PASSTHROUGH: ${{ matrix.fuse-passthrough }}
+        FUSE_MANAGER: ${{ matrix.fuse-manager }}
+        TRANSFER_SERVICE: ${{ matrix.transfer-service }}
       run: make integration
 
   test-optimize:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Optimize
     strategy:
       fail-fast: false
@@ -68,14 +107,14 @@ jobs:
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test for optimize subcommand of ctr-remote
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
       run: make test-optimize
 
   test-kind:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: Kind
     strategy:
       fail-fast: false
@@ -88,7 +127,7 @@ jobs:
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test for pulling image from private registry on Kubernetes
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
@@ -96,7 +135,7 @@ jobs:
       run: make test-kind
 
   test-criauth:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: CRIAuth
     strategy:
       fail-fast: false
@@ -109,7 +148,7 @@ jobs:
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test for pulling image from private registry on Kubernetes with CRI keychain mode
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
@@ -117,7 +156,7 @@ jobs:
       run: make test-criauth
 
   test-cri-containerd:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: CRIValidationContainerd
     strategy:
       fail-fast: false
@@ -125,6 +164,9 @@ jobs:
         buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version
         builtin: ["true", "false"]
         metadata-store: ["memory", "db"]
+        fuse-passthrough: ["true", "false"]
+        fuse-manager: ["true", "false"]
+        transfer-service: ["true", "false"]
         exclude:
         - buildargs: ""
           builtin: "true"
@@ -132,68 +174,119 @@ jobs:
           builtin: "true"
         - metadata-store: "db"
           buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - fuse-passthrough: "true"
+          builtin: "true"
+        - fuse-passthrough: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - fuse-passthrough: "true"
+          metadata-store: "db"
+        - fuse-manager: "true"
+          builtin: "true"
+        - fuse-manager: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - transfer-service: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - transfer-service: "true"
+          builtin: "true"
+        - transfer-service: "true"
+          metadata-store: "db"
+        - transfer-service: "true"
+          fuse-passthrough: "true"
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Validate containerd through CRI
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
         BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
         METADATA_STORE: ${{ matrix.metadata-store }}
+        FUSE_PASSTHROUGH: ${{ matrix.fuse-passthrough }}
+        FUSE_MANAGER: ${{ matrix.fuse-manager }}
+        TRANSFER_SERVICE: ${{ matrix.transfer-service }}
       run: make test-cri-containerd
 
   test-cri-cri-o:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: CRIValidationCRIO
     strategy:
       fail-fast: false
       matrix:
         metadata-store: ["memory", "db"]
     steps:
-    - uses: actions/checkout@v2
+    - name: Install the latest docker
+      run: |
+        sudo apt-get remove moby-cli moby-engine
+        wget -O get-docker.sh https://get.docker.com
+        sh get-docker.sh
+    - uses: actions/checkout@v4
     - name: Validate CRI-O through CRI
       env:
+        DOCKER_BUILD_ARGS: "--build-arg=RUNC_VERSION=v1.0.3"
         METADATA_STORE: ${{ matrix.metadata-store }}
-      run: make test-cri-o
+      run: |
+        # needed to pass "runtime should output OOMKilled reason" test
+        sudo swapoff -a
+        make test-cri-o
+
+  test-podman:
+    runs-on: ubuntu-24.04
+    name: PodmanRootless
+    steps:
+    - uses: actions/checkout@v4
+    - name: Test Podman (rootless)
+      run: make test-podman
 
   test-k3s:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: K3S
     steps:
-    - uses: actions/setup-go@v2
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.16.x'
+        go-version: '1.24.x'
     - name: Install k3d
       run: |
-        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.0.0/install.sh | bash
+        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.6.3/install.sh | bash
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
     - name: Install yq
       run: |
         sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64
         sudo chmod +x /usr/local/bin/yq
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test with k3s
       run: make test-k3s
 
+  test-ipfs:
+    runs-on: ubuntu-24.04
+    name: IPFS
+    steps:
+    - uses: actions/checkout@v4
+    - name: Run test
+      run: make test-ipfs
+
   test-k3s-argo-workflow:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: K3SArgoWorkflow
     env:
       RESULT_DIR: ${{ github.workspace }}/argo-workflow/
     steps:
-    - uses: actions/setup-go@v2
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.16.x'
+        go-version: '1.24.x'
     - name: Install k3d
       run: |
-        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.0.0/install.sh | bash
+        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.6.3/install.sh | bash
     - name: Install argo worklflow
       run: |
         wget -q https://github.com/argoproj/argo-workflows/releases/download/v3.0.10/argo-linux-amd64.gz
         gunzip argo-linux-amd64.gz
         sudo mv argo-linux-amd64 /usr/local/bin/argo
         sudo chmod +x /usr/local/bin/argo
-    - uses: actions/checkout@v2
+    - name: Workaround for freeing up more disk space
+      # https://github.com/actions/runner-images/issues/2606
+      run: |
+        sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
+        sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
+    - uses: actions/checkout@v4
     - name: Prepare directories
       run: mkdir "${RESULT_DIR}"
     - name: Get instance information
@@ -205,7 +298,7 @@ jobs:
       env:
         RESULT: ${{ env.RESULT_DIR }}/result.json
       run: make test-k3s-argo-workflow
-    - uses: actions/upload-artifact@v2.2.4
+    - uses: actions/upload-artifact@v4
       with:
         name: k3s-argo-workflow
         path: ${{ env.RESULT_DIR }}
@@ -218,19 +311,43 @@ jobs:
 
   project:
     name: Project Checks
-    runs-on: ubuntu-20.04
-    timeout-minutes: 5
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
     steps:
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.17.x'
-      - uses: actions/checkout@v2
+          go-version: '1.24.x'
+      - uses: actions/checkout@v4
         with:
           path: src/github.com/containerd/stargz-snapshotter
           fetch-depth: 25
-      - uses: containerd/project-checks@v1
+      - uses: containerd/project-checks@v1.2.2
         with:
           working-directory: src/github.com/containerd/stargz-snapshotter
+          # go-licenses-ignore is set because go-licenses cannot correctly detect the license of the following packages:
+          # * estargz packages: Apache-2.0 and BSD-3-Clause dual license
+          #   (https://github.com/containerd/stargz-snapshotter/blob/main/NOTICE.md)
+          #
+          # The list of the CNCF-approved licenses can be found here:
+          # https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md
+          #
+          # hashicorp packages: MPL-2.0
+          # (https://github.com/hashicorp/go-cleanhttp/blob/master/LICENSE,
+          #  https://github.com/hashicorp/go-retryablehttp/blob/master/LICENSE)
+          # Note: MPL-2.0 is not in the CNCF-approved licenses list, but these packages are allowed as exceptions.
+          # See CNCF licensing exceptions:
+          # https://github.com/cncf/foundation/blob/main/license-exceptions/CNCF-licensing-exceptions.csv
+          go-licenses-ignore: |
+            github.com/containerd/stargz-snapshotter/estargz
+            github.com/containerd/stargz-snapshotter/estargz/errorutil
+            github.com/containerd/stargz-snapshotter/estargz/externaltoc
+            github.com/containerd/stargz-snapshotter/estargz/zstdchunked
+            github.com/hashicorp/go-cleanhttp
+            github.com/hashicorp/go-retryablehttp
       - name: Check proto generated code
         run: make validate-generated
         working-directory: src/github.com/containerd/stargz-snapshotter
+      - run: ./script/util/verify-no-patent.sh
+        working-directory: src/github.com/containerd/stargz-snapshotter
+      - run: make validate-vendor
+        working-directory: src/github.com/containerd/stargz-snapshotter

.golangci.yml

@@ -1,26 +1,54 @@
-# This is applied to `estargz` submodule as well.
-# https://golangci-lint.run/usage/configuration#config-file
-
+version: "2"
 linters:
   enable:
-    - structcheck
-    - varcheck
-    - staticcheck
-    - unconvert
-    - gofmt
-    - goimports
-    - revive
-    - ineffassign
-    - vet
-    - unused
+    - depguard
     - misspell
+    - revive
+    - unconvert
   disable:
     - errcheck
-
-run:
-  deadline: 4m
-  skip-dirs:
-    - docs
-    - images
-    - out
-    - script
+  settings:
+    depguard:
+      rules:
+        main:
+          deny:
+            - pkg: github.com/containerd/containerd/errdefs
+              desc: The containerd errdefs package was migrated to a separate module. Use github.com/containerd/errdefs instead.
+            - pkg: github.com/containerd/containerd/log
+              desc: The containerd log package was migrated to a separate module. Use github.com/containerd/log instead.
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - linters:
+          - revive
+        text: unused-parameter
+      - linters:
+          - revive
+        text: redefines-builtin-id
+    paths:
+      - docs
+      - images
+      - out
+      - script
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - docs
+      - images
+      - out
+      - script
+      - third_party$
+      - builtin$
+      - examples$

Dockerfile

@@ -12,40 +12,45 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-ARG CONTAINERD_VERSION=v1.6.0-beta.1
-ARG RUNC_VERSION=v1.0.2
-ARG CNI_PLUGINS_VERSION=v1.0.1
-ARG NERDCTL_VERSION=0.12.1
+ARG CONTAINERD_VERSION=v2.1.3
+ARG RUNC_VERSION=v1.3.0
+ARG CNI_PLUGINS_VERSION=v1.7.1
+ARG NERDCTL_VERSION=2.1.3
 
-ARG PODMAN_VERSION=v3.4.0
-ARG CRIO_VERSION=8d4df4ea25cd6446f91ee9944ac92c1c726cf475
-ARG CONMON_VERSION=v2.0.29
-ARG COMMON_VERSION=v0.42.1
-ARG CRIO_TEST_PAUSE_IMAGE_NAME=k8s.gcr.io/pause:3.5
+ARG PODMAN_VERSION=v5.5.2
+ARG CRIO_VERSION=v1.33.2
+ARG CONMON_VERSION=v2.1.13
+ARG COMMON_VERSION=v0.63.0
+ARG CRIO_TEST_PAUSE_IMAGE_NAME=registry.k8s.io/pause:3.6
+ARG NETAVARK_VERSION=v1.15.2
+
+ARG CONTAINERIZED_SYSTEMD_VERSION=v0.1.1
+ARG SLIRP4NETNS_VERSION=v1.3.3
+ARG PAUSE_IMAGE_NAME_TEST=registry.k8s.io/pause:3.10.1
 
 # Used in CI
-ARG CRI_TOOLS_VERSION=v1.22.0
+ARG CRI_TOOLS_VERSION=v1.30.1
 
 # Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg.
 # If TARGETARCH isn't supported by the builder, the default value is "amd64".
 
-FROM golang:1.17-bullseye AS golang-base
+FROM golang:1.24-bullseye AS golang-base
 
 # Build containerd
-FROM golang-base AS containerd-dev
+FROM --platform=$BUILDPLATFORM golang:1.24-bullseye AS containerd-dev
 ARG CONTAINERD_VERSION
-RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \
-    git clone -b ${CONTAINERD_VERSION} --depth 1 \
+ARG TARGETARCH
+RUN git clone -b ${CONTAINERD_VERSION} --depth 1 \
               https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \
     cd $GOPATH/src/github.com/containerd/containerd && \
-    make && DESTDIR=/out/ PREFIX= make install
+    GOARCH=$TARGETARCH make && DESTDIR=/out/ PREFIX= make install
 
 # Build containerd with builtin stargz snapshotter
-FROM golang-base AS containerd-snapshotter-dev
+FROM --platform=$BUILDPLATFORM golang:1.24-bullseye AS containerd-snapshotter-dev
 ARG CONTAINERD_VERSION
+ARG TARGETARCH
 COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
-RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \
-    git clone -b ${CONTAINERD_VERSION} --depth 1 \
+RUN git clone -b ${CONTAINERD_VERSION} --depth 1 \
       https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \
     cd $GOPATH/src/github.com/containerd/containerd && \
     echo 'require github.com/containerd/stargz-snapshotter v0.0.0' >> go.mod && \
@@ -61,10 +66,10 @@ RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \
       echo 'replace github.com/containerd/stargz-snapshotter/estargz => '$GOPATH'/src/github.com/containerd/stargz-snapshotter/estargz' >> integration/client/go.mod ; \
     fi && \
     echo 'package main \nimport _ "github.com/containerd/stargz-snapshotter/service/plugin"' > cmd/containerd/builtins_stargz_snapshotter.go && \
-    make vendor && make && DESTDIR=/out/ PREFIX= make install
+    make vendor && GOARCH=$TARGETARCH make && DESTDIR=/out/ PREFIX= make install
 
 # Build runc
-FROM golang-base AS runc-dev
+FROM golang:1.24-bullseye AS runc-dev
 ARG RUNC_VERSION
 RUN apt-get update -y && apt-get install -y libseccomp-dev && \
     git clone -b ${RUNC_VERSION} --depth 1 \
@@ -73,15 +78,17 @@ RUN apt-get update -y && apt-get install -y libseccomp-dev && \
     make && make install PREFIX=/out/
 
 # Build stargz snapshotter
-FROM golang-base AS snapshotter-dev
+FROM --platform=$BUILDPLATFORM golang:1.24-bullseye AS snapshotter-dev
 ARG TARGETARCH
 ARG GOARM
 ARG SNAPSHOTTER_BUILD_FLAGS
 ARG CTR_REMOTE_BUILD_FLAGS
 COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
+ARG CGO_ENABLED
 RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
     PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make containerd-stargz-grpc && \
-    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make ctr-remote
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make ctr-remote && \
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make stargz-fuse-manager
 
 # Build stargz store
 FROM golang-base AS stargz-store-dev
@@ -90,8 +97,9 @@ ARG GOARM
 ARG SNAPSHOTTER_BUILD_FLAGS
 ARG CTR_REMOTE_BUILD_FLAGS
 COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
+ARG CGO_ENABLED
 RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
-    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make stargz-store
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make stargz-store stargz-store-helper
 
 # Build podman
 FROM golang-base AS podman-dev
@@ -103,7 +111,8 @@ RUN apt-get update -y && apt-get install -y libseccomp-dev libgpgme-dev && \
     make && make install PREFIX=/out/
 
 # Build CRI-O
-FROM golang-base AS cri-o-dev
+# FROM golang-base AS cri-o-dev
+FROM golang:1.24-bullseye AS cri-o-dev
 ARG CRIO_VERSION
 RUN apt-get update -y && apt-get install -y libseccomp-dev libgpgme-dev && \
     git clone https://github.com/cri-o/cri-o $GOPATH/src/github.com/cri-o/cri-o && \
@@ -115,7 +124,7 @@ RUN apt-get update -y && apt-get install -y libseccomp-dev libgpgme-dev && \
 # Build conmon
 FROM golang-base AS conmon-dev
 ARG CONMON_VERSION
-RUN apt-get update -y && apt-get install -y gcc git libc6-dev libglib2.0-dev pkg-config make && \
+RUN apt-get update -y && apt-get install -y gcc git libc6-dev libglib2.0-dev pkg-config make libseccomp-dev && \
     git clone -b ${CONMON_VERSION} --depth 1 \
               https://github.com/containers/conmon $GOPATH/src/github.com/containers/conmon && \
     cd $GOPATH/src/github.com/containers/conmon && \
@@ -137,7 +146,7 @@ COPY --from=stargz-store-dev /out/* /
 FROM golang-base AS containerd-base
 ARG TARGETARCH
 ARG NERDCTL_VERSION
-RUN apt-get update -y && apt-get --no-install-recommends install -y fuse && \
+RUN apt-get update -y && apt-get --no-install-recommends install -y fuse3 && \
     curl -sSL --output /tmp/nerdctl.tgz https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${TARGETARCH:-amd64}.tar.gz && \
     tar zxvf /tmp/nerdctl.tgz -C /usr/local/bin && \
     rm -f /tmp/nerdctl.tgz
@@ -153,7 +162,7 @@ RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr
 FROM golang-base AS containerd-snapshotter-base
 ARG TARGETARCH
 ARG NERDCTL_VERSION
-RUN apt-get update -y && apt-get --no-install-recommends install -y fuse && \
+RUN apt-get update -y && apt-get --no-install-recommends install -y fuse3 && \
     curl -sSL --output /tmp/nerdctl.tgz https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${TARGETARCH:-amd64}.tar.gz && \
     tar zxvf /tmp/nerdctl.tgz -C /usr/local/bin && \
     rm -f /tmp/nerdctl.tgz
@@ -163,12 +172,13 @@ COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/
 RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr
 
 # Base image which contains podman with stargz-store
-FROM golang-base AS podman-base
+FROM ubuntu:24.04 AS podman-base
 ARG TARGETARCH
 ARG CNI_PLUGINS_VERSION
 ARG PODMAN_VERSION
-RUN apt-get update -y && apt-get --no-install-recommends install -y fuse libgpgme-dev \
-                         iptables libyajl-dev && \
+ARG NETAVARK_VERSION
+RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y fuse3 libgpgme-dev \
+                         iptables libyajl-dev curl ca-certificates libglib2.0 libseccomp-dev wget && \
     # Make CNI plugins manipulate iptables instead of nftables
     # as this test runs in a Docker container that network is configured with iptables.
     # c.f. https://github.com/moby/moby/issues/26824
@@ -177,21 +187,51 @@ RUN apt-get update -y && apt-get --no-install-recommends install -y fuse libgpgm
     curl -qsSL https://raw.githubusercontent.com/containers/podman/${PODMAN_VERSION}/cni/87-podman-bridge.conflist | tee /etc/cni/net.d/87-podman-bridge.conflist && \
     curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin
 
+RUN mkdir /tmp/netavark ; \
+    wget -O /tmp/netavark/netavark.gz https://github.com/containers/netavark/releases/download/${NETAVARK_VERSION}/netavark.gz ; \
+    gunzip /tmp/netavark/netavark.gz ; \
+    mkdir -p /usr/local/libexec/podman ; \
+    mv /tmp/netavark/netavark /usr/local/libexec/podman/ ; \
+    chmod 0755 /usr/local/libexec/podman/netavark
+
 COPY --from=podman-dev /out/bin/* /usr/local/bin/
 COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
 COPY --from=conmon-dev /out/bin/* /usr/local/bin/
 COPY --from=containers-common-dev /out/seccomp.json /usr/share/containers/
 COPY --from=stargz-store-dev /out/* /usr/local/bin/
 
-# Image which can be used as all-in-one single node demo environment
-FROM snapshotter-base AS cind
-COPY ./script/config/ /
-COPY ./script/cind/ /
-VOLUME /var/lib/containerd
-VOLUME /var/lib/containerd-stargz-grpc
-VOLUME /run/containerd-stargz-grpc
-ENV CONTAINERD_SNAPSHOTTER=stargz
-ENTRYPOINT [ "/entrypoint.sh" ]
+# Image for testing rootless Podman with Stargz Store.
+# This takes the same approach as nerdctl CI: https://github.com/containerd/nerdctl/blob/6341c8320984f7148b92dd33472d8eaca6dba756/Dockerfile#L302-L326
+FROM podman-base AS podman-rootless
+ARG CONTAINERIZED_SYSTEMD_VERSION
+ARG SLIRP4NETNS_VERSION
+RUN apt-get update -y && apt-get install -y \
+                         systemd systemd-sysv dbus dbus-user-session \
+                         openssh-server openssh-client uidmap
+RUN curl -o /usr/local/bin/slirp4netns --fail -L https://github.com/rootless-containers/slirp4netns/releases/download/${SLIRP4NETNS_VERSION}/slirp4netns-$(uname -m) && \
+    chmod +x /usr/local/bin/slirp4netns && \
+    curl -L -o /docker-entrypoint.sh https://raw.githubusercontent.com/AkihiroSuda/containerized-systemd/${CONTAINERIZED_SYSTEMD_VERSION}/docker-entrypoint.sh && \
+    chmod +x /docker-entrypoint.sh && \
+    curl -L -o /etc/containers/policy.json https://raw.githubusercontent.com/containers/skopeo/master/default-policy.json
+# storage.conf plugs Stargz Store into Podman as an Additional Layer Store
+COPY ./script/podman/config/storage.conf /home/rootless/.config/containers/storage.conf
+# Stargz Store systemd service for rootless Podman
+COPY ./script/podman/config/podman-rootless-stargz-store.service /home/rootless/.config/systemd/user/
+
+COPY ./script/podman/config/containers.conf /home/rootless/.config/containers/containers.conf
+
+# test-podman-rootless.sh logins to the user via SSH
+COPY ./script/podman/config/test-podman-rootless.sh /test-podman-rootless.sh
+RUN ssh-keygen -q -t rsa -f /root/.ssh/id_rsa -N '' && \
+    useradd -m -s /bin/bash rootless && \
+    mkdir -p -m 0700 /home/rootless/.ssh && \
+    cp -a /root/.ssh/id_rsa.pub /home/rootless/.ssh/authorized_keys && \
+    mkdir -p /home/rootless/.local/share /home/rootless/.local/share/stargz-store/store && \
+    chown -R rootless:rootless /home/rootless
+VOLUME /home/rootless/.local/share
+
+ENTRYPOINT ["/docker-entrypoint.sh", "/test-podman-rootless.sh"]
+CMD ["/bin/bash", "--login", "-i"]
 
 # Image which can be used for interactive demo environment
 FROM containerd-base AS demo
@@ -206,24 +246,21 @@ RUN apt-get update && apt-get install -y iptables && \
     curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin
 
 # Image which can be used as a node image for KinD (containerd with builtin snapshotter)
-FROM kindest/node:v1.22.2 AS kind-builtin-snapshotter
-# see https://medium.com/nttlabs/ubuntu-21-10-and-fedora-35-do-not-work-on-docker-20-10-9-1cd439d9921
-ADD https://github.com/AkihiroSuda/clone3-workaround/releases/download/v1.0.0/clone3-workaround.x86_64 /clone3-workaround
-RUN chmod 755 /clone3-workaround
+FROM kindest/node:v1.33.2 AS kind-builtin-snapshotter
 COPY --from=containerd-snapshotter-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/
 COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/
 COPY ./script/config/ /
-RUN /clone3-workaround apt-get update -y && /clone3-workaround apt-get install --no-install-recommends -y fuse
-ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
+RUN apt-get update -y && apt-get install --no-install-recommends -y fuse3
+ENTRYPOINT [ "/usr/local/bin/kind-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init" ]
 
 # Image for testing CRI-O with Stargz Store.
 # NOTE: This cannot be used for the node image of KinD.
-FROM ubuntu:20.04 AS crio-stargz-store
+FROM ubuntu:24.04 AS crio-stargz-store
 ARG CNI_PLUGINS_VERSION
 ARG CRIO_TEST_PAUSE_IMAGE_NAME
 ENV container docker
 RUN apt-get update -y && apt-get install --no-install-recommends -y \
-                         ca-certificates fuse libgpgme-dev libglib2.0-dev curl \
+                         ca-certificates fuse3 libgpgme-dev libglib2.0-dev curl \
                          iptables conntrack systemd systemd-sysv && \
     DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y tzdata && \
     # Make CNI plugins manipulate iptables instead of nftables
@@ -234,7 +271,10 @@ RUN apt-get update -y && apt-get install --no-install-recommends -y \
     curl -sSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin && \
     echo ${CRIO_TEST_PAUSE_IMAGE_NAME} > /pause_name && \
     mkdir -p /etc/sysconfig && \
-    echo CRIO_RUNTIME_OPTIONS=--pause-image=${CRIO_TEST_PAUSE_IMAGE_NAME} > /etc/sysconfig/crio
+    echo CRIO_RUNTIME_OPTIONS=--pause-image=${CRIO_TEST_PAUSE_IMAGE_NAME} > /etc/sysconfig/crio && \
+    # Necessary to pass CRI tests: https://github.com/kubernetes-sigs/cri-tools/pull/905
+    mkdir -p /etc/crio/crio.conf.d && \
+    printf '[crio.runtime]\nseccomp_use_default_when_empty = false\n' > /etc/crio/crio.conf.d/02-seccomp.conf
 
 COPY --from=stargz-store-dev /out/* /usr/local/bin/
 COPY --from=cri-o-dev /out/bin/* /usr/local/bin/
@@ -247,13 +287,10 @@ COPY ./script/config-cri-o/ /
 ENTRYPOINT [ "/usr/local/bin/entrypoint" ]
 
 # Image which can be used as a node image for KinD
-FROM kindest/node:v1.22.2
-# see https://medium.com/nttlabs/ubuntu-21-10-and-fedora-35-do-not-work-on-docker-20-10-9-1cd439d9921
-ADD https://github.com/AkihiroSuda/clone3-workaround/releases/download/v1.0.0/clone3-workaround.x86_64 /clone3-workaround
-RUN chmod 755 /clone3-workaround
+FROM kindest/node:v1.33.2
 COPY --from=containerd-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/
 COPY --from=snapshotter-dev /out/* /usr/local/bin/
 COPY ./script/config/ /
-RUN /clone3-workaround apt-get update -y && /clone3-workaround apt-get install --no-install-recommends -y fuse && \
+RUN apt-get update -y && apt-get install --no-install-recommends -y fuse3 && \
     systemctl enable stargz-snapshotter
-ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
+ENTRYPOINT [ "/usr/local/bin/kind-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init" ]

Makefile

@@ -21,13 +21,14 @@ PREFIX ?= $(CURDIR)/out/
 PKG=github.com/containerd/stargz-snapshotter
 VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
 REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
-GO_LD_FLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(GO_EXTRA_LDFLAGS)'
+GO_BUILD_LDFLAGS ?= -s -w
+GO_LD_FLAGS=-ldflags '$(GO_BUILD_LDFLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(GO_EXTRA_LDFLAGS)'
 
-CMD=containerd-stargz-grpc ctr-remote stargz-store
+CMD=containerd-stargz-grpc ctr-remote stargz-store stargz-fuse-manager
 
 CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))
 
-.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow vendor
+.PHONY: all build check install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow vendor
 
 all: build
 
@@ -44,6 +45,12 @@ ctr-remote: FORCE
 stargz-store: FORCE
 	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store
 
+stargz-store-helper: FORCE
+	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store/helper
+
+stargz-fuse-manager: FORCE
+	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-fuse-manager
+
 check:
 	@echo "$@"
 	@GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
@@ -51,9 +58,6 @@ check:
 	@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
 	@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
 
-install-check-tools:
-	@curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.42.1
-
 install:
 	@echo "$@"
 	@mkdir -p $(CMD_DESTDIR)/bin
@@ -74,10 +78,10 @@ validate-generated:
 	@./script/generated-files/generate.sh validate
 
 vendor:
-	@GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
 	@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
-	@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
 	@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
+	@GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
+	@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
 
 test:
 	@echo "$@"
@@ -110,6 +114,9 @@ test-cri-containerd:
 test-cri-o:
 	@./script/cri-o/test.sh
 
+test-podman:
+	@./script/podman/test.sh
+
 test-criauth:
 	@./script/criauth/test.sh
 
@@ -118,3 +125,13 @@ test-k3s:
 
 test-k3s-argo-workflow:
 	@./script/k3s-argo-workflow/run.sh
+
+test-ipfs:
+	@./script/ipfs/test.sh
+
+validate-vendor:
+	$(eval TMPDIR := $(shell mktemp -d))
+	@cp -R $(CURDIR) ${TMPDIR}
+	@(cd ${TMPDIR}/stargz-snapshotter && make vendor)
+	@diff -r -u -q $(CURDIR) ${TMPDIR}/stargz-snapshotter
+	@rm -rf ${TMPDIR}

README.md

@@ -47,6 +47,7 @@ Stargz Snapshotter is a **non-core** sub-project of containerd.
 
 - For more details about stargz snapshotter plugin and its configuration, refer to [Containerd Stargz Snapshotter Plugin Overview](/docs/overview.md).
 - For more details about setup lazy pulling of eStargz with containerd, CRI-O, Podman, systemd, etc., refer to [Install Stargz Snapshotter and Stargz Store](./docs/INSTALL.md).
+- For more details about integration status of eStargz with tools in commuinty, refer to [Integration of eStargz with other tools](./docs/integration.md)
 
 For using stargz snapshotter on kubernetes nodes, you need the following configuration to containerd as well as run stargz snapshotter daemon on the node.
 We assume that you are using containerd (> v1.4.2) as a CRI runtime.
@@ -61,6 +62,8 @@ version = 2
   [proxy_plugins.stargz]
     type = "snapshot"
     address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
 
 # Use stargz snapshotter through CRI
 [plugins."io.containerd.grpc.v1.cri".containerd]
@@ -68,18 +71,18 @@ version = 2
   disable_snapshot_annotations = false
 ```
 
-**Note that `disable_snapshot_annotations = false` is required since containerd > v1.4.2**
-
 You can try our [prebuilt](/Dockerfile) [KinD](https://github.com/kubernetes-sigs/kind) node image that contains the above configuration.
 
 ```console
-$ kind create cluster --name stargz-demo --image ghcr.io/stargz-containers/estargz-kind-node:0.7.0
+$ kind create cluster --name stargz-demo --image ghcr.io/containerd/stargz-snapshotter:0.12.1-kind
 ```
 
-> kind binary v0.11.x or newer is recommended for `estargz-kind-node:0.7.0`.
+:information_source: kind binary v0.16.x or newer is recommended for `ghcr.io/containerd/stargz-snapshotter:0.12.1-kind`.
+
+:information_source: You can get latest node images from [`ghcr.io/containerd/stargz-snapshotter:${VERSION}-kind`](https://github.com/orgs/containerd/packages/container/package/stargz-snapshotter) namespace.
 
 Then you can create eStargz pods on the cluster.
-In this example, we create a stargz-converted Node.js pod (`ghcr.io/stargz-containers/node:13.13.0-esgz`) as a demo.
+In this example, we create a stargz-converted Node.js pod (`ghcr.io/stargz-containers/node:17.8.0-esgz`) as a demo.
 
 ```yaml
 apiVersion: v1
@@ -89,7 +92,7 @@ metadata:
 spec:
   containers:
   - name: nodejs-stargz
-    image: ghcr.io/stargz-containers/node:13.13.0-esgz
+    image: ghcr.io/stargz-containers/node:17.8.0-esgz
     command: ["node"]
     args:
     - -e
@@ -102,10 +105,10 @@ spec:
     - containerPort: 80
 ```
 
-The following command lazily pulls `ghcr.io/stargz-containers/node:13.13.0-esgz` from Github Container Registry and creates the pod so the time to take for it is shorter than the original image `library/node:13.13`.
+The following command lazily pulls `ghcr.io/stargz-containers/node:17.8.0-esgz` from Github Container Registry and creates the pod so the time to take for it is shorter than the original image `library/node:13.13`.
 
 ```console
-$ kubectl --context kind-stargz-demo apply -f stargz-pod.yaml && kubectl get po nodejs -w
+$ kubectl --context kind-stargz-demo apply -f stargz-pod.yaml && kubectl --context kind-stargz-demo get po nodejs -w
 $ kubectl --context kind-stargz-demo port-forward nodejs 8080:80 &
 $ curl 127.0.0.1:8080
 Hello World!
@@ -126,31 +129,61 @@ This section describes some of them.
 
 You can try our pre-converted eStargz images on ghcr.io listed in [Trying pre-converted images](/docs/pre-converted-images.md).
 
-### Registry-side conversion with `estargz.kontain.me`
+### Building eStargz images using BuildKit
 
-You can convert arbitrary images into eStargz on the registry-side, using [`estargz.kontain.me`](https://estargz.kontain.me).
-`estargz.kontain.me/[image]` serves eStargz-converted version of an arbitrary public image.
+BuildKit supports building eStargz image since v0.10.
 
-For example, the following Kubernetes manifest performs lazy pulling of eStargz-formatted version of `docker.io/library/nginx:1.21.1` that is converted by `estargz.kontain.me`.
+You can try it using [Docker Buildx](https://docs.docker.com/buildx/working-with-buildx/).
+The following command builds an eStargz image and push it to `ghcr.io/ktock/hello:esgz`.
+Flags `oci-mediatypes=true,compression=estargz` enable to build eStargz.
 
-```yaml
-apiVersion: v1
-kind: Pod
-metadata:
-  name: nginx
-spec:
-  containers:
-  - name: nginx
-    image: estargz.kontain.me/docker.io/library/nginx:1.21.1
-    ports:
-    - containerPort: 80
+```
+$ docker buildx build -t ghcr.io/ktock/hello:esgz \
+    -o type=registry,oci-mediatypes=true,compression=estargz,force-compression=true \
+    /tmp/buildctx/
 ```
 
-> WARNING: Before trying this method, read [caveats from kontain.me](https://github.com/imjasonh/kontain.me#caveats). If you rely on it in production, you should copy the image to your own registry or build eStargz by your own using `ctr-remote` as described in the following.
+> NOTE1: `force-compression=true` isn't needed if the base image is already eStargz.
 
-### Creating eStargz using `ctr-remote`
+> NOTE2: Docker still does not support lazy pulling of eStargz.
 
-In this section, we introduce [`ctr-remote`](/docs/ctr-remote.md) command for converting images into eStargz with optimization for reading files.
+eStargz-enabled BuildKit (v0.10) will be [included to Docker v22.XX](https://github.com/moby/moby/blob/v22.06.0-beta.0/vendor.mod#L51) however you can build eStargz images with the prior version using Buildx [driver](https://github.com/docker/buildx/blob/master/docs/reference/buildx_create.md#-set-the-builder-driver-to-use---driver) feature.
+You can enable the specific version of BuildKit using [`docker buildx create`](https://docs.docker.com/engine/reference/commandline/buildx_create/) (this example specifies `v0.10.3`).
+
+```
+$ docker buildx create --use --name v0.10.3 --driver docker-container --driver-opt image=moby/buildkit:v0.10.3
+$ docker buildx inspect --bootstrap v0.10.3
+```
+
+### Building eStargz images using Kaniko
+
+[Kaniko](https://github.com/GoogleContainerTools/kaniko) is an image builder runnable in containers and Kubernetes.
+Since v1.5.0, it experimentally supports building eStargz.
+`GGCR_EXPERIMENT_ESTARGZ=1` is needed.
+
+```console
+$ docker run --rm -e GGCR_EXPERIMENT_ESTARGZ=1 \
+    -v /tmp/buildctx:/workspace -v ~/.docker/config.json:/kaniko/.docker/config.json:ro \
+    gcr.io/kaniko-project/executor:v1.8.1 --destination ghcr.io/ktock/hello:esgz
+```
+
+### Building eStargz images using nerdctl
+
+[nerdctl](https://github.com/containerd/nerdctl), Docker-compatible CLI of containerd, supports building eStargz images.
+
+```console
+$ nerdctl build -t ghcr.io/ktock/hello:1 /tmp/buildctx
+$ nerdctl image convert --estargz --oci ghcr.io/ktock/hello:1 ghcr.io/ktock/hello:esgz
+$ nerdctl push ghcr.io/ktock/hello:esgz
+```
+
+> NOTE: `--estargz` should be specified in conjunction with `--oci`
+
+Please refer to nerdctl document for details for further information (e.g. lazy pulling): https://github.com/containerd/nerdctl/blob/master/docs/stargz.md
+
+### Creating eStargz images using `ctr-remote`
+
+[`ctr-remote`](/docs/ctr-remote.md) allows converting an image into eStargz with optimizing it.
 As shown in the above benchmarking result, on-demand lazy pulling improves the performance of pull but causes runtime performance penalty because reading files induce remotely downloading contents.
 For solving this, `ctr-remote` has *workload-based* optimization for images.
 

analyzer/analyzer.go

@@ -17,34 +17,34 @@
 package analyzer
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/signal"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
 
 	"github.com/containerd/console"
-	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/cio"
-	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/cmd/ctr/commands/tasks"
-	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/mount"
-	"github.com/containerd/containerd/oci"
-	"github.com/containerd/containerd/platforms"
-	"github.com/containerd/containerd/snapshots"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands/tasks"
+	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/containerd/v2/pkg/cio"
+	"github.com/containerd/containerd/v2/pkg/oci"
+	"github.com/containerd/errdefs"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/analyzer/fanotify"
 	"github.com/containerd/stargz-snapshotter/analyzer/recorder"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/pkg/errors"
 	"github.com/rs/xid"
 )
 
@@ -62,7 +62,7 @@ func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ..
 		return "", fmt.Errorf("wait-on-signal option cannot be used with terminal option")
 	}
 
-	target, err := ioutil.TempDir("", "target")
+	target, err := os.MkdirTemp("", "target")
 	if err != nil {
 		return "", err
 	}
@@ -99,7 +99,7 @@ func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ..
 	// Spawn a fanotifier process in a new mount namespace and setup recorder.
 	fanotifier, err := fanotify.SpawnFanotifier("/proc/self/exe")
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to spawn fanotifier")
+		return "", fmt.Errorf("failed to spawn fanotifier: %w", err)
 	}
 	defer func() {
 		if err := fanotifier.Close(); err != nil {
@@ -163,6 +163,7 @@ func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ..
 	defer container.Delete(ctx, containerd.WithSnapshotCleanup)
 	var ioCreator cio.Creator
 	var con console.Console
+	waitLine := newLineWaiter(aOpts.waitLineOut)
 	stdinC := newLazyReadCloser(os.Stdin)
 	if aOpts.terminal {
 		if !aOpts.stdin {
@@ -174,11 +175,11 @@ func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ..
 			return "", err
 		}
 		// On terminal mode, the "stderr" field is unused.
-		ioCreator = cio.NewCreator(cio.WithStreams(con, con, nil), cio.WithTerminal)
+		ioCreator = cio.NewCreator(cio.WithStreams(con, waitLine.registerWriter(con), nil), cio.WithTerminal)
 	} else if aOpts.stdin {
-		ioCreator = cio.NewCreator(cio.WithStreams(stdinC, os.Stdout, os.Stderr))
+		ioCreator = cio.NewCreator(cio.WithStreams(stdinC, waitLine.registerWriter(os.Stdout), os.Stderr))
 	} else {
-		ioCreator = cio.NewCreator(cio.WithStreams(nil, os.Stdout, os.Stderr))
+		ioCreator = cio.NewCreator(cio.WithStreams(nil, waitLine.registerWriter(os.Stdout), os.Stderr))
 	}
 	task, err := container.NewTask(ctx, ioCreator)
 	if err != nil {
@@ -195,11 +196,15 @@ func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ..
 	}
 	defer rc.Close()
 	if err := fanotifier.Start(); err != nil {
-		return "", errors.Wrapf(err, "failed to start fanotifier")
+		return "", fmt.Errorf("failed to start fanotifier: %w", err)
 	}
 	var fanotifierClosed bool
 	var fanotifierClosedMu sync.Mutex
 	go func() {
+		var successCount int
+		defer func() {
+			log.G(ctx).Debugf("success record %d path", successCount)
+		}()
 		for {
 			path, err := fanotifier.GetPath()
 			if err != nil {
@@ -217,8 +222,12 @@ func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ..
 			if err := rc.Record(path); err != nil {
 				log.G(ctx).WithError(err).Debugf("failed to record %q", path)
 			}
+			successCount++
 		}
 	}()
+	if err := task.Start(ctx); err != nil {
+		return "", err
+	}
 	if aOpts.terminal {
 		if err := tasks.HandleConsoleResize(ctx, task, con); err != nil {
 			log.G(ctx).WithError(err).Error("failed to resize console")
@@ -227,9 +236,6 @@ func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ..
 		sigc := commands.ForwardAllSignals(ctx, task)
 		defer commands.StopCatch(sigc)
 	}
-	if err := task.Start(ctx); err != nil {
-		return "", err
-	}
 
 	// Wait until the task exit
 	var status containerd.ExitStatus
@@ -245,7 +251,7 @@ func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ..
 			aOpts.period = defaultPeriod
 		}
 		log.G(ctx).Infof("waiting for %v ...", aOpts.period)
-		status, killOk, err = waitOnTimeout(ctx, container, task, aOpts.period)
+		status, killOk, err = waitOnTimeout(ctx, container, task, aOpts.period, waitLine)
 		if err != nil {
 			return "", err
 		}
@@ -288,7 +294,7 @@ func mountImage(ctx context.Context, ss snapshots.Snapshotter, image containerd.
 		if err := ss.Remove(ctx, mountpoint); err != nil && !errdefs.IsNotFound(err) {
 			log.G(ctx).WithError(err).Warnf("failed to cleanup snapshot after mount error")
 		}
-		return nil, errors.Wrapf(err, "failed to mount rootfs at %q", mountpoint)
+		return nil, fmt.Errorf("failed to mount rootfs at %q: %w", mountpoint, err)
 	}
 	return func() {
 		if err := mount.UnmountAll(mountpoint, 0); err != nil {
@@ -322,7 +328,7 @@ func waitOnSignal(ctx context.Context, container containerd.Container, task cont
 	}
 }
 
-func waitOnTimeout(ctx context.Context, container containerd.Container, task containerd.Task, period time.Duration) (containerd.ExitStatus, bool, error) {
+func waitOnTimeout(ctx context.Context, container containerd.Container, task containerd.Task, period time.Duration, line *lineWaiter) (containerd.ExitStatus, bool, error) {
 	statusC, err := task.Wait(ctx)
 	if err != nil {
 		return containerd.ExitStatus{}, false, err
@@ -330,15 +336,17 @@ func waitOnTimeout(ctx context.Context, container containerd.Container, task con
 	select {
 	case status := <-statusC:
 		return status, true, nil
+	case l := <-line.waitCh:
+		log.G(ctx).Infof("Waiting line detected %q; killing task", l)
 	case <-time.After(period):
 		log.G(ctx).Warnf("killing task. the time period to monitor access log (%s) has timed out", period.String())
-		status, err := killTask(ctx, container, task, statusC)
-		if err != nil {
-			log.G(ctx).WithError(err).Warnf("failed to kill container")
-			return containerd.ExitStatus{}, false, nil
-		}
-		return status, true, nil
 	}
+	status, err := killTask(ctx, container, task, statusC)
+	if err != nil {
+		log.G(ctx).WithError(err).Warnf("failed to kill container")
+		return containerd.ExitStatus{}, false, nil
+	}
+	return status, true, nil
 }
 
 func killTask(ctx context.Context, container containerd.Container, task containerd.Task, statusC <-chan containerd.ExitStatus) (containerd.ExitStatus, error) {
@@ -347,7 +355,7 @@ func killTask(ctx context.Context, container containerd.Container, task containe
 		return containerd.ExitStatus{}, err
 	}
 	if err := task.Kill(ctx, sig, containerd.WithKillAll); err != nil && !errdefs.IsNotFound(err) {
-		return containerd.ExitStatus{}, errors.Wrapf(err, "forward SIGKILL")
+		return containerd.ExitStatus{}, fmt.Errorf("forward SIGKILL: %w", err)
 	}
 	select {
 	case status := <-statusC:
@@ -396,3 +404,37 @@ func (s *lazyReadCloser) Read(p []byte) (int, error) {
 	}
 	return n, err
 }
+
+func newLineWaiter(s string) *lineWaiter {
+	return &lineWaiter{
+		waitCh:   make(chan string),
+		waitLine: s,
+	}
+}
+
+type lineWaiter struct {
+	waitCh   chan string
+	waitLine string
+}
+
+func (lw *lineWaiter) registerWriter(w io.Writer) io.Writer {
+	if lw.waitLine == "" {
+		return w
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		scanner := bufio.NewScanner(pr)
+		for scanner.Scan() {
+			if strings.Contains(scanner.Text(), lw.waitLine) {
+				lw.waitCh <- lw.waitLine
+			}
+		}
+		if _, err := io.Copy(io.Discard, pr); err != nil {
+			pr.CloseWithError(err)
+			return
+		}
+	}()
+
+	return io.MultiWriter(w, pw)
+}

analyzer/fanotify/conn/conn.go

@@ -24,8 +24,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/pkg/errors"
 )
 
 const (
@@ -74,11 +72,11 @@ func (nc *Client) GetPath() (string, error) {
 	}
 	fd, err := strconv.ParseInt(mes[len(mesFdPrefix):], 10, 32)
 	if err != nil {
-		return "", errors.Wrapf(err, "invalid fd %q", mes)
+		return "", fmt.Errorf("invalid fd %q: %w", mes, err)
 	}
 	path, err := os.Readlink(fmt.Sprintf("/proc/%d/fd/%d", nc.servicePid, fd))
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to get link from fd %q", mes)
+		return "", fmt.Errorf("failed to get link from fd %q: %w", mes, err)
 	}
 	return path, writeMessage(nc.w, mesAck)
 }

analyzer/fanotify/fanotify.go

@@ -17,6 +17,7 @@
 package fanotify
 
 import (
+	"errors"
 	"fmt"
 	"os/exec"
 	"sync"
@@ -24,7 +25,6 @@ import (
 	"time"
 
 	"github.com/containerd/stargz-snapshotter/analyzer/fanotify/conn"
-	"github.com/hashicorp/go-multierror"
 )
 
 // Fanotifier monitors "/" mountpoint of a new mount namespace and notifies all
@@ -59,14 +59,15 @@ func SpawnFanotifier(fanotifierBin string) (*Fanotifier, error) {
 
 		// Connect to the spawned fanotifier over stdio
 		conn: conn.NewClient(notifyR, notifyW, cmd.Process.Pid, 5*time.Second),
-		closeFunc: func() (allErr error) {
+		closeFunc: func() error {
+			var errs []error
 			if err := notifyR.Close(); err != nil {
-				allErr = multierror.Append(allErr, err)
+				errs = append(errs, err)
 			}
 			if err := notifyW.Close(); err != nil {
-				allErr = multierror.Append(allErr, err)
+				errs = append(errs, err)
 			}
-			return
+			return errors.Join(errs...)
 		},
 	}, nil
 }

analyzer/fanotify/service/service.go

@@ -25,7 +25,6 @@ import (
 	"time"
 
 	"github.com/containerd/stargz-snapshotter/analyzer/fanotify/conn"
-	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 )
 
@@ -36,12 +35,12 @@ func Serve(target string, r io.Reader, w io.Writer) error {
 
 	fd, err := unix.FanotifyInit(unix.FAN_CLASS_NOTIF, unix.O_RDONLY)
 	if err != nil {
-		return errors.Wrapf(err, "fanotify_init")
+		return fmt.Errorf("fanotify_init: %w", err)
 	}
 
 	// This blocks until the client tells us to start monitoring the target mountpoint.
 	if err := sConn.WaitStart(); err != nil {
-		return errors.Wrapf(err, "waiting for start inst")
+		return fmt.Errorf("waiting for start inst: %w", err)
 	}
 
 	// Start monitoring the target mountpoint.
@@ -51,12 +50,12 @@ func Serve(target string, r io.Reader, w io.Writer) error {
 		unix.AT_FDCWD,
 		target,
 	); err != nil {
-		return errors.Wrapf(err, "fanotify_mark")
+		return fmt.Errorf("fanotify_mark: %w", err)
 	}
 
 	// Notify "started" state to the client.
 	if err := sConn.SendStarted(); err != nil {
-		return errors.Wrapf(err, "failed to send started message")
+		return fmt.Errorf("failed to send started message: %w", err)
 	}
 
 	nr := bufio.NewReader(os.NewFile(uintptr(fd), ""))
@@ -66,10 +65,10 @@ func Serve(target string, r io.Reader, w io.Writer) error {
 			if err == io.EOF {
 				break
 			}
-			return errors.Wrapf(err, "read fanotify fd")
+			return fmt.Errorf("read fanotify fd: %w", err)
 		}
 		if event.Vers != unix.FANOTIFY_METADATA_VERSION {
-			return fmt.Errorf("Fanotify version mismatch %d(got) != %d(want)",
+			return fmt.Errorf("fanotify version mismatch %d(got) != %d(want)",
 				event.Vers, unix.FANOTIFY_METADATA_VERSION)
 		}
 		if event.Fd < 0 {
@@ -85,10 +84,10 @@ func Serve(target string, r io.Reader, w io.Writer) error {
 		// descriptor and let the client resolve the path of this file using /proc of
 		// this process.
 		if err := sConn.SendFd(int(event.Fd)); err != nil {
-			return errors.Wrapf(err, "failed to send fd %d to client", fd)
+			return fmt.Errorf("failed to send fd %d to client: %w", fd, err)
 		}
 		if err := unix.Close(int(event.Fd)); err != nil {
-			return errors.Wrapf(err, "Close(fd)")
+			return fmt.Errorf("Close(fd): %w", err)
 		}
 
 		continue

analyzer/option.go

@@ -19,8 +19,8 @@ package analyzer
 import (
 	"time"
 
-	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/oci"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/oci"
 )
 
 type analyzerOpts struct {
@@ -30,6 +30,7 @@ type analyzerOpts struct {
 	specOpts     SpecOpts
 	terminal     bool
 	stdin        bool
+	waitLineOut  string
 }
 
 // Option is runtime configuration of analyzer container
@@ -79,3 +80,11 @@ func WithSnapshotter(snapshotter string) Option {
 		opts.snapshotter = snapshotter
 	}
 }
+
+// WithWaitLineOut specifies a substring of a stdout line to be waited.
+// When this line is detected, the container will be killed.
+func WithWaitLineOut(s string) Option {
+	return func(opts *analyzerOpts) {
+		opts.waitLineOut = s
+	}
+}

analyzer/recorder/recorder.go

@@ -26,18 +26,17 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/containerd/containerd/archive/compression"
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/images/converter/uncompress"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/images/converter/uncompress"
+	"github.com/containerd/containerd/v2/pkg/archive/compression"
+	"github.com/containerd/errdefs"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/recorder"
 	"github.com/containerd/stargz-snapshotter/util/containerdutil"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 	"github.com/rs/xid"
 	"golang.org/x/sync/errgroup"
 )
@@ -88,14 +87,14 @@ func imageRecorderFromManifest(ctx context.Context, cs content.Store, manifestDe
 		log.G(ctx).Infof("analyzing blob %q", desc.Digest)
 		readerAt, err := cs.ReaderAt(ctx, desc)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to get reader blob %v", desc.Digest)
+			return nil, fmt.Errorf("failed to get reader blob %v: %w", desc.Digest, err)
 		}
 		defer readerAt.Close()
 		r := io.Reader(io.NewSectionReader(readerAt, 0, desc.Size))
 		if !uncompress.IsUncompressedType(desc.MediaType) {
 			r, err = compression.DecompressStream(r)
 			if err != nil {
-				return nil, errors.Wrapf(err, "cannot decompress layer %v", desc.Digest)
+				return nil, fmt.Errorf("cannot decompress layer %v: %w", desc.Digest, err)
 			}
 		}
 		eg.Go(func() error {
@@ -120,7 +119,7 @@ func imageRecorderFromManifest(ctx context.Context, cs content.Store, manifestDe
 	recordW, err := content.OpenWriter(ctx, cs,
 		content.WithRef(fmt.Sprintf("recorder-%v", xid.New().String())))
 	if err != nil {
-		return nil, errors.Wrapf(err, "faeild to open writer for recorder")
+		return nil, fmt.Errorf("failed to open writer for recorder: %w", err)
 	}
 	return &ImageRecorder{
 		r:              recorder.New(recordW),
@@ -152,7 +151,7 @@ func (r *ImageRecorder) Record(name string) error {
 		}
 		whDir := cleanEntryName(path.Join(path.Dir("/"+name), whiteoutOpaqueDir))
 		if _, ok := r.index[i][whDir]; ok {
-			return fmt.Errorf("Parent dir of %q is a deleted directory", name)
+			return fmt.Errorf("parent dir of %q is a deleted directory", name)
 		}
 	}
 	if index < 0 {

analyzer/recorder/recorder_test.go

@@ -22,14 +22,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"testing"
 
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/content/local"
-	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/plugins/content/local"
+	"github.com/containerd/errdefs"
 	"github.com/containerd/stargz-snapshotter/recorder"
 	"github.com/containerd/stargz-snapshotter/util/testutil"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -202,7 +201,7 @@ func TestNodeIndex(t *testing.T) {
 		},
 	}
 
-	tempDir, err := ioutil.TempDir("", "test-recorder")
+	tempDir, err := os.MkdirTemp("", "test-recorder")
 	if err != nil {
 		t.Fatalf("failed to prepare content store dir: %v", err)
 	}

cache/cache.go

@@ -18,17 +18,16 @@ package cache
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
 
-	"github.com/containerd/stargz-snapshotter/util/lrucache"
+	"github.com/containerd/stargz-snapshotter/util/cacheutil"
 	"github.com/containerd/stargz-snapshotter/util/namedmutex"
-	"github.com/hashicorp/go-multierror"
-	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -51,11 +50,11 @@ type DirectoryCacheConfig struct {
 
 	// DataCache is an on-memory cache of the data.
 	// OnEvicted will be overridden and replaced for internal use.
-	DataCache *lrucache.Cache
+	DataCache *cacheutil.LRUCache
 
 	// FdCache is a cache for opened file descriptors.
 	// OnEvicted will be overridden and replaced for internal use.
-	FdCache *lrucache.Cache
+	FdCache *cacheutil.LRUCache
 
 	// BufPool will be used for pooling bytes.Buffer.
 	BufPool *sync.Pool
@@ -63,6 +62,9 @@ type DirectoryCacheConfig struct {
 	// Direct forcefully enables direct mode for all operation in cache.
 	// Thus operation won't use on-memory caches.
 	Direct bool
+
+	// FadvDontNeed forcefully clean fscache pagecache for saving memory.
+	FadvDontNeed bool
 }
 
 // TODO: contents validation.
@@ -84,6 +86,9 @@ type BlobCache interface {
 type Reader interface {
 	io.ReaderAt
 	Close() error
+
+	// If a blob is backed by a file, it should return *os.File so that it can be used for FUSE passthrough
+	GetReaderAt() io.ReaderAt
 }
 
 // Writer enables the client to cache byte data. Commit() must be
@@ -96,7 +101,8 @@ type Writer interface {
 }
 
 type cacheOpt struct {
-	direct bool
+	direct      bool
+	passThrough bool
 }
 
 type Option func(o *cacheOpt) *cacheOpt
@@ -112,6 +118,15 @@ func Direct() Option {
 	}
 }
 
+// PassThrough option indicates whether to enable FUSE passthrough mode
+// to improve local file read performance.
+func PassThrough() Option {
+	return func(o *cacheOpt) *cacheOpt {
+		o.passThrough = true
+		return o
+	}
+}
+
 func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache, error) {
 	if !filepath.IsAbs(directory) {
 		return nil, fmt.Errorf("dir cache path must be an absolute path; got %q", directory)
@@ -130,7 +145,7 @@ func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache
 		if maxEntry == 0 {
 			maxEntry = defaultMaxLRUCacheEntry
 		}
-		dataCache = lrucache.New(maxEntry)
+		dataCache = cacheutil.NewLRUCache(maxEntry)
 		dataCache.OnEvicted = func(key string, value interface{}) {
 			value.(*bytes.Buffer).Reset()
 			bufPool.Put(value)
@@ -142,7 +157,7 @@ func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache
 		if maxEntry == 0 {
 			maxEntry = defaultMaxCacheFds
 		}
-		fdCache = lrucache.New(maxEntry)
+		fdCache = cacheutil.NewLRUCache(maxEntry)
 		fdCache.OnEvicted = func(key string, value interface{}) {
 			value.(*os.File).Close()
 		}
@@ -162,6 +177,7 @@ func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache
 		wipDirectory: wipdir,
 		bufPool:      bufPool,
 		direct:       config.Direct,
+		fadvDontNeed: config.FadvDontNeed,
 	}
 	dc.syncAdd = config.SyncAdd
 	return dc, nil
@@ -169,16 +185,17 @@ func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache
 
 // directoryCache is a cache implementation which backend is a directory.
 type directoryCache struct {
-	cache        *lrucache.Cache
-	fileCache    *lrucache.Cache
+	cache        *cacheutil.LRUCache
+	fileCache    *cacheutil.LRUCache
 	wipDirectory string
 	directory    string
 	wipLock      *namedmutex.NamedMutex
 
 	bufPool *sync.Pool
 
-	syncAdd bool
-	direct  bool
+	syncAdd      bool
+	direct       bool
+	fadvDontNeed bool
 
 	closed   bool
 	closedMu sync.Mutex
@@ -223,7 +240,7 @@ func (dc *directoryCache) Get(key string, opts ...Option) (Reader, error) {
 	//       or simply report the cache miss?
 	file, err := os.Open(dc.cachePath(key))
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to open blob file for %q", key)
+		return nil, fmt.Errorf("failed to open blob file for %q: %w", key, err)
 	}
 
 	// If "direct" option is specified, do not cache the file on memory.
@@ -231,8 +248,22 @@ func (dc *directoryCache) Get(key string, opts ...Option) (Reader, error) {
 	// that won't be accessed immediately.
 	if dc.direct || opt.direct {
 		return &reader{
-			ReaderAt:  file,
-			closeFunc: func() error { return file.Close() },
+			ReaderAt: file,
+			closeFunc: func() error {
+				if dc.fadvDontNeed {
+					if err := dropFilePageCache(file); err != nil {
+						fmt.Printf("Warning: failed to drop page cache: %v\n", err)
+					}
+				}
+
+				// In passthough model, close will be toke over by go-fuse
+				// If "passThrough" option is specified, "direct" option also will
+				// be specified, so adding this branch here is enough
+				if opt.passThrough {
+					return nil
+				}
+				return file.Close()
+			},
 		}, nil
 	}
 
@@ -275,13 +306,20 @@ func (dc *directoryCache) Add(key string, opts ...Option) (Writer, error) {
 			// Commit the cache contents
 			c := dc.cachePath(key)
 			if err := os.MkdirAll(filepath.Dir(c), os.ModePerm); err != nil {
-				var allErr error
+				var errs []error
 				if err := os.Remove(wip.Name()); err != nil {
-					allErr = multierror.Append(allErr, err)
+					errs = append(errs, err)
 				}
-				return multierror.Append(allErr,
-					errors.Wrapf(err, "failed to create cache directory %q", c))
+				errs = append(errs, fmt.Errorf("failed to create cache directory %q: %w", c, err))
+				return errors.Join(errs...)
 			}
+
+			if dc.fadvDontNeed {
+				if err := dropFilePageCache(wip); err != nil {
+					fmt.Printf("Warning: failed to drop page cache: %v\n", err)
+				}
+			}
+
 			return os.Rename(wip.Name(), c)
 		},
 		abortFunc: func() error {
@@ -366,7 +404,7 @@ func (dc *directoryCache) cachePath(key string) string {
 }
 
 func (dc *directoryCache) wipFile(key string) (*os.File, error) {
-	return ioutil.TempFile(dc.wipDirectory, key+"-*")
+	return os.CreateTemp(dc.wipDirectory, key+"-*")
 }
 
 func NewMemoryCache() BlobCache {
@@ -386,7 +424,7 @@ func (mc *MemoryCache) Get(key string, opts ...Option) (Reader, error) {
 	defer mc.mu.Unlock()
 	b, ok := mc.Membuf[key]
 	if !ok {
-		return nil, fmt.Errorf("Missed cache: %q", key)
+		return nil, fmt.Errorf("missed cache: %q", key)
 	}
 	return &reader{bytes.NewReader(b.Bytes()), func() error { return nil }}, nil
 }
@@ -416,6 +454,10 @@ type reader struct {
 
 func (r *reader) Close() error { return r.closeFunc() }
 
+func (r *reader) GetReaderAt() io.ReaderAt {
+	return r.ReaderAt
+}
+
 type writer struct {
 	io.WriteCloser
 	commitFunc func() error
@@ -440,3 +482,16 @@ func (w *writeCloser) Close() error { return w.closeFunc() }
 func nopWriteCloser(w io.Writer) io.WriteCloser {
 	return &writeCloser{w, func() error { return nil }}
 }
+
+func dropFilePageCache(file *os.File) error {
+	if file == nil {
+		return nil
+	}
+
+	fd := file.Fd()
+	err := unix.Fadvise(int(fd), 0, 0, unix.FADV_DONTNEED)
+	if err != nil {
+		return fmt.Errorf("posix_fadvise failed, ret=%d", err)
+	}
+	return nil
+}

cache/cache_test.go

@@ -26,7 +26,6 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"testing"
 )
@@ -39,7 +38,7 @@ func TestDirectoryCache(t *testing.T) {
 
 	// with enough memory cache
 	newCache := func() (BlobCache, cleanFunc) {
-		tmp, err := ioutil.TempDir("", "testcache")
+		tmp, err := os.MkdirTemp("", "testcache")
 		if err != nil {
 			t.Fatalf("failed to make tempdir: %v", err)
 		}
@@ -56,7 +55,7 @@ func TestDirectoryCache(t *testing.T) {
 
 	// with smaller memory cache
 	newCache = func() (BlobCache, cleanFunc) {
-		tmp, err := ioutil.TempDir("", "testcache")
+		tmp, err := os.MkdirTemp("", "testcache")
 		if err != nil {
 			t.Fatalf("failed to make tempdir: %v", err)
 		}

cmd/containerd-stargz-grpc/db/db.go

@@ -23,39 +23,41 @@ import (
 	"sort"
 
 	"github.com/containerd/stargz-snapshotter/metadata"
-	"github.com/pkg/errors"
 	bolt "go.etcd.io/bbolt"
 )
 
 // Metadata package stores filesystem metadata in the following schema.
 //
 // - filesystems
-//   - *filesystem id*                  : bucket for each filesystem keyed by a unique string.
+//   - *filesystem id*                    : bucket for each filesystem keyed by a unique string.
 //     - nodes
-//       - *node id*                    : bucket for each node keyed by a uniqe uint64.
-//         - size : <varint>            : size of the regular node.
-//         - modtime : <varint>         : modification time of the node.
-//         - linkName : <string>        : link target of symlink
-//         - mode : <uvarint>           : permission and mode bits (os.FileMode).
-//         - uid : <varint>             : uid of the owner.
-//         - gid : <varint>             : gid of the owner.
-//         - devMajor : <varint>        : the major device number for device
-//         - devMinor : <varint>        : the minor device number for device
-//         - xattrKey : <string>        : key of the first extended attribute.
-//         - xattrValue : <string>      : value of the first extended attribute
-//         - xattrsExtra                : 2nd and the following extended attribute.
-//           - *key* : <string>         : map of key to value string
-//         - numLink : <varint>         : the number of links pointing to this node.
+//       - *node id*                      : bucket for each node keyed by a uniqe uint64.
+//         - size : <varint>              : size of the regular node.
+//         - modtime : <varint>           : modification time of the node.
+//         - linkName : <string>          : link target of symlink
+//         - mode : <uvarint>             : permission and mode bits (os.FileMode).
+//         - uid : <varint>               : uid of the owner.
+//         - gid : <varint>               : gid of the owner.
+//         - devMajor : <varint>          : the major device number for device
+//         - devMinor : <varint>          : the minor device number for device
+//         - xattrKey : <string>          : key of the first extended attribute.
+//         - xattrValue : <string>        : value of the first extended attribute
+//         - xattrsExtra                  : 2nd and the following extended attribute.
+//           - *key* : <string>           : map of key to value string
+//         - numLink : <varint>           : the number of links pointing to this node.
 //     - metadata
-//       - *node id*                    : bucket for each node keyed by a uniqe uint64.
-//         - childName : <string>       : base name of the first child
-//         - childID   : <node id>      : id of the first child
-//         - childrenExtra              : 2nd and following child nodes of directory.
-//           - *basename* : <node id>   : map of basename string to the child node id
-//         - chunk : <encoded>          : information of the first chunkn
-//         - chunksExtra                : 2nd and following chunks (this is rarely used so we can avoid the cost of creating the bucket)
-//           - *offset* : <encoded>     : keyed by gzip header offset (varint) in the estargz file to the chunk.
-//         - nextOffset : <varint>      : the offset of the next node with a non-zero offset.
+//       - *node id*                      : bucket for each node keyed by a uniqe uint64.
+//         - childName : <string>         : base name of the first child
+//         - childID   : <node id>        : id of the first child
+//         - childrenExtra                : 2nd and following child nodes of directory.
+//           - *basename* : <node id>     : map of basename string to the child node id
+//         - chunk : <encoded>            : information of the first chunkn
+//         - chunksExtra                  : 2nd and following chunks (this is rarely used so we can avoid the cost of creating the bucket)
+//           - *chunk offset* : <encoded> : keyed by chunk offset (varint) in the estargz file to the chunk.
+//         - nextOffset : <varint>        : the offset of the next node with a non-zero offset.
+//     - stream
+//       - *offset*                       : bucket for each chunk stream that have multiple inner chunks.
+//         - *innerOffset* : node id      : node id that has the contents at the keyed innerOffset.
 
 var (
 	bucketKeyFilesystems = []byte("filesystems")
@@ -81,6 +83,8 @@ var (
 	bucketKeyChunk         = []byte("chunk")
 	bucketKeyChunksExtra   = []byte("chunksExtra")
 	bucketKeyNextOffset    = []byte("nextOffset")
+
+	bucketKeyStream = []byte("stream")
 )
 
 type childEntry struct {
@@ -93,6 +97,7 @@ type chunkEntry struct {
 	chunkOffset int64
 	chunkSize   int64
 	chunkDigest string
+	innerOffset int64 // -1 indicates that no following chunks in the stream.
 }
 
 type metadataEntry struct {
@@ -133,6 +138,22 @@ func getMetadata(tx *bolt.Tx, fsID string) (*bolt.Bucket, error) {
 	return md, nil
 }
 
+func getStream(tx *bolt.Tx, fsID string) (*bolt.Bucket, error) {
+	filesystems := tx.Bucket(bucketKeyFilesystems)
+	if filesystems == nil {
+		return nil, fmt.Errorf("fs %q not found: no fs is registered", fsID)
+	}
+	lbkt := filesystems.Bucket([]byte(fsID))
+	if lbkt == nil {
+		return nil, fmt.Errorf("fs bucket for %q not found", fsID)
+	}
+	st := lbkt.Bucket(bucketKeyStream)
+	if st == nil {
+		return nil, fmt.Errorf("stream bucket for fs %q not found", fsID)
+	}
+	return st, nil
+}
+
 func getNodeBucketByID(nodes *bolt.Bucket, id uint32) (*bolt.Bucket, error) {
 	b := nodes.Bucket(encodeID(id))
 	if b == nil {
@@ -226,7 +247,7 @@ func writeAttr(b *bolt.Bucket, attr *metadata.Attr) error {
 				}
 			}
 			if err := xbkt.Put([]byte(k), v); err != nil {
-				return errors.Wrapf(err, "failed to set xattr %q=%q", k, string(v))
+				return fmt.Errorf("failed to set xattr %q=%q: %w", k, string(v), err)
 			}
 		}
 	}
@@ -320,6 +341,60 @@ func readChunks(b *bolt.Bucket, size int64) (chunks []chunkEntry, err error) {
 	return
 }
 
+type chunkEntryWithID struct {
+	chunkEntry
+	id uint32
+}
+
+func readInnerChunks(tx *bolt.Tx, fsID string, off int64) (chunks []chunkEntryWithID, err error) {
+	sb, err := getStream(tx, fsID)
+	if err != nil {
+		return nil, err
+	}
+	offEncoded, err := encodeInt(off)
+	if err != nil {
+		return nil, err
+	}
+	ob := sb.Bucket(offEncoded)
+	if ob == nil {
+		return nil, fmt.Errorf("inner chunk bucket for %d not found", off)
+	}
+	nodes, err := getNodes(tx, fsID)
+	if err != nil {
+		return nil, fmt.Errorf("nodes bucket of %q not found: %w", fsID, err)
+	}
+	metadataEntries, err := getMetadata(tx, fsID)
+	if err != nil {
+		return nil, fmt.Errorf("metadata bucket of %q not found: %w", fsID, err)
+	}
+	if err := ob.ForEach(func(_, v []byte) error {
+		nodeid := decodeID(v)
+		b, err := getNodeBucketByID(nodes, nodeid)
+		if err != nil {
+			return fmt.Errorf("failed to get file bucket %d: %w", nodeid, err)
+		}
+		size, _ := binary.Varint(b.Get(bucketKeySize))
+		if md, err := getMetadataBucketByID(metadataEntries, nodeid); err == nil {
+			nodeChunks, err := readChunks(md, size)
+			if err != nil {
+				return fmt.Errorf("failed to get chunks: %w", err)
+			}
+			for _, e := range nodeChunks {
+				if e.offset == off {
+					chunks = append(chunks, chunkEntryWithID{e, nodeid})
+				}
+			}
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	sort.Slice(chunks, func(i, j int) bool {
+		return chunks[i].innerOffset < chunks[j].innerOffset
+	})
+	return chunks, nil
+}
+
 func readChild(md *bolt.Bucket, base string) (uint32, error) {
 	if base == string(md.Get(bucketKeyChildName)) {
 		return decodeID(md.Get(bucketKeyChildID)), nil
@@ -344,10 +419,10 @@ func writeMetadataEntry(md *bolt.Bucket, m *metadataEntry) error {
 			break
 		}
 		if err := md.Put(bucketKeyChildID, encodeID(firstChild.id)); err != nil {
-			return errors.Wrapf(err, "failed to put id of first child %q", firstChildName)
+			return fmt.Errorf("failed to put id of first child %q: %w", firstChildName, err)
 		}
 		if err := md.Put(bucketKeyChildName, []byte(firstChildName)); err != nil {
-			return errors.Wrapf(err, "failed to put name first child %q", firstChildName)
+			return fmt.Errorf("failed to put name first child %q: %w", firstChildName, err)
 		}
 		if len(m.children) > 1 {
 			var cbkt *bolt.Bucket
@@ -369,7 +444,7 @@ func writeMetadataEntry(md *bolt.Bucket, m *metadataEntry) error {
 					}
 				}
 				if err := cbkt.Put([]byte(c.base), encodeID(c.id)); err != nil {
-					return errors.Wrapf(err, "failed to add child ID %q", c.id)
+					return fmt.Errorf("failed to add child ID %q: %w", c.id, err)
 				}
 			}
 		}
@@ -377,7 +452,7 @@ func writeMetadataEntry(md *bolt.Bucket, m *metadataEntry) error {
 	if len(m.chunks) > 0 {
 		first := m.chunks[0]
 		if err := md.Put(bucketKeyChunk, encodeChunkEntry(first)); err != nil {
-			return errors.Wrapf(err, "failed to set chunk %q", first.offset)
+			return fmt.Errorf("failed to set chunk %q: %w", first.offset, err)
 		}
 		var cbkt *bolt.Bucket
 		for _, e := range m.chunks[1:] {
@@ -394,39 +469,41 @@ func writeMetadataEntry(md *bolt.Bucket, m *metadataEntry) error {
 					return err
 				}
 			}
-			eoff, err := encodeInt(e.offset)
+			ecoff, err := encodeInt(e.chunkOffset)
 			if err != nil {
 				return err
 			}
-			if err := cbkt.Put(eoff, encodeChunkEntry(e)); err != nil {
+			if err := cbkt.Put(ecoff, encodeChunkEntry(e)); err != nil {
 				return err
 			}
 		}
 	}
 	if m.nextOffset > 0 {
 		if err := putInt(md, bucketKeyNextOffset, m.nextOffset); err != nil {
-			return errors.Wrapf(err, "failed to set next offset value %d", m.nextOffset)
+			return fmt.Errorf("failed to set next offset value %d: %w", m.nextOffset, err)
 		}
 	}
 	return nil
 }
 
 func encodeChunkEntry(e chunkEntry) []byte {
-	eb := make([]byte, 16+len([]byte(e.chunkDigest)))
+	eb := make([]byte, 24+len([]byte(e.chunkDigest)))
 	binary.BigEndian.PutUint64(eb[0:8], uint64(e.chunkOffset))
 	binary.BigEndian.PutUint64(eb[8:16], uint64(e.offset))
-	copy(eb[16:], []byte(e.chunkDigest))
+	binary.BigEndian.PutUint64(eb[16:24], uint64(e.innerOffset))
+	copy(eb[24:], []byte(e.chunkDigest))
 	return eb
 }
 
 func decodeChunkEntry(d []byte) (e chunkEntry, _ error) {
-	if len(d) < 16 {
+	if len(d) < 24 {
 		return e, fmt.Errorf("mulformed chunk entry (len:%d)", len(d))
 	}
 	e.chunkOffset = int64(binary.BigEndian.Uint64(d[0:8]))
 	e.offset = int64(binary.BigEndian.Uint64(d[8:16]))
-	if len(d) > 16 {
-		e.chunkDigest = string(d[16:])
+	e.innerOffset = int64(binary.BigEndian.Uint64(d[16:24]))
+	if len(d) > 24 {
+		e.chunkDigest = string(d[24:])
 	}
 	return e, nil
 }

cmd/containerd-stargz-grpc/db/reader.go

@@ -20,9 +20,9 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math"
 	"os"
 	"path"
@@ -35,11 +35,10 @@ import (
 	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containerd/stargz-snapshotter/metadata"
 	"github.com/goccy/go-json"
-	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
 	"github.com/rs/xid"
 	bolt "go.etcd.io/bbolt"
+	errbolt "go.etcd.io/bbolt/errors"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -75,7 +74,7 @@ func NewReader(db *bolt.DB, sr *io.SectionReader, opts ...metadata.Option) (meta
 	var rOpts metadata.Options
 	for _, o := range opts {
 		if err := o(&rOpts); err != nil {
-			return nil, errors.Wrapf(err, "failed to apply option")
+			return nil, fmt.Errorf("failed to apply option: %w", err)
 		}
 	}
 
@@ -100,7 +99,7 @@ func NewReader(db *bolt.DB, sr *io.SectionReader, opts ...metadata.Option) (meta
 		rOpts.Telemetry.GetFooterLatency(start)
 	}
 
-	var allErr error
+	var errs []error
 	var tocR io.ReadCloser
 	var decompressor metadata.Decompressor
 	for _, d := range decompressors {
@@ -109,33 +108,35 @@ func NewReader(db *bolt.DB, sr *io.SectionReader, opts ...metadata.Option) (meta
 		maybeTocBytes := footer[:fOffset]
 		_, tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:])
 		if err != nil {
-			allErr = multierror.Append(allErr, err)
+			errs = append(errs, err)
 			continue
 		}
-		if tocSize <= 0 {
+		if tocOffset >= 0 && tocSize <= 0 {
 			tocSize = sr.Size() - tocOffset - fSize
 		}
-		if tocSize < int64(len(maybeTocBytes)) {
+		if tocOffset >= 0 && tocSize < int64(len(maybeTocBytes)) {
 			maybeTocBytes = maybeTocBytes[:tocSize]
 		}
 		tocR, err = decompressTOC(d, sr, tocOffset, tocSize, maybeTocBytes, rOpts)
 		if err != nil {
-			allErr = multierror.Append(allErr, err)
+			errs = append(errs, err)
 			continue
 		}
 		decompressor = d
 		break
 	}
+
+	allErr := errors.Join(errs...)
 	if tocR == nil {
 		if allErr == nil {
 			return nil, fmt.Errorf("failed to get the reader of TOC: unknown")
 		}
-		return nil, errors.Wrapf(allErr, "failed to get the reader of TOC")
+		return nil, fmt.Errorf("failed to get the reader of TOC: %w", allErr)
 	}
 	defer tocR.Close()
 	r := &reader{sr: sr, db: db, initG: new(errgroup.Group), decompressor: decompressor}
 	if err := r.init(tocR, rOpts); err != nil {
-		return nil, errors.Wrapf(err, "failed to initialize matadata")
+		return nil, fmt.Errorf("failed to initialize matadata: %w", err)
 	}
 	return r, nil
 }
@@ -150,6 +151,20 @@ func maxFooterSize(blobSize int64, decompressors ...metadata.Decompressor) (res
 }
 
 func decompressTOC(d metadata.Decompressor, sr *io.SectionReader, tocOff, tocSize int64, tocBytes []byte, opts metadata.Options) (io.ReadCloser, error) {
+	if tocOff < 0 {
+		// This means that TOC isn't contained in the blob.
+		// We pass nil reader to DecompressTOC and expect that it acquires TOC from
+		// the external location.
+		start := time.Now()
+		tocR, err := d.DecompressTOC(nil)
+		if err != nil {
+			return nil, err
+		}
+		if opts.Telemetry != nil && opts.Telemetry.GetTocLatency != nil {
+			opts.Telemetry.GetTocLatency(start)
+		}
+		return tocR, nil
+	}
 	if len(tocBytes) > 0 {
 		start := time.Now() // before getting TOC
 		tocR, err := d.DecompressTOC(bytes.NewReader(tocBytes))
@@ -209,10 +224,10 @@ func (r *reader) init(decompressedR io.Reader, rOpts metadata.Options) (retErr e
 	for i := 0; i < 100; i++ {
 		fsID := xid.New().String()
 		if err := r.initRootNode(fsID); err != nil {
-			if errors.Is(err, bolt.ErrBucketExists) {
+			if errors.Is(err, errbolt.ErrBucketExists) {
 				continue // try with another id
 			}
-			return errors.Wrapf(err, "failed to initialize root node %q", fsID)
+			return fmt.Errorf("failed to initialize root node %q: %w", fsID, err)
 		}
 		ok = true
 		break
@@ -221,24 +236,26 @@ func (r *reader) init(decompressedR io.Reader, rOpts metadata.Options) (retErr e
 		return fmt.Errorf("failed to get a unique id for metadata reader")
 	}
 
-	f, err := ioutil.TempFile("", "")
+	f, err := os.CreateTemp("", "")
 	if err != nil {
 		return err
 	}
-	closeFunc := func() (closeErr error) {
+	closeFunc := func() error {
 		name := f.Name()
+		var errs []error
 		if err := f.Close(); err != nil {
-			closeErr = multierror.Append(closeErr, err)
+			errs = append(errs, err)
 		}
 		if err := os.Remove(name); err != nil {
-			closeErr = multierror.Append(closeErr, err)
+			errs = append(errs, err)
 		}
-		return
+		return errors.Join(errs...)
 	}
 	defer func() {
 		if retErr != nil {
 			if err := closeFunc(); err != nil {
-				retErr = multierror.Append(retErr, err)
+				retErr = errors.Join(retErr, err)
+				return
 			}
 		}
 	}()
@@ -247,7 +264,7 @@ func (r *reader) init(decompressedR io.Reader, rOpts metadata.Options) (retErr e
 	}
 	dgstr := digest.Canonical.Digester()
 	if _, err := io.Copy(f, io.TeeReader(decompressedR, dgstr.Hash())); err != nil {
-		return errors.Wrapf(err, "failed to read TOC")
+		return fmt.Errorf("failed to read TOC: %w", err)
 	}
 	r.tocDigest = dgstr.Digest()
 
@@ -283,6 +300,9 @@ func (r *reader) initRootNode(fsID string) error {
 		if _, err := lbkt.CreateBucket(bucketKeyMetadata); err != nil {
 			return err
 		}
+		if _, err := lbkt.CreateBucket(bucketKeyStream); err != nil {
+			return err
+		}
 		nodes, err := lbkt.CreateBucket(bucketKeyNodes)
 		if err != nil {
 			return err
@@ -311,7 +331,7 @@ func (r *reader) initNodes(tr io.Reader) error {
 	for {
 		t, err := dec.Token()
 		if err != nil {
-			return errors.Wrapf(err, "failed to get JSON token")
+			return fmt.Errorf("failed to get JSON token: %w", err)
 		}
 		if ele, ok := t.(string); ok {
 			if ele == "version" {
@@ -328,13 +348,14 @@ func (r *reader) initNodes(tr io.Reader) error {
 		}
 	}
 	md := make(map[uint32]*metadataEntry)
+	st := make(map[int64]map[int64]uint32)
 	if err := r.db.Batch(func(tx *bolt.Tx) (err error) {
 		nodes, err := getNodes(tx, r.fsID)
 		if err != nil {
 			return err
 		}
 		nodes.FillPercent = 1.0 // we only do sequential write to this bucket
-		var wantNextOffsetID uint32
+		var wantNextOffsetID []uint32
 		var lastEntBucketID uint32
 		var lastEntSize int64
 		var attr metadata.Attr
@@ -362,15 +383,15 @@ func (r *reader) initNodes(tr io.Reader) error {
 				if ent.Type == "hardlink" {
 					id, err = getIDByName(md, ent.LinkName, r.rootID)
 					if err != nil {
-						return errors.Wrapf(err, "%q is a hardlink but cannot get link destination %q", ent.Name, ent.LinkName)
+						return fmt.Errorf("%q is a hardlink but cannot get link destination %q: %w", ent.Name, ent.LinkName, err)
 					}
 					b, err = getNodeBucketByID(nodes, id)
 					if err != nil {
-						return errors.Wrapf(err, "cannot get hardlink destination %q ==> %q (%d)", ent.Name, ent.LinkName, id)
+						return fmt.Errorf("cannot get hardlink destination %q ==> %q (%d): %w", ent.Name, ent.LinkName, id, err)
 					}
 					numLink, _ := binary.Varint(b.Get(bucketKeyNumLink))
 					if err := putInt(b, bucketKeyNumLink, numLink+1); err != nil {
-						return errors.Wrapf(err, "cannot put NumLink of %q ==> %q", ent.Name, ent.LinkName)
+						return fmt.Errorf("cannot put NumLink of %q ==> %q: %w", ent.Name, ent.LinkName, err)
 					}
 				} else {
 					// Write node bucket
@@ -381,7 +402,7 @@ func (r *reader) initNodes(tr io.Reader) error {
 						if err == nil {
 							b, err = getNodeBucketByID(nodes, id)
 							if err != nil {
-								return errors.Wrapf(err, "failed to get directory bucket %d", id)
+								return fmt.Errorf("failed to get directory bucket %d: %w", id, err)
 							}
 							found = true
 							ent.NumLink = readNumLink(b)
@@ -403,27 +424,30 @@ func (r *reader) initNodes(tr io.Reader) error {
 						}
 					}
 					if err := writeAttr(b, attrFromTOCEntry(&ent, &attr)); err != nil {
-						return errors.Wrapf(err, "failed to set attr to %d(%q)", id, ent.Name)
+						return fmt.Errorf("failed to set attr to %d(%q): %w", id, ent.Name, err)
 					}
 				}
 
 				pdirName := parentDir(ent.Name)
 				pid, pb, err := r.getOrCreateDir(nodes, md, pdirName, r.rootID)
 				if err != nil {
-					return errors.Wrapf(err, "failed to create parent directory %q of %q", pdirName, ent.Name)
+					return fmt.Errorf("failed to create parent directory %q of %q: %w", pdirName, ent.Name, err)
 				}
 				if err := setChild(md, pb, pid, path.Base(ent.Name), id, ent.Type == "dir"); err != nil {
 					return err
 				}
 
-				if ent.Offset > 0 && wantNextOffsetID > 0 {
-					if md[wantNextOffsetID] == nil {
-						md[wantNextOffsetID] = &metadataEntry{}
+				if ent.Offset > 0 && ent.InnerOffset == 0 && len(wantNextOffsetID) > 0 {
+					for _, i := range wantNextOffsetID {
+						if md[i] == nil {
+							md[i] = &metadataEntry{}
+						}
+						md[i].nextOffset = ent.Offset
 					}
-					md[wantNextOffsetID].nextOffset = ent.Offset
+					wantNextOffsetID = nil
 				}
 				if ent.Type == "reg" && ent.Size > 0 {
-					wantNextOffsetID = id
+					wantNextOffsetID = append(wantNextOffsetID, id)
 				}
 
 				lastEntSize = ent.Size
@@ -433,21 +457,41 @@ func (r *reader) initNodes(tr io.Reader) error {
 				if md[lastEntBucketID] == nil {
 					md[lastEntBucketID] = &metadataEntry{}
 				}
-				ce := chunkEntry{ent.Offset, ent.ChunkOffset, ent.ChunkSize, ent.ChunkDigest}
+				ce := chunkEntry{ent.Offset, ent.ChunkOffset, ent.ChunkSize, ent.ChunkDigest, ent.InnerOffset}
 				md[lastEntBucketID].chunks = append(md[lastEntBucketID].chunks, ce)
+				if _, ok := st[ent.Offset]; !ok {
+					st[ent.Offset] = make(map[int64]uint32)
+				}
+				st[ent.Offset][ent.InnerOffset] = lastEntBucketID
 			}
 		}
-		if wantNextOffsetID > 0 {
-			if md[wantNextOffsetID] == nil {
-				md[wantNextOffsetID] = &metadataEntry{}
+		if len(wantNextOffsetID) > 0 {
+			for _, i := range wantNextOffsetID {
+				if md[i] == nil {
+					md[i] = &metadataEntry{}
+				}
+				md[i].nextOffset = r.sr.Size()
 			}
-			md[wantNextOffsetID].nextOffset = r.sr.Size()
 		}
 		return nil
 	}); err != nil {
 		return err
 	}
 
+	for mdK, d := range md {
+		for cK, ce := range d.chunks {
+			if len(st[ce.offset]) == 1 {
+				for ioff := range st[ce.offset] {
+					if ioff == 0 {
+						// This stream contains only 1 chunk with innerOffset=0. No need to record innerOffsets.
+						md[mdK].chunks[cK].innerOffset = -1 // indicates no following chunks in this stream.
+					}
+					break
+				}
+			}
+		}
+	}
+
 	addendum := make([]struct {
 		id []byte
 		md *metadataEntry
@@ -480,6 +524,62 @@ func (r *reader) initNodes(tr io.Reader) error {
 		return err
 	}
 
+	addendumStream := make([]struct {
+		offset []byte
+		st     map[int64]uint32
+	}, len(st))
+	i = 0
+	for off, s := range st {
+		singleStream := false
+		if len(s) == 1 {
+			for ioff := range s {
+				if ioff == 0 {
+					singleStream = true
+				}
+				break
+			}
+		}
+		if singleStream {
+			continue // This stream contains only 1 chunk with innerOffset=0. No need to record.
+		}
+		offKey, err := encodeInt(off)
+		if err != nil {
+			return err
+		}
+		addendumStream[i].offset, addendumStream[i].st = offKey, s
+		i++
+	}
+	addendumStream = addendumStream[:i]
+	if len(addendumStream) > 0 {
+		sort.Slice(addendumStream, func(i, j int) bool {
+			return bytes.Compare(addendumStream[i].offset, addendumStream[j].offset) < 0
+		})
+		if err := r.db.Batch(func(tx *bolt.Tx) (err error) {
+			stream, err := getStream(tx, r.fsID)
+			if err != nil {
+				return err
+			}
+			stream.FillPercent = 1.0 // we only do sequential write to this bucket
+			for _, s := range addendumStream {
+				stbkt, err := stream.CreateBucket(s.offset)
+				if err != nil {
+					return err
+				}
+				for innerOffset, nodeid := range s.st {
+					iOffKey, err := encodeInt(innerOffset)
+					if err != nil {
+						return err
+					}
+					if err := stbkt.Put(iOffKey, encodeID(nodeid)); err != nil {
+						return fmt.Errorf("failed to put inner offset info of %d: %w", nodeid, err)
+					}
+				}
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -513,7 +613,7 @@ func (r *reader) getOrCreateDir(nodes *bolt.Bucket, md map[uint32]*metadataEntry
 	} else {
 		b, err = getNodeBucketByID(nodes, id)
 		if err != nil {
-			return 0, nil, errors.Wrapf(err, "failed to get dir bucket %d", id)
+			return 0, nil, fmt.Errorf("failed to get dir bucket %d: %w", id, err)
 		}
 	}
 	return id, b, nil
@@ -521,7 +621,10 @@ func (r *reader) getOrCreateDir(nodes *bolt.Bucket, md map[uint32]*metadataEntry
 
 func (r *reader) waitInit() error {
 	// TODO: add timeout
-	return errors.Wrapf(r.initG.Wait(), "initialization failed")
+	if err := r.initG.Wait(); err != nil {
+		return fmt.Errorf("initialization failed: %w", err)
+	}
+	return nil
 }
 
 func (r *reader) view(fn func(tx *bolt.Tx) error) error {
@@ -558,7 +661,7 @@ func (r *reader) GetOffset(id uint32) (offset int64, _ error) {
 	if err := r.view(func(tx *bolt.Tx) error {
 		metadataEntries, err := getMetadata(tx, r.fsID)
 		if err != nil {
-			return errors.Wrapf(err, "metadata bucket of %q not found for searching offset of %d", r.fsID, id)
+			return fmt.Errorf("metadata bucket of %q not found for searching offset of %d: %w", r.fsID, id, err)
 		}
 		nodes, err := getNodes(tx, r.fsID)
 		if err != nil {
@@ -591,11 +694,11 @@ func (r *reader) GetAttr(id uint32) (attr metadata.Attr, _ error) {
 		if err := r.db.View(func(tx *bolt.Tx) error {
 			nodes, err := getNodes(tx, r.fsID)
 			if err != nil {
-				return errors.Wrapf(err, "nodes bucket of %q not found for sarching attr %d", r.fsID, id)
+				return fmt.Errorf("nodes bucket of %q not found for sarching attr %d: %w", r.fsID, id, err)
 			}
 			b, err := getNodeBucketByID(nodes, id)
 			if err != nil {
-				return errors.Wrapf(err, "failed to get attr bucket %d", id)
+				return fmt.Errorf("failed to get attr bucket %d: %w", id, err)
 			}
 			return readAttr(b, &attr)
 		}); err != nil {
@@ -606,11 +709,11 @@ func (r *reader) GetAttr(id uint32) (attr metadata.Attr, _ error) {
 	if err := r.view(func(tx *bolt.Tx) error {
 		nodes, err := getNodes(tx, r.fsID)
 		if err != nil {
-			return errors.Wrapf(err, "nodes bucket of %q not found for sarching attr %d", r.fsID, id)
+			return fmt.Errorf("nodes bucket of %q not found for sarching attr %d: %w", r.fsID, id, err)
 		}
 		b, err := getNodeBucketByID(nodes, id)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get attr bucket %d", id)
+			return fmt.Errorf("failed to get attr bucket %d: %w", id, err)
 		}
 		return readAttr(b, &attr)
 	}); err != nil {
@@ -624,23 +727,23 @@ func (r *reader) GetChild(pid uint32, base string) (id uint32, attr metadata.Att
 	if err := r.view(func(tx *bolt.Tx) error {
 		metadataEntries, err := getMetadata(tx, r.fsID)
 		if err != nil {
-			return errors.Wrapf(err, "metadata bucket of %q not found for getting child of %d", r.fsID, pid)
+			return fmt.Errorf("metadata bucket of %q not found for getting child of %d: %w", r.fsID, pid, err)
 		}
 		md, err := getMetadataBucketByID(metadataEntries, pid)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get parent metadata %d", pid)
+			return fmt.Errorf("failed to get parent metadata %d: %w", pid, err)
 		}
 		id, err = readChild(md, base)
 		if err != nil {
-			return errors.Wrapf(err, "failed to read child %q of %d", base, pid)
+			return fmt.Errorf("failed to read child %q of %d: %w", base, pid, err)
 		}
 		nodes, err := getNodes(tx, r.fsID)
 		if err != nil {
-			return errors.Wrapf(err, "nodes bucket of %q not found for getting child of %d", r.fsID, pid)
+			return fmt.Errorf("nodes bucket of %q not found for getting child of %d: %w", r.fsID, pid, err)
 		}
 		child, err := getNodeBucketByID(nodes, id)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get child bucket %d", id)
+			return fmt.Errorf("failed to get child bucket %d: %w", id, err)
 		}
 		return readAttr(child, &attr)
 	}); err != nil {
@@ -660,7 +763,7 @@ func (r *reader) ForeachChild(id uint32, f func(name string, id uint32, mode os.
 	if err := r.view(func(tx *bolt.Tx) error {
 		metadataEntries, err := getMetadata(tx, r.fsID)
 		if err != nil {
-			return errors.Wrapf(err, "nodes bucket of %q not found for getting child of %d", r.fsID, id)
+			return fmt.Errorf("nodes bucket of %q not found for getting child of %d: %w", r.fsID, id, err)
 		}
 		md, err := getMetadataBucketByID(metadataEntries, id)
 		if err != nil {
@@ -674,12 +777,12 @@ func (r *reader) ForeachChild(id uint32, f func(name string, id uint32, mode os.
 			if nodes == nil {
 				nodes, err = getNodes(tx, r.fsID)
 				if err != nil {
-					return errors.Wrapf(err, "nodes bucket of %q not found for getting children of %d", r.fsID, id)
+					return fmt.Errorf("nodes bucket of %q not found for getting children of %d: %w", r.fsID, id, err)
 				}
 			}
 			firstChild, err := getNodeBucketByID(nodes, firstID)
 			if err != nil {
-				return errors.Wrapf(err, "failed to get first child bucket %d", firstID)
+				return fmt.Errorf("failed to get first child bucket %d: %w", firstID, err)
 			}
 			mode, _ := binary.Uvarint(firstChild.Get(bucketKeyMode))
 			children[string(firstName)] = childInfo{firstID, os.FileMode(uint32(mode))}
@@ -692,14 +795,14 @@ func (r *reader) ForeachChild(id uint32, f func(name string, id uint32, mode os.
 		if nodes == nil {
 			nodes, err = getNodes(tx, r.fsID)
 			if err != nil {
-				return errors.Wrapf(err, "nodes bucket of %q not found for getting children of %d", r.fsID, id)
+				return fmt.Errorf("nodes bucket of %q not found for getting children of %d: %w", r.fsID, id, err)
 			}
 		}
 		return cbkt.ForEach(func(k, v []byte) error {
 			id := decodeID(v)
 			child, err := getNodeBucketByID(nodes, id)
 			if err != nil {
-				return errors.Wrapf(err, "failed to get child bucket %d", id)
+				return fmt.Errorf("failed to get child bucket %d: %w", id, err)
 			}
 			mode, _ := binary.Uvarint(child.Get(bucketKeyMode))
 			children[string(k)] = childInfo{id, os.FileMode(uint32(mode))}
@@ -716,8 +819,19 @@ func (r *reader) ForeachChild(id uint32, f func(name string, id uint32, mode os.
 	return nil
 }
 
+// OpenFileWithPreReader returns a section reader of the specified node.
+// When it reads other ranges than required by the returned reader (e.g. when the target range is located in
+// a large chunk with innerOffset), these chunks are passed to the callback so that it can be cached for futural use.
+func (r *reader) OpenFileWithPreReader(id uint32, preRead func(nid uint32, chunkOffset, chunkSize int64, chunkDigest string, r io.Reader) error) (metadata.File, error) {
+	return r.openFile(id, preRead)
+}
+
 // OpenFile returns a section reader of the specified node.
 func (r *reader) OpenFile(id uint32) (metadata.File, error) {
+	return r.openFile(id, nil)
+}
+
+func (r *reader) openFile(id uint32, preRead func(id uint32, chunkOffset, chunkSize int64, chunkDigest string, r io.Reader) error) (metadata.File, error) {
 	var chunks []chunkEntry
 	var size int64
 
@@ -725,11 +839,11 @@ func (r *reader) OpenFile(id uint32) (metadata.File, error) {
 	if err := r.view(func(tx *bolt.Tx) error {
 		nodes, err := getNodes(tx, r.fsID)
 		if err != nil {
-			return errors.Wrapf(err, "nodes bucket of %q not found for opening %d", r.fsID, id)
+			return fmt.Errorf("nodes bucket of %q not found for opening %d: %w", r.fsID, id, err)
 		}
 		b, err := getNodeBucketByID(nodes, id)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get file bucket %d", id)
+			return fmt.Errorf("failed to get file bucket %d: %w", id, err)
 		}
 		size, _ = binary.Varint(b.Get(bucketKeySize))
 		m, _ := binary.Uvarint(b.Get(bucketKeyMode))
@@ -739,12 +853,12 @@ func (r *reader) OpenFile(id uint32) (metadata.File, error) {
 
 		metadataEntries, err := getMetadata(tx, r.fsID)
 		if err != nil {
-			return errors.Wrapf(err, "metadata bucket of %q not found for opening %d", r.fsID, id)
+			return fmt.Errorf("metadata bucket of %q not found for opening %d: %w", r.fsID, id, err)
 		}
 		if md, err := getMetadataBucketByID(metadataEntries, id); err == nil {
 			chunks, err = readChunks(md, size)
 			if err != nil {
-				return errors.Wrapf(err, "failed to get chunks")
+				return fmt.Errorf("failed to get chunks: %w", err)
 			}
 			nextOffset, _ = binary.Varint(md.Get(bucketKeyNextOffset))
 		}
@@ -757,6 +871,7 @@ func (r *reader) OpenFile(id uint32) (metadata.File, error) {
 		size:       size,
 		ents:       chunks,
 		nextOffset: nextOffset,
+		preRead:    preRead,
 	}
 	return &file{io.NewSectionReader(fr, 0, size), chunks}, nil
 }
@@ -783,6 +898,7 @@ type fileReader struct {
 	size       int64
 	ents       []chunkEntry
 	nextOffset int64
+	preRead    func(id uint32, chunkOffset, chunkSize int64, chunkDigest string, r io.Reader) error
 }
 
 // ReadAt reads file payload of this file.
@@ -828,11 +944,56 @@ func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) {
 		return 0, fmt.Errorf("fileReader.ReadAt.decompressor.Reader: %v", err)
 	}
 	defer dr.Close()
-	base := off - ent.chunkOffset
-	if n, err := io.CopyN(ioutil.Discard, dr, base); n != base || err != nil {
-		return 0, fmt.Errorf("discard of %d bytes = %v, %v", base, n, err)
+
+	// Stream that doesn't contain multiple chunks is indicated as ent.innerOffset < 0.
+	if fr.preRead == nil || ent.innerOffset < 0 {
+		base := off - ent.chunkOffset
+		if ent.innerOffset > 0 {
+			base += ent.innerOffset
+		}
+		if n, err := io.CopyN(io.Discard, dr, base); n != base || err != nil {
+			return 0, fmt.Errorf("discard of %d bytes = %v, %v", base, n, err)
+		}
+		return io.ReadFull(dr, p)
 	}
-	return io.ReadFull(dr, p)
+
+	var innerChunks []chunkEntryWithID
+	if err := fr.r.view(func(tx *bolt.Tx) error {
+		innerChunks, err = readInnerChunks(tx, fr.r.fsID, ent.offset)
+		return err
+	}); err != nil {
+		return 0, err
+	}
+	var found bool
+	var nr int64
+	var retN int
+	var retErr error
+	for _, e := range innerChunks {
+		// Fully read the previous chunk reader so that the seek position goes at the current chunk offset
+		if in, err := io.CopyN(io.Discard, dr, e.innerOffset-nr); err != nil || in != e.innerOffset-nr {
+			return 0, fmt.Errorf("discard of remaining %d bytes != %v, %v", e.innerOffset-nr, in, err)
+		}
+		nr += e.innerOffset - nr
+		if e.innerOffset == ent.innerOffset {
+			found = true
+			base := off - ent.chunkOffset
+			if n, err := io.CopyN(io.Discard, dr, base); n != base || err != nil {
+				return 0, fmt.Errorf("discard of offset %d bytes != %v, %v", off, n, err)
+			}
+			retN, retErr = io.ReadFull(dr, p)
+			nr += base + int64(retN)
+			continue
+		}
+		cr := &countReader{r: io.LimitReader(dr, e.chunkSize)}
+		if err := fr.preRead(e.id, e.chunkOffset, e.chunkSize, e.chunkDigest, cr); err != nil {
+			return 0, fmt.Errorf("failed to pre read: %w", err)
+		}
+		nr += cr.n
+	}
+	if !found {
+		return 0, fmt.Errorf("fileReader.ReadAt: target entry not found")
+	}
+	return retN, retErr
 }
 
 // TODO: share it with memory pkg
@@ -884,7 +1045,7 @@ func setChild(md map[uint32]*metadataEntry, pb *bolt.Bucket, pid uint32, base st
 	if isDir {
 		numLink, _ := binary.Varint(pb.Get(bucketKeyNumLink))
 		if err := putInt(pb, bucketKeyNumLink, numLink+1); err != nil {
-			return errors.Wrapf(err, "cannot add numlink for children")
+			return fmt.Errorf("cannot add numlink for children: %w", err)
 		}
 	}
 	return nil
@@ -920,6 +1081,7 @@ func resetEnt(ent *estargz.TOCEntry) {
 	ent.ChunkOffset = 0
 	ent.ChunkSize = 0
 	ent.ChunkDigest = ""
+	ent.InnerOffset = 0
 }
 
 func positive(n int64) int64 {
@@ -983,3 +1145,14 @@ func (r *reader) NumOfChunks(id uint32) (i int, _ error) {
 	}
 	return
 }
+
+type countReader struct {
+	r io.Reader
+	n int64
+}
+
+func (cr *countReader) Read(p []byte) (n int, err error) {
+	n, err = cr.r.Read(p)
+	cr.n += int64(n)
+	return
+}

cmd/containerd-stargz-grpc/db/reader_test.go

@@ -0,0 +1,153 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package db
+
+import (
+	"io"
+	"os"
+	"testing"
+
+	"github.com/containerd/stargz-snapshotter/fs/layer"
+	fsreader "github.com/containerd/stargz-snapshotter/fs/reader"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	"github.com/containerd/stargz-snapshotter/metadata/testutil"
+	bolt "go.etcd.io/bbolt"
+)
+
+func TestReader(t *testing.T) {
+	testRunner := &testutil.TestRunner{
+		TestingT: t,
+		Runner: func(testingT testutil.TestingT, name string, run func(t testutil.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	testutil.TestReader(testRunner, newTestableReader)
+}
+
+func TestFSReader(t *testing.T) {
+	testRunner := &fsreader.TestRunner{
+		TestingT: t,
+		Runner: func(testingT fsreader.TestingT, name string, run func(t fsreader.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	fsreader.TestSuiteReader(testRunner, newStore)
+}
+
+func TestFSLayer(t *testing.T) {
+	testRunner := &layer.TestRunner{
+		TestingT: t,
+		Runner: func(testingT layer.TestingT, name string, run func(t layer.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	layer.TestSuiteLayer(testRunner, newStore)
+}
+
+func newTestableReader(sr *io.SectionReader, opts ...metadata.Option) (testutil.TestableReader, error) {
+	f, err := os.CreateTemp("", "readertestdb")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+	db, err := bolt.Open(f.Name(), 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+	r, err := NewReader(db, sr, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return &testableReadCloser{
+		TestableReader: r.(*reader),
+		closeFn: func() error {
+			db.Close()
+			return os.Remove(f.Name())
+		},
+	}, nil
+}
+
+func newStore(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error) {
+	f, err := os.CreateTemp("", "readertestdb")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	db, err := bolt.Open(f.Name(), 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+	r, err := NewReader(db, sr, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return &readCloser{
+		Reader: r,
+		closeFn: func() error {
+			db.Close()
+			return os.Remove(f.Name())
+		},
+	}, nil
+}
+
+type readCloser struct {
+	metadata.Reader
+	closeFn func() error
+}
+
+func (r *readCloser) Close() error {
+	r.closeFn()
+	return r.Reader.Close()
+}
+
+type testableReadCloser struct {
+	testutil.TestableReader
+	closeFn func() error
+}
+
+func (r *testableReadCloser) Close() error {
+	r.closeFn()
+	return r.TestableReader.Close()
+}

cmd/containerd-stargz-grpc/fsopts/fsopts.go

@@ -0,0 +1,80 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fsopts
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path/filepath"
+
+	"github.com/containerd/log"
+	dbmetadata "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/db"
+	ipfs "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/ipfs"
+	"github.com/containerd/stargz-snapshotter/fs"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
+	bolt "go.etcd.io/bbolt"
+)
+
+type Config struct {
+	EnableIpfs    bool
+	MetadataStore string
+	OpenBoltDB    func(string) (*bolt.DB, error)
+}
+
+const (
+	memoryMetadataType = "memory"
+	dbMetadataType     = "db"
+)
+
+func ConfigFsOpts(ctx context.Context, rootDir string, config *Config) ([]fs.Option, error) {
+	fsOpts := []fs.Option{fs.WithMetricsLogLevel(log.InfoLevel)}
+
+	if config.EnableIpfs {
+		fsOpts = append(fsOpts, fs.WithResolveHandler("ipfs", new(ipfs.ResolveHandler)))
+	}
+
+	mt, err := getMetadataStore(rootDir, config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure metadata store: %w", err)
+	}
+	fsOpts = append(fsOpts, fs.WithMetadataStore(mt))
+
+	return fsOpts, nil
+}
+
+func getMetadataStore(rootDir string, config *Config) (metadata.Store, error) {
+	switch config.MetadataStore {
+	case "", memoryMetadataType:
+		return memorymetadata.NewReader, nil
+	case dbMetadataType:
+		if config.OpenBoltDB == nil {
+			return nil, fmt.Errorf("bolt DB is not configured")
+		}
+		db, err := config.OpenBoltDB(filepath.Join(rootDir, "metadata.db"))
+		if err != nil {
+			return nil, err
+		}
+		return func(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error) {
+			return dbmetadata.NewReader(db, sr, opts...)
+		}, nil
+	default:
+		return nil, fmt.Errorf("unknown metadata store type: %v; must be %v or %v",
+			config.MetadataStore, memoryMetadataType, dbMetadataType)
+	}
+}

cmd/containerd-stargz-grpc/ipfs/resolvehandler.go

@@ -21,86 +21,59 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
+	"os"
 
 	"github.com/containerd/stargz-snapshotter/fs/remote"
 	"github.com/containerd/stargz-snapshotter/ipfs"
-	httpapi "github.com/ipfs/go-ipfs-http-client"
-	iface "github.com/ipfs/interface-go-ipfs-core"
-	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+	ipfsclient "github.com/containerd/stargz-snapshotter/ipfs/client"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 type ResolveHandler struct{}
 
 func (r *ResolveHandler) Handle(ctx context.Context, desc ocispec.Descriptor) (remote.Fetcher, int64, error) {
-	p, err := ipfs.GetPath(desc)
+	cid, err := ipfs.GetCID(desc)
 	if err != nil {
 		return nil, 0, err
 	}
-	client, err := httpapi.NewLocalApi()
+	var ipath string
+	if idir := os.Getenv("IPFS_PATH"); idir != "" {
+		ipath = idir
+	}
+	// HTTP is only supported as of now. We can add https support here if needed (e.g. for connecting to it via proxy, etc)
+	iurl, err := ipfsclient.GetIPFSAPIAddress(ipath, "http")
 	if err != nil {
 		return nil, 0, err
 	}
-	n, err := client.Unixfs().Get(ctx, p)
+	client := ipfsclient.New(iurl)
+	info, err := client.StatCID(cid)
 	if err != nil {
 		return nil, 0, err
 	}
-	if _, ok := n.(interface {
-		io.ReaderAt
-	}); !ok {
-		return nil, 0, fmt.Errorf("ReaderAt is not implemented")
-	}
-	defer n.Close()
-	s, err := n.Size()
-	if err != nil {
-		return nil, 0, err
-	}
-	return &fetcher{client, p}, s, nil
+	return &fetcher{cid: cid, size: int64(info.Size), client: client}, int64(info.Size), nil
 }
 
 type fetcher struct {
-	api  iface.CoreAPI
-	path ipath.Path
+	cid  string
+	size int64
+
+	client *ipfsclient.Client
 }
 
 func (f *fetcher) Fetch(ctx context.Context, off int64, size int64) (io.ReadCloser, error) {
-	n, err := f.api.Unixfs().Get(ctx, f.path)
-	if err != nil {
-		return nil, err
+	if off > f.size {
+		return nil, fmt.Errorf("offset is larger than the size of the blob %d(offset) > %d(blob size)", off, f.size)
 	}
-	ra, ok := n.(interface {
-		io.ReaderAt
-	})
-	if !ok {
-		return nil, fmt.Errorf("ReaderAt is not implemented")
-	}
-	return &readCloser{
-		Reader:    io.NewSectionReader(ra, off, size),
-		closeFunc: n.Close,
-	}, nil
+	o, s := int(off), int(size)
+	return f.client.Get("/ipfs/"+f.cid, &o, &s)
 }
 
 func (f *fetcher) Check() error {
-	n, err := f.api.Unixfs().Get(context.Background(), f.path)
-	if err != nil {
-		return err
-	}
-	if _, ok := n.(interface {
-		io.ReaderAt
-	}); !ok {
-		return fmt.Errorf("ReaderAt is not implemented")
-	}
-	return n.Close()
+	_, err := f.client.StatCID(f.cid)
+	return err
 }
 
 func (f *fetcher) GenID(off int64, size int64) string {
-	sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", f.path.String(), off, size)))
+	sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", f.cid, off, size)))
 	return fmt.Sprintf("%x", sum)
 }
-
-type readCloser struct {
-	io.Reader
-	closeFunc func() error
-}
-
-func (r *readCloser) Close() error { return r.closeFunc() }

cmd/containerd-stargz-grpc/main.go

@@ -25,42 +25,38 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path/filepath"
 	"time"
 
 	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
-	"github.com/containerd/containerd/contrib/snapshotservice"
-	"github.com/containerd/containerd/defaults"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/pkg/dialer"
-	"github.com/containerd/containerd/snapshots"
-	"github.com/containerd/containerd/sys"
-	ipfs "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/ipfs"
-	"github.com/containerd/stargz-snapshotter/fs"
+	"github.com/containerd/containerd/v2/contrib/snapshotservice"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/containerd/v2/pkg/sys"
+	"github.com/containerd/log"
+	"github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/fsopts"
+	"github.com/containerd/stargz-snapshotter/fusemanager"
 	"github.com/containerd/stargz-snapshotter/service"
-	"github.com/containerd/stargz-snapshotter/service/keychain/cri"
-	"github.com/containerd/stargz-snapshotter/service/keychain/dockerconfig"
-	"github.com/containerd/stargz-snapshotter/service/keychain/kubeconfig"
-	"github.com/containerd/stargz-snapshotter/service/resolver"
+	"github.com/containerd/stargz-snapshotter/service/keychain/keychainconfig"
+	snbase "github.com/containerd/stargz-snapshotter/snapshot"
 	"github.com/containerd/stargz-snapshotter/version"
 	sddaemon "github.com/coreos/go-systemd/v22/daemon"
 	metrics "github.com/docker/go-metrics"
 	"github.com/pelletier/go-toml"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
+	bolt "go.etcd.io/bbolt"
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/backoff"
-	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
 const (
 	defaultAddress             = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
 	defaultConfigPath          = "/etc/containerd-stargz-grpc/config.toml"
-	defaultLogLevel            = logrus.InfoLevel
+	defaultLogLevel            = log.InfoLevel
 	defaultRootDir             = "/var/lib/containerd-stargz-grpc"
 	defaultImageServiceAddress = "/run/containerd/containerd.sock"
+	defaultFuseManagerAddress  = "/run/containerd-stargz-grpc/fuse-manager.sock"
+	fuseManagerBin             = "stargz-fuse-manager"
 )
 
 var (
@@ -75,22 +71,40 @@ type snapshotterConfig struct {
 	service.Config
 
 	// MetricsAddress is address for the metrics API
-	MetricsAddress string `toml:"metrics_address"`
+	MetricsAddress string `toml:"metrics_address" json:"metrics_address"`
 
 	// NoPrometheus is a flag to disable the emission of the metrics
-	NoPrometheus bool `toml:"no_prometheus"`
+	NoPrometheus bool `toml:"no_prometheus" json:"no_prometheus"`
 
 	// DebugAddress is a Unix domain socket address where the snapshotter exposes /debug/ endpoints.
-	DebugAddress string `toml:"debug_address"`
+	DebugAddress string `toml:"debug_address" json:"debug_address"`
 
 	// IPFS is a flag to enbale lazy pulling from IPFS.
-	IPFS bool `toml:"ipfs"`
+	IPFS bool `toml:"ipfs" json:"ipfs"`
+
+	// MetadataStore is the type of the metadata store to use.
+	MetadataStore string `toml:"metadata_store" default:"memory" json:"metadata_store"`
+
+	// FuseManagerConfig is configuration for fusemanager
+	FuseManagerConfig `toml:"fuse_manager" json:"fuse_manager"`
+}
+
+type FuseManagerConfig struct {
+	// Enable is whether detach fusemanager or not
+	Enable bool `toml:"enable" default:"false" json:"enable"`
+
+	// Address is address for the fusemanager's GRPC server (default: "/run/containerd-stargz-grpc/fuse-manager.sock")
+	Address string `toml:"address" json:"address"`
+
+	// Path is path to the fusemanager's executable (default: looking for a binary "stargz-fuse-manager")
+	Path string `toml:"path" json:"path"`
 }
 
 func main() {
-	rand.Seed(time.Now().UnixNano())
+	rand.Seed(time.Now().UnixNano()) //nolint:staticcheck // Global math/rand seed is deprecated, but still used by external dependencies
 	flag.Parse()
-	lvl, err := logrus.ParseLevel(*logLevel)
+	log.SetFormat(log.JSONFormat)
+	err := log.SetLevel(*logLevel)
 	if err != nil {
 		log.L.WithError(err).Fatal("failed to prepare logger")
 	}
@@ -98,23 +112,19 @@ func main() {
 		fmt.Println("containerd-stargz-grpc", version.Version, version.Revision)
 		return
 	}
-	logrus.SetLevel(lvl)
-	logrus.SetFormatter(&logrus.JSONFormatter{
-		TimestampFormat: log.RFC3339NanoFixed,
-	})
 
 	var (
 		ctx    = log.WithLogger(context.Background(), log.L)
 		config snapshotterConfig
 	)
 	// Streams log of standard lib (go-fuse uses this) into debug log
-	// Snapshotter should use "github.com/containerd/containerd/log" otherwize
+	// Snapshotter should use "github.com/containerd/log" otherwize
 	// logs are always printed as "debug" mode.
-	golog.SetOutput(log.G(ctx).WriterLevel(logrus.DebugLevel))
+	golog.SetOutput(log.G(ctx).WriterLevel(log.DebugLevel))
 
 	// Get configuration from specified file
 	tree, err := toml.LoadFile(*configPath)
-	if err != nil && !(os.IsNotExist(err) && *configPath == defaultConfigPath) {
+	if err != nil && (!os.IsNotExist(err) || *configPath != defaultConfigPath) {
 		log.G(ctx).WithError(err).Fatalf("failed to load config file %q", *configPath)
 	}
 	if err := tree.Unmarshal(&config); err != nil {
@@ -128,53 +138,126 @@ func main() {
 	// Create a gRPC server
 	rpc := grpc.NewServer()
 
+	// Configure FUSE passthrough
+	// Always set Direct to true to ensure that
+	// *directoryCache.Get always return *os.File instead of buffer
+	if config.PassThrough {
+		config.Direct = true
+	}
+
 	// Configure keychain
-	credsFuncs := []resolver.Credential{dockerconfig.NewDockerconfigKeychain(ctx)}
-	if config.Config.KubeconfigKeychainConfig.EnableKeychain {
-		var opts []kubeconfig.Option
-		if kcp := config.Config.KubeconfigKeychainConfig.KubeconfigPath; kcp != "" {
-			opts = append(opts, kubeconfig.WithKubeconfigPath(kcp))
-		}
-		credsFuncs = append(credsFuncs, kubeconfig.NewKubeconfigKeychain(ctx, opts...))
+	keyChainConfig := keychainconfig.Config{
+		EnableKubeKeychain:         config.KubeconfigKeychainConfig.EnableKeychain,
+		EnableCRIKeychain:          config.CRIKeychainConfig.EnableKeychain,
+		KubeconfigPath:             config.KubeconfigPath,
+		DefaultImageServiceAddress: defaultImageServiceAddress,
+		ImageServicePath:           config.ImageServicePath,
 	}
-	if config.Config.CRIKeychainConfig.EnableKeychain {
-		// connects to the backend CRI service (defaults to containerd socket)
-		criAddr := defaultImageServiceAddress
-		if cp := config.CRIKeychainConfig.ImageServicePath; cp != "" {
-			criAddr = cp
-		}
-		connectCRI := func() (runtime.ImageServiceClient, error) {
-			// TODO: make gRPC options configurable from config.toml
-			backoffConfig := backoff.DefaultConfig
-			backoffConfig.MaxDelay = 3 * time.Second
-			connParams := grpc.ConnectParams{
-				Backoff: backoffConfig,
-			}
-			gopts := []grpc.DialOption{
-				grpc.WithInsecure(),
-				grpc.WithConnectParams(connParams),
-				grpc.WithContextDialer(dialer.ContextDialer),
-				grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
-				grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
-			}
-			conn, err := grpc.Dial(dialer.DialAddress(criAddr), gopts...)
+
+	var rs snapshots.Snapshotter
+	fuseManagerConfig := config.FuseManagerConfig
+	if fuseManagerConfig.Enable {
+		fmPath := fuseManagerConfig.Path
+		if fmPath == "" {
+			var err error
+			fmPath, err = exec.LookPath(fuseManagerBin)
 			if err != nil {
-				return nil, err
+				log.G(ctx).WithError(err).Fatalf("failed to find fusemanager bin")
 			}
-			return runtime.NewImageServiceClient(conn), nil
 		}
-		f, criServer := cri.NewCRIKeychain(ctx, connectCRI)
-		runtime.RegisterImageServiceServer(rpc, criServer)
-		credsFuncs = append(credsFuncs, f)
-	}
-	var fsOpts []fs.Option
-	if config.IPFS {
-		fsOpts = append(fsOpts, fs.WithResolveHandler("ipfs", new(ipfs.ResolveHandler)))
-	}
-	rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config,
-		service.WithCredsFuncs(credsFuncs...), service.WithFilesystemOptions(fsOpts...))
-	if err != nil {
-		log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
+		fmAddr := fuseManagerConfig.Address
+		if fmAddr == "" {
+			fmAddr = defaultFuseManagerAddress
+		}
+
+		if !filepath.IsAbs(fmAddr) {
+			log.G(ctx).WithError(err).Fatalf("fuse manager address must be an absolute path: %s", fmAddr)
+		}
+		managerNewlyStarted, err := fusemanager.StartFuseManager(ctx, fmPath, fmAddr, filepath.Join(*rootDir, "fusestore.db"), *logLevel, filepath.Join(*rootDir, "stargz-fuse-manager.log"))
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to start fusemanager")
+		}
+
+		fuseManagerConfig := fusemanager.Config{
+			Config:                     config.Config,
+			IPFS:                       config.IPFS,
+			MetadataStore:              config.MetadataStore,
+			DefaultImageServiceAddress: defaultImageServiceAddress,
+		}
+
+		fs, err := fusemanager.NewManagerClient(ctx, *rootDir, fmAddr, &fuseManagerConfig)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure fusemanager")
+		}
+		flags := []snbase.Opt{snbase.AsynchronousRemove}
+		// "managerNewlyStarted" being true indicates that the FUSE manager is newly started. To
+		// fully recover the snapshotter and the FUSE manager's state, we need to restore
+		// all snapshot mounts. If managerNewlyStarted is false, the existing FUSE manager maintains
+		// snapshot mounts so we don't need to restore them.
+		if !managerNewlyStarted {
+			flags = append(flags, snbase.NoRestore)
+		}
+		rs, err = snbase.NewSnapshotter(ctx, filepath.Join(*rootDir, "snapshotter"), fs, flags...)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
+		}
+		log.G(ctx).Infof("Start snapshotter with fusemanager mode")
+	} else {
+		crirpc := rpc
+		// For CRI keychain, if listening path is different from stargz-snapshotter's socket, prepare for the dedicated grpc server and the socket.
+		serveCRISocket := config.CRIKeychainConfig.EnableKeychain && config.ListenPath != "" && config.ListenPath != *address
+		if serveCRISocket {
+			crirpc = grpc.NewServer()
+		}
+		credsFuncs, err := keychainconfig.ConfigKeychain(ctx, crirpc, &keyChainConfig)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure keychain")
+		}
+		if serveCRISocket {
+			addr := config.ListenPath
+			// Prepare the directory for the socket
+			if err := os.MkdirAll(filepath.Dir(addr), 0700); err != nil {
+				log.G(ctx).WithError(err).Fatalf("failed to create directory %q", filepath.Dir(addr))
+			}
+
+			// Try to remove the socket file to avoid EADDRINUSE
+			if err := os.RemoveAll(addr); err != nil {
+				log.G(ctx).WithError(err).Fatalf("failed to remove %q", addr)
+			}
+
+			// Listen and serve
+			l, err := net.Listen("unix", addr)
+			if err != nil {
+				log.G(ctx).WithError(err).Fatalf("error on listen socket %q", addr)
+			}
+			go func() {
+				if err := crirpc.Serve(l); err != nil {
+					log.G(ctx).WithError(err).Errorf("error on serving CRI via socket %q", addr)
+				}
+			}()
+		}
+
+		fsConfig := fsopts.Config{
+			EnableIpfs:    config.IPFS,
+			MetadataStore: config.MetadataStore,
+			OpenBoltDB: func(p string) (*bolt.DB, error) {
+				return bolt.Open(p, 0600, &bolt.Options{
+					NoFreelistSync:  true,
+					InitialMmapSize: 64 * 1024 * 1024,
+					FreelistType:    bolt.FreelistMapType,
+				})
+			},
+		}
+		fsOpts, err := fsopts.ConfigFsOpts(ctx, *rootDir, &fsConfig)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure fs config")
+		}
+
+		rs, err = service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config,
+			service.WithCredsFuncs(credsFuncs...), service.WithFilesystemOptions(fsOpts...))
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
+		}
 	}
 
 	cleanup, err := serve(ctx, rpc, *address, rs, config)
@@ -182,7 +265,18 @@ func main() {
 		log.G(ctx).WithError(err).Fatalf("failed to serve snapshotter")
 	}
 
-	if cleanup {
+	// When FUSE manager is disabled, FUSE servers are goroutines in the
+	// contaienrd-stargz-grpc process. So killing containerd-stargz-grpc will
+	// result in all FUSE mount becoming unavailable with leaving all resources
+	// (e.g. temporary cache) on the node. To ensure graceful shutdown, we
+	// should always cleanup mounts and associated resources here.
+	//
+	// When FUSE manager is enabled, those mounts are still under the control by
+	// the FUSE manager so we need to avoid cleaning them up unless explicitly
+	// commanded via SIGINT. The user can use SIGINT to gracefully killing the FUSE
+	// manager before rebooting the node for ensuring that the all snapshots are
+	// unmounted with cleaning up associated temporary resources.
+	if cleanup || !fuseManagerConfig.Enable {
 		log.G(ctx).Debug("Closing the snapshotter")
 		rs.Close()
 	}
@@ -198,12 +292,12 @@ func serve(ctx context.Context, rpc *grpc.Server, addr string, rs snapshots.Snap
 
 	// Prepare the directory for the socket
 	if err := os.MkdirAll(filepath.Dir(addr), 0700); err != nil {
-		return false, errors.Wrapf(err, "failed to create directory %q", filepath.Dir(addr))
+		return false, fmt.Errorf("failed to create directory %q: %w", filepath.Dir(addr), err)
 	}
 
 	// Try to remove the socket file to avoid EADDRINUSE
 	if err := os.RemoveAll(addr); err != nil {
-		return false, errors.Wrapf(err, "failed to remove %q", addr)
+		return false, fmt.Errorf("failed to remove %q: %w", addr, err)
 	}
 
 	errCh := make(chan error, 1)
@@ -212,13 +306,13 @@ func serve(ctx context.Context, rpc *grpc.Server, addr string, rs snapshots.Snap
 	if config.MetricsAddress != "" && !config.NoPrometheus {
 		l, err := net.Listen("tcp", config.MetricsAddress)
 		if err != nil {
-			return false, errors.Wrapf(err, "failed to get listener for metrics endpoint")
+			return false, fmt.Errorf("failed to get listener for metrics endpoint: %w", err)
 		}
 		m := http.NewServeMux()
 		m.Handle("/metrics", metrics.Handler())
 		go func() {
 			if err := http.Serve(l, m); err != nil {
-				errCh <- errors.Wrapf(err, "error on serving metrics via socket %q", addr)
+				errCh <- fmt.Errorf("error on serving metrics via socket %q: %w", addr, err)
 			}
 		}()
 	}
@@ -227,11 +321,11 @@ func serve(ctx context.Context, rpc *grpc.Server, addr string, rs snapshots.Snap
 		log.G(ctx).Infof("listen %q for debugging", config.DebugAddress)
 		l, err := sys.GetLocalListener(config.DebugAddress, 0, 0)
 		if err != nil {
-			return false, errors.Wrapf(err, "failed to listen %q", config.DebugAddress)
+			return false, fmt.Errorf("failed to listen %q: %w", config.DebugAddress, err)
 		}
 		go func() {
 			if err := http.Serve(l, debugServerMux()); err != nil {
-				errCh <- errors.Wrapf(err, "error on serving a debug endpoint via socket %q", addr)
+				errCh <- fmt.Errorf("error on serving a debug endpoint via socket %q: %w", addr, err)
 			}
 		}()
 	}
@@ -239,11 +333,11 @@ func serve(ctx context.Context, rpc *grpc.Server, addr string, rs snapshots.Snap
 	// Listen and serve
 	l, err := net.Listen("unix", addr)
 	if err != nil {
-		return false, errors.Wrapf(err, "error on listen socket %q", addr)
+		return false, fmt.Errorf("error on listen socket %q: %w", addr, err)
 	}
 	go func() {
 		if err := rpc.Serve(l); err != nil {
-			errCh <- errors.Wrapf(err, "error on serving via socket %q", addr)
+			errCh <- fmt.Errorf("error on serving via socket %q: %w", addr, err)
 		}
 	}()
 

cmd/ctr-remote/commands/convert.go

@@ -18,26 +18,32 @@ package commands
 
 import (
 	"compress/gzip"
+	gocontext "context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
+	"os/signal"
 
-	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/images/converter"
-	"github.com/containerd/containerd/images/converter/uncompress"
-	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/containerd/containerd/v2/core/images/converter/uncompress"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	esgzexternaltocconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz/externaltoc"
 	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
 	"github.com/containerd/stargz-snapshotter/recorder"
+	"github.com/klauspost/compress/zstd"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // ConvertCommand converts an image
-var ConvertCommand = cli.Command{
+var ConvertCommand = &cli.Command{
 	Name:      "convert",
 	Usage:     "convert an image",
 	ArgsUsage: "[flags] <source_ref> <target_ref>...",
@@ -50,45 +56,72 @@ When '--all-platforms' is given all images in a manifest list must be available.
 `,
 	Flags: []cli.Flag{
 		// estargz flags
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "estargz",
 			Usage: "convert legacy tar(.gz) layers to eStargz for lazy pulling. Should be used in conjunction with '--oci'",
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:  "estargz-record-in",
 			Usage: "Read 'ctr-remote optimize --record-out=<FILE>' record file",
 		},
-		cli.IntFlag{
+		&cli.IntFlag{
 			Name:  "estargz-compression-level",
 			Usage: "eStargz compression level",
 			Value: gzip.BestCompression,
 		},
-		cli.IntFlag{
+		&cli.IntFlag{
 			Name:  "estargz-chunk-size",
 			Usage: "eStargz chunk size",
 			Value: 0,
 		},
+		&cli.IntFlag{
+			Name:  "estargz-min-chunk-size",
+			Usage: "The minimal number of bytes of data must be written in one gzip stream. Note that this adds a TOC property that old reader doesn't understand.",
+			Value: 0,
+		},
+		&cli.BoolFlag{
+			Name:  "estargz-external-toc",
+			Usage: "Separate TOC JSON into another image (called \"TOC image\"). The name of TOC image is the original + \"-esgztoc\" suffix. Both eStargz and the TOC image should be pushed to the same registry. stargz-snapshotter refers to the TOC image when it pulls the result eStargz image.",
+		},
+		&cli.BoolFlag{
+			Name:  "estargz-keep-diff-id",
+			Usage: "convert to esgz without changing diffID (cannot be used in conjunction with '--estargz-record-in'. must be specified with '--estargz-external-toc')",
+		},
 		// zstd:chunked flags
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "zstdchunked",
 			Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked). Must be used in conjunction with '--oci'.",
 		},
+		&cli.StringFlag{
+			Name:  "zstdchunked-record-in",
+			Usage: "Read 'ctr-remote optimize --record-out=<FILE>' record file",
+		},
+		&cli.IntFlag{
+			Name:  "zstdchunked-compression-level",
+			Usage: "zstd:chunked compression level",
+			Value: 3, // SpeedDefault; see also https://pkg.go.dev/github.com/klauspost/compress/zstd#EncoderLevel
+		},
+		&cli.IntFlag{
+			Name:  "zstdchunked-chunk-size",
+			Usage: "zstd:chunked chunk size",
+			Value: 0,
+		},
 		// generic flags
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "uncompress",
 			Usage: "convert tar.gz layers to uncompressed tar layers",
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "oci",
 			Usage: "convert Docker media types to OCI media types",
 		},
 		// platform flags
-		cli.StringSliceFlag{
+		&cli.StringSliceFlag{
 			Name:  "platform",
 			Usage: "Convert content for a specific platform",
 			Value: &cli.StringSlice{},
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "all-platforms",
 			Usage: "Convert content for all platforms",
 		},
@@ -112,7 +145,7 @@ When '--all-platforms' is given all images in a manifest list must be available.
 				for _, ps := range pss {
 					p, err := platforms.Parse(ps)
 					if err != nil {
-						return errors.Wrapf(err, "invalid platform %q", ps)
+						return fmt.Errorf("invalid platform %q: %w", ps, err)
 					}
 					all = append(all, p)
 				}
@@ -124,14 +157,33 @@ When '--all-platforms' is given all images in a manifest list must be available.
 		convertOpts = append(convertOpts, converter.WithPlatform(platformMC))
 
 		var layerConvertFunc converter.ConvertFunc
+		var finalize func(ctx gocontext.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
 		if context.Bool("estargz") {
 			esgzOpts, err := getESGZConvertOpts(context)
 			if err != nil {
 				return err
 			}
-			layerConvertFunc = estargzconvert.LayerConvertFunc(esgzOpts...)
+			if context.Bool("estargz-external-toc") {
+				if !context.Bool("estargz-keep-diff-id") {
+					layerConvertFunc, finalize = esgzexternaltocconvert.LayerConvertFunc(esgzOpts, context.Int("estargz-compression-level"))
+				} else {
+					if context.String("estargz-record-in") != "" {
+						return fmt.Errorf("option --estargz-keep-diff-id conflicts with --estargz-record-in")
+					}
+					layerConvertFunc, finalize = esgzexternaltocconvert.LayerConvertLossLessFunc(esgzexternaltocconvert.LayerConvertLossLessConfig{
+						CompressionLevel: context.Int("estargz-compression-level"),
+						ChunkSize:        context.Int("estargz-chunk-size"),
+						MinChunkSize:     context.Int("estargz-min-chunk-size"),
+					})
+				}
+			} else {
+				if context.Bool("estargz-keep-diff-id") {
+					return fmt.Errorf("option --estargz-keep-diff-id must be used with --estargz-external-toc")
+				}
+				layerConvertFunc = estargzconvert.LayerConvertFunc(esgzOpts...)
+			}
 			if !context.Bool("oci") {
-				logrus.Warn("option --estargz should be used in conjunction with --oci")
+				log.L.Warn("option --estargz should be used in conjunction with --oci")
 			}
 			if context.Bool("uncompress") {
 				return errors.New("option --estargz conflicts with --uncompress")
@@ -142,11 +194,12 @@ When '--all-platforms' is given all images in a manifest list must be available.
 		}
 
 		if context.Bool("zstdchunked") {
-			esgzOpts, err := getESGZConvertOpts(context)
+			esgzOpts, err := getZstdchunkedConvertOpts(context)
 			if err != nil {
 				return err
 			}
-			layerConvertFunc = zstdchunkedconvert.LayerConvertFunc(esgzOpts...)
+			layerConvertFunc = zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(
+				zstd.EncoderLevelFromZstd(context.Int("zstdchunked-compression-level")), esgzOpts...)
 			if !context.Bool("oci") {
 				return errors.New("option --zstdchunked must be used in conjunction with --oci")
 			}
@@ -174,10 +227,40 @@ When '--all-platforms' is given all images in a manifest list must be available.
 		}
 		defer cancel()
 
+		ctx, done, err := client.WithLease(ctx)
+		if err != nil {
+			return err
+		}
+		defer done(ctx)
+
+		sigCh := make(chan os.Signal, 1)
+		signal.Notify(sigCh, os.Interrupt)
+		go func() {
+			// Cleanly cancel conversion
+			select {
+			case s := <-sigCh:
+				log.G(ctx).Infof("Got %v", s)
+				cancel()
+			case <-ctx.Done():
+			}
+		}()
 		newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
 		if err != nil {
 			return err
 		}
+		if finalize != nil {
+			newI, err := finalize(ctx, client.ContentStore(), targetRef, &newImg.Target)
+			if err != nil {
+				return err
+			}
+			is := client.ImageService()
+			_ = is.Delete(ctx, newI.Name)
+			finimg, err := is.Create(ctx, *newI)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(context.App.Writer, "extra image:", finimg.Name)
+		}
 		fmt.Fprintln(context.App.Writer, newImg.Target.Digest.String())
 		return nil
 	},
@@ -187,6 +270,7 @@ func getESGZConvertOpts(context *cli.Context) ([]estargz.Option, error) {
 	esgzOpts := []estargz.Option{
 		estargz.WithCompressionLevel(context.Int("estargz-compression-level")),
 		estargz.WithChunkSize(context.Int("estargz-chunk-size")),
+		estargz.WithMinChunkSize(context.Int("estargz-min-chunk-size")),
 	}
 	if estargzRecordIn := context.String("estargz-record-in"); estargzRecordIn != "" {
 		paths, err := readPathsFromRecordFile(estargzRecordIn)
@@ -200,6 +284,22 @@ func getESGZConvertOpts(context *cli.Context) ([]estargz.Option, error) {
 	return esgzOpts, nil
 }
 
+func getZstdchunkedConvertOpts(context *cli.Context) ([]estargz.Option, error) {
+	esgzOpts := []estargz.Option{
+		estargz.WithChunkSize(context.Int("zstdchunked-chunk-size")),
+	}
+	if zstdchunkedRecordIn := context.String("zstdchunked-record-in"); zstdchunkedRecordIn != "" {
+		paths, err := readPathsFromRecordFile(zstdchunkedRecordIn)
+		if err != nil {
+			return nil, err
+		}
+		esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths))
+		var ignored []string
+		esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored))
+	}
+	return esgzOpts, nil
+}
+
 func readPathsFromRecordFile(filename string) ([]string, error) {
 	r, err := os.Open(filename)
 	if err != nil {

cmd/ctr-remote/commands/flags.go

@@ -21,108 +21,125 @@ import (
 	gocontext "context"
 	"encoding/csv"
 	"encoding/json"
+	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 
-	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/containers"
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/contrib/nvidia"
-	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/oci"
-	"github.com/containerd/containerd/pkg/netns"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/contrib/nvidia"
+	"github.com/containerd/containerd/v2/core/containers"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/pkg/netns"
+	"github.com/containerd/containerd/v2/pkg/oci"
 	gocni "github.com/containerd/go-cni"
-	"github.com/hashicorp/go-multierror"
+	"github.com/containerd/log"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/pkg/errors"
 	"github.com/rs/xid"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 const netnsMountDir = "/var/run/netns"
 
+func parseGPUs(gpuStr string) ([]int, bool) {
+	if gpuStr == "" {
+		return nil, false
+	}
+	if gpuStr == "all" {
+		return nil, true
+	}
+	parts := strings.Split(gpuStr, ",")
+	var devices []int
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if device, err := strconv.Atoi(part); err == nil {
+			devices = append(devices, device)
+		}
+	}
+	return devices, false
+}
+
 var samplerFlags = []cli.Flag{
-	cli.BoolFlag{
+	&cli.BoolFlag{
 		Name:  "terminal,t",
 		Usage: "enable terminal for sample container. must be specified with i option",
 	},
-	cli.BoolFlag{
+	&cli.BoolFlag{
 		Name:  "i",
 		Usage: "attach stdin to the container",
 	},
-	cli.IntFlag{
+	&cli.IntFlag{
 		Name:  "period",
 		Usage: "time period to monitor access log",
 		Value: defaultPeriod,
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "user",
 		Usage: "user/group name to override image's default config(user[:group])",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "cwd",
 		Usage: "working dir to override image's default config",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "args",
 		Usage: "command arguments to override image's default config(in JSON array)",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "entrypoint",
 		Usage: "entrypoint to override image's default config(in JSON array)",
 	},
-	cli.StringSliceFlag{
+	&cli.StringSliceFlag{
 		Name:  "env",
 		Usage: "environment valulable to add or override to the image's default config",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "env-file",
 		Usage: "specify additional container environment variables in a file(i.e. FOO=bar, one per line)",
 	},
-	cli.StringSliceFlag{
+	&cli.StringSliceFlag{
 		Name:  "mount",
 		Usage: "additional mounts for the container (e.g. type=foo,source=/path,destination=/target,options=bind)",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "dns-nameservers",
 		Usage: "comma-separated nameservers added to the container's /etc/resolv.conf",
 		Value: "8.8.8.8",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "dns-search-domains",
 		Usage: "comma-separated search domains added to the container's /etc/resolv.conf",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "dns-options",
 		Usage: "comma-separated options added to the container's /etc/resolv.conf",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "add-hosts",
 		Usage: "comma-separated hosts configuration (host:IP) added to container's /etc/hosts",
 	},
-	cli.BoolFlag{
+	&cli.BoolFlag{
 		Name:  "cni",
 		Usage: "enable CNI-based networking",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "cni-plugin-conf-dir",
 		Usage: "path to the CNI plugins configuration directory",
 	},
-	cli.StringFlag{
+	&cli.StringFlag{
 		Name:  "cni-plugin-dir",
 		Usage: "path to the CNI plugins binary directory",
 	},
-	cli.IntSliceFlag{
+	&cli.StringFlag{
 		Name:  "gpus",
-		Usage: "add gpus to the container",
+		Usage: "add gpus to the container (comma-separated list of indices or 'all')",
 	},
-	cli.BoolFlag{
+	&cli.BoolFlag{
 		Name:  "net-host",
 		Usage: "enable host networking in the container",
 	},
@@ -131,30 +148,31 @@ var samplerFlags = []cli.Flag{
 func getSpecOpts(clicontext *cli.Context) func(image containerd.Image, rootfs string) (opts []oci.SpecOpts, done func() error, rErr error) {
 	return func(image containerd.Image, rootfs string) (opts []oci.SpecOpts, done func() error, rErr error) {
 		var cleanups []func() error
-		done = func() (allErr error) {
+		done = func() error {
+			var errs []error
 			for i := len(cleanups) - 1; i >= 0; i-- {
 				if err := cleanups[i](); err != nil {
-					allErr = multierror.Append(allErr, err)
+					errs = append(errs, err)
 				}
 			}
-			return
+			return errors.Join(errs...)
 		}
 		defer func() {
 			if rErr != nil {
 				if err := done(); err != nil {
-					rErr = errors.Wrap(rErr, "failed to cleanup")
+					rErr = fmt.Errorf("failed to cleanup: %w", rErr)
 				}
 			}
 		}()
 
 		entrypointOpt, err := withEntrypointArgs(clicontext, image)
 		if err != nil {
-			rErr = errors.Wrapf(err, "failed to parse entrypoint and arg flags")
+			rErr = fmt.Errorf("failed to parse entrypoint and arg flags: %w", err)
 			return
 		}
 		resolverOpt, cleanup, err := withResolveConfig(clicontext)
 		if err != nil {
-			rErr = errors.Wrapf(err, "failed to parse DNS-related flags")
+			rErr = fmt.Errorf("failed to parse DNS-related flags: %w", err)
 			return
 		}
 		cleanups = append(cleanups, cleanup)
@@ -162,7 +180,7 @@ func getSpecOpts(clicontext *cli.Context) func(image containerd.Image, rootfs st
 		for _, mount := range clicontext.StringSlice("mount") {
 			m, err := parseMountFlag(mount)
 			if err != nil {
-				rErr = errors.Wrapf(err, "failed to parse mount flag %q", mount)
+				rErr = fmt.Errorf("failed to parse mount flag %q: %w", mount, err)
 				return
 			}
 			mounts = append(mounts, m)
@@ -197,7 +215,7 @@ func getSpecOpts(clicontext *cli.Context) func(image containerd.Image, rootfs st
 			var nOpt oci.SpecOpts
 			nOpt, cleanup, err = withCNI(clicontext)
 			if err != nil {
-				rErr = errors.Wrapf(err, "failed to parse CNI-related flags")
+				rErr = fmt.Errorf("failed to parse CNI-related flags: %w", err)
 				return
 			}
 			cleanups = append(cleanups, cleanup)
@@ -205,16 +223,21 @@ func getSpecOpts(clicontext *cli.Context) func(image containerd.Image, rootfs st
 		}
 		if clicontext.Bool("net-host") {
 			if runtime.GOOS == "windows" {
-				logrus.Warn("option --net-host is not supported on Windows")
+				log.L.Warn("option --net-host is not supported on Windows")
 			} else {
 				opts = append(opts, oci.WithHostNamespace(runtimespec.NetworkNamespace), oci.WithHostHostsFile, oci.WithHostResolvconf)
 			}
 		}
 		if clicontext.IsSet("gpus") {
 			if runtime.GOOS == "windows" {
-				logrus.Warn("option --gpus is not supported on Windows")
+				log.L.Warn("option --gpus is not supported on Windows")
 			} else {
-				opts = append(opts, nvidia.WithGPUs(nvidia.WithDevices(clicontext.IntSlice("gpus")...), nvidia.WithAllCapabilities))
+				devices, useAll := parseGPUs(clicontext.String("gpus"))
+				if useAll {
+					opts = append(opts, nvidia.WithGPUs(nvidia.WithAllCapabilities))
+				} else if len(devices) > 0 {
+					opts = append(opts, nvidia.WithGPUs(nvidia.WithDevices(devices...), nvidia.WithAllCapabilities))
+				}
 			}
 		}
 
@@ -226,13 +249,13 @@ func withEntrypointArgs(clicontext *cli.Context, image containerd.Image) (oci.Sp
 	var eFlag []string
 	if eStr := clicontext.String("entrypoint"); eStr != "" {
 		if err := json.Unmarshal([]byte(eStr), &eFlag); err != nil {
-			return nil, errors.Wrapf(err, "invalid option \"entrypoint\"")
+			return nil, fmt.Errorf("invalid option \"entrypoint\": %w", err)
 		}
 	}
 	var aFlag []string
 	if aStr := clicontext.String("args"); aStr != "" {
 		if err := json.Unmarshal([]byte(aStr), &aFlag); err != nil {
-			return nil, errors.Wrapf(err, "invalid option \"args\"")
+			return nil, fmt.Errorf("invalid option \"args\": %w", err)
 		}
 	}
 	return func(ctx gocontext.Context, client oci.Client, container *containers.Container, s *runtimespec.Spec) error {
@@ -267,18 +290,19 @@ func withEntrypointArgs(clicontext *cli.Context, image containerd.Image) (oci.Sp
 
 func withCNI(clicontext *cli.Context) (specOpt oci.SpecOpts, done func() error, rErr error) {
 	var cleanups []func() error
-	done = func() (allErr error) {
+	done = func() error {
+		var errs []error
 		for i := len(cleanups) - 1; i >= 0; i-- {
 			if err := cleanups[i](); err != nil {
-				allErr = multierror.Append(allErr, err)
+				errs = append(errs, err)
 			}
 		}
-		return
+		return errors.Join(errs...)
 	}
 	defer func() {
 		if rErr != nil {
 			if err := done(); err != nil {
-				rErr = errors.Wrap(rErr, "failed to cleanup")
+				rErr = fmt.Errorf("failed to cleanup: %w", rErr)
 			}
 		}
 	}()
@@ -286,7 +310,7 @@ func withCNI(clicontext *cli.Context) (specOpt oci.SpecOpts, done func() error,
 	// Create a new network namespace for configuring it with CNI plugins
 	ns, err := netns.NewNetNS(netnsMountDir)
 	if err != nil {
-		rErr = errors.Wrapf(err, "failed to prepare netns")
+		rErr = fmt.Errorf("failed to prepare netns: %w", err)
 		return
 	}
 	cleanups = append(cleanups, ns.Remove)
@@ -304,13 +328,13 @@ func withCNI(clicontext *cli.Context) (specOpt oci.SpecOpts, done func() error,
 	cniopts = append(cniopts, gocni.WithDefaultConf)
 	network, err := gocni.New(cniopts...)
 	if err != nil {
-		rErr = errors.Wrap(err, "failed to prepare CNI plugins")
+		rErr = fmt.Errorf("failed to prepare CNI plugins: %w", err)
 		return
 	}
 	id := xid.New().String()
 	ctx := gocontext.Background()
 	if _, err := network.Setup(ctx, id, ns.GetPath()); err != nil {
-		rErr = errors.Wrap(err, "failed to setup netns with CNI plugins")
+		rErr = fmt.Errorf("failed to setup netns with CNI plugins: %w", err)
 		return
 	}
 	cleanups = append(cleanups, func() error {
@@ -328,7 +352,7 @@ func withResolveConfig(clicontext *cli.Context) (specOpt oci.SpecOpts, cleanup f
 	defer func() {
 		if rErr != nil {
 			if err := cleanup(); err != nil {
-				rErr = errors.Wrap(rErr, "failed to cleanup")
+				rErr = fmt.Errorf("failed to cleanup: %w", rErr)
 			}
 		}
 	}()
@@ -339,7 +363,7 @@ func withResolveConfig(clicontext *cli.Context) (specOpt oci.SpecOpts, cleanup f
 	}
 
 	// Generate /etc/hosts and /etc/resolv.conf
-	resolvDir, err := ioutil.TempDir("", "tmpetc")
+	resolvDir, err := os.MkdirTemp("", "tmpetc")
 	if err != nil {
 		return nil, nil, err
 	}
@@ -351,7 +375,7 @@ func withResolveConfig(clicontext *cli.Context) (specOpt oci.SpecOpts, cleanup f
 	)
 	for _, n := range nameservers {
 		if _, err := fmt.Fprintf(buf, "nameserver %s\n", n); err != nil {
-			rErr = errors.Wrap(err, "failed to prepare nameserver of /etc/resolv.conf")
+			rErr = fmt.Errorf("failed to prepare nameserver of /etc/resolv.conf: %w", err)
 			return
 		}
 	}
@@ -359,19 +383,19 @@ func withResolveConfig(clicontext *cli.Context) (specOpt oci.SpecOpts, cleanup f
 	if len(searches) > 0 {
 		_, err := fmt.Fprintf(buf, "search %s\n", strings.Join(searches, " "))
 		if err != nil {
-			rErr = errors.Wrap(err, "failed to prepare search contents of /etc/resolv.conf")
+			rErr = fmt.Errorf("failed to prepare search contents of /etc/resolv.conf: %w", err)
 			return
 		}
 	}
 	if len(dnsopts) > 0 {
 		_, err := fmt.Fprintf(buf, "options %s\n", strings.Join(dnsopts, " "))
 		if err != nil {
-			rErr = errors.Wrap(err, "failed to prepare options contents of /etc/resolv.conf")
+			rErr = fmt.Errorf("failed to prepare options contents of /etc/resolv.conf: %w", err)
 			return
 		}
 	}
-	if err := ioutil.WriteFile(etcResolvConfPath, buf.Bytes(), 0644); err != nil {
-		rErr = errors.Wrap(err, "failed to write contents to /etc/resolv.conf")
+	if err := os.WriteFile(etcResolvConfPath, buf.Bytes(), 0644); err != nil {
+		rErr = fmt.Errorf("failed to write contents to /etc/resolv.conf: %w", err)
 		return
 	}
 	buf.Reset() // Reusing for /etc/hosts
@@ -389,7 +413,7 @@ func withResolveConfig(clicontext *cli.Context) (specOpt oci.SpecOpts, cleanup f
 		{"ip6-allrouters", "ff02::2"},
 	} {
 		if _, err := fmt.Fprintf(buf, "%s\t%s\n", h.ip, h.host); err != nil {
-			rErr = errors.Wrap(err, "failed to write default hosts to /etc/hosts")
+			rErr = fmt.Errorf("failed to write default hosts to /etc/hosts: %w", err)
 			return
 		}
 	}
@@ -401,12 +425,12 @@ func withResolveConfig(clicontext *cli.Context) (specOpt oci.SpecOpts, cleanup f
 		}
 		// TODO: Validate them
 		if _, err := fmt.Fprintf(buf, "%s\t%s\n", parts[1], parts[0]); err != nil {
-			rErr = errors.Wrap(err, "failed to write extra hosts to /etc/hosts")
+			rErr = fmt.Errorf("failed to write extra hosts to /etc/hosts: %w", err)
 			return
 		}
 	}
-	if err := ioutil.WriteFile(etcHostsPath, buf.Bytes(), 0644); err != nil {
-		rErr = errors.Wrap(err, "failed to write contents to /etc/hosts")
+	if err := os.WriteFile(etcHostsPath, buf.Bytes(), 0644); err != nil {
+		rErr = fmt.Errorf("failed to write contents to /etc/hosts: %w", err)
 		return
 	}
 

cmd/ctr-remote/commands/get-toc-digest.go

@@ -18,31 +18,31 @@ package commands
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 
-	"github.com/containerd/containerd/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containerd/stargz-snapshotter/estargz/zstdchunked"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // GetTOCDigestCommand outputs TOC info of a layer
-var GetTOCDigestCommand = cli.Command{
+var GetTOCDigestCommand = &cli.Command{
 	Name:      "get-toc-digest",
 	Usage:     "get the digest of TOC of a layer",
 	ArgsUsage: "<layer digest>",
 	Flags: []cli.Flag{
 		// zstd:chunked flags
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "zstdchunked",
 			Usage: "parse layer as zstd:chunked",
 		},
 		// other flags for debugging
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "dump-toc",
 			Usage: "dump TOC instead of digest. Note that the dumped TOC might be formatted with indents so may have different digest against the original in the layer",
 		},
@@ -75,7 +75,7 @@ var GetTOCDigestCommand = cli.Command{
 		}
 		footer := make([]byte, footerSize)
 		if _, err := ra.ReadAt(footer, ra.Size()-int64(footerSize)); err != nil {
-			return errors.Wrapf(err, "error reading footer")
+			return fmt.Errorf("error reading footer: %w", err)
 		}
 
 		var decompressor estargz.Decompressor
@@ -86,20 +86,20 @@ var GetTOCDigestCommand = cli.Command{
 
 		_, tocOff, tocSize, err := decompressor.ParseFooter(footer)
 		if err != nil {
-			return errors.Wrapf(err, "error parsing footer")
+			return fmt.Errorf("error parsing footer: %w", err)
 		}
 		if tocSize <= 0 {
 			tocSize = ra.Size() - tocOff - int64(footerSize)
 		}
 		toc, tocDgst, err := decompressor.ParseTOC(io.NewSectionReader(ra, tocOff, tocSize))
 		if err != nil {
-			return errors.Wrapf(err, "error parsing TOC")
+			return fmt.Errorf("error parsing TOC: %w", err)
 		}
 
 		if clicontext.Bool("dump-toc") {
 			tocJSON, err := json.MarshalIndent(toc, "", "\t")
 			if err != nil {
-				return errors.Wrapf(err, "failed to marshal toc")
+				return fmt.Errorf("failed to marshal toc: %w", err)
 			}
 			fmt.Println(string(tocJSON))
 			return nil

cmd/ctr-remote/commands/ipfs-push.go

@@ -17,38 +17,38 @@
 package commands
 
 import (
+	"errors"
 	"fmt"
 
-	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/images/converter"
-	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/ipfs"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
-	httpapi "github.com/ipfs/go-ipfs-http-client"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // IPFSPushCommand pushes an image to IPFS
-var IPFSPushCommand = cli.Command{
+var IPFSPushCommand = &cli.Command{
 	Name:      "ipfs-push",
 	Usage:     "push an image to IPFS (experimental)",
 	ArgsUsage: "[flags] <image_ref>",
 	Flags: []cli.Flag{
 		// platform flags
-		cli.StringSliceFlag{
+		&cli.StringSliceFlag{
 			Name:  "platform",
 			Usage: "Add content for a specific platform",
 			Value: &cli.StringSlice{},
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "all-platforms",
 			Usage: "Add content for all platforms",
 		},
-		cli.BoolTFlag{
+		&cli.BoolFlag{
 			Name:  "estargz",
+			Value: true,
 			Usage: "Convert the image into eStargz",
 		},
 	},
@@ -67,7 +67,7 @@ var IPFSPushCommand = cli.Command{
 				for _, ps := range pss {
 					p, err := platforms.Parse(ps)
 					if err != nil {
-						return errors.Wrapf(err, "invalid platform %q", ps)
+						return fmt.Errorf("invalid platform %q: %w", ps, err)
 					}
 					all = append(all, p)
 				}
@@ -83,21 +83,16 @@ var IPFSPushCommand = cli.Command{
 		}
 		defer cancel()
 
-		ipfsClient, err := httpapi.NewLocalApi()
-		if err != nil {
-			return err
-		}
-
 		var layerConvert converter.ConvertFunc
 		if context.Bool("estargz") {
 			layerConvert = estargzconvert.LayerConvertFunc()
 		}
-		p, err := ipfs.Push(ctx, client, ipfsClient, srcRef, layerConvert, platformMC)
+		p, err := ipfs.Push(ctx, client, srcRef, layerConvert, platformMC)
 		if err != nil {
 			return err
 		}
-		logrus.WithField("CID", p.Cid().String()).Infof("Pushed")
-		fmt.Println(p.Cid().String())
+		log.L.WithField("CID", p).Infof("Pushed")
+		fmt.Println(p)
 
 		return nil
 	},

cmd/ctr-remote/commands/notify.go

@@ -21,11 +21,11 @@ import (
 	"os"
 
 	"github.com/containerd/stargz-snapshotter/analyzer/fanotify/service"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 // FanotifyCommand notifies filesystem event under the specified directory.
-var FanotifyCommand = cli.Command{
+var FanotifyCommand = &cli.Command{
 	Name:   "fanotify",
 	Hidden: true,
 	Action: func(context *cli.Context) error {

cmd/ctr-remote/commands/optimize.go

@@ -20,76 +20,103 @@ import (
 	"compress/gzip"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
+	"os/signal"
 	"time"
 
-	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/images/converter"
-	"github.com/containerd/containerd/platforms"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/analyzer"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containerd/stargz-snapshotter/estargz/zstdchunked"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	esgzexternaltocconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz/externaltoc"
 	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
 	"github.com/containerd/stargz-snapshotter/recorder"
 	"github.com/containerd/stargz-snapshotter/util/containerdutil"
+	"github.com/klauspost/compress/zstd"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 const defaultPeriod = 10
 
 // OptimizeCommand converts and optimizes an image
-var OptimizeCommand = cli.Command{
+var OptimizeCommand = &cli.Command{
 	Name:      "optimize",
 	Usage:     "optimize an image with user-specified workload",
 	ArgsUsage: "[flags] <source_ref> <target_ref>...",
 	Flags: append([]cli.Flag{
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "reuse",
 			Usage: "reuse eStargz (already optimized) layers without further conversion",
 		},
-		cli.StringSliceFlag{
+		&cli.StringSliceFlag{
 			Name:  "platform",
 			Usage: "Pull content from a specific platform",
 			Value: &cli.StringSlice{},
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "all-platforms",
 			Usage: "targeting all platform of the source image",
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "wait-on-signal",
 			Usage: "ignore context cancel and keep the container running until it receives SIGINT (Ctrl + C) sent manually",
 		},
-		cli.BoolFlag{
+		&cli.StringFlag{
+			Name:  "wait-on-line",
+			Usage: "Substring of a stdout line to be waited. When this string is detected, the container will be killed.",
+		},
+		&cli.BoolFlag{
 			Name:  "no-optimize",
 			Usage: "convert image without optimization",
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:  "record-out",
 			Usage: "record the monitor log to the specified file",
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "oci",
 			Usage: "convert Docker media types to OCI media types",
 		},
-		cli.IntFlag{
+		&cli.IntFlag{
 			Name:  "estargz-compression-level",
-			Usage: "eStargz compression level (only applied to gzip as of now)",
+			Usage: "eStargz compression level",
 			Value: gzip.BestCompression,
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
+			Name:  "estargz-external-toc",
+			Usage: "Separate TOC JSON into another image (called \"TOC image\"). The name of TOC image is the original + \"-esgztoc\" suffix. Both eStargz and the TOC image should be pushed to the same registry. stargz-snapshotter refers to the TOC image when it pulls the result eStargz image.",
+		},
+		&cli.IntFlag{
+			Name:  "estargz-chunk-size",
+			Usage: "eStargz chunk size (not applied to zstd:chunked)",
+			Value: 0,
+		},
+		&cli.IntFlag{
+			Name:  "estargz-min-chunk-size",
+			Usage: "The minimal number of bytes of data must be written in one gzip stream. Note that this adds a TOC property that old reader doesn't understand (not applied to zstd:chunked)",
+			Value: 0,
+		},
+		&cli.BoolFlag{
 			Name:  "zstdchunked",
 			Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked)",
 		},
+		&cli.IntFlag{
+			Name:  "zstdchunked-compression-level",
+			Usage: "zstd:chunked compression level",
+			Value: 3, // SpeedDefault; see also https://pkg.go.dev/github.com/klauspost/compress/zstd#EncoderLevel
+		},
 	}, samplerFlags...),
 	Action: func(clicontext *cli.Context) error {
 		convertOpts := []converter.Opt{}
@@ -108,7 +135,7 @@ var OptimizeCommand = cli.Command{
 				for _, ps := range pss {
 					p, err := platforms.Parse(ps)
 					if err != nil {
-						return errors.Wrapf(err, "invalid platform %q", ps)
+						return fmt.Errorf("invalid platform %q: %w", ps, err)
 					}
 					all = append(all, p)
 				}
@@ -143,25 +170,64 @@ var OptimizeCommand = cli.Command{
 		}
 		if recordOutFile := clicontext.String("record-out"); recordOutFile != "" {
 			if err := writeContentFile(ctx, client, recordOut, recordOutFile); err != nil {
-				return errors.Wrapf(err, "failed output record file")
+				return fmt.Errorf("failed output record file: %w", err)
 			}
 		}
 		var f converter.ConvertFunc
+		var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
 		if clicontext.Bool("zstdchunked") {
-			f = zstdchunkedconvert.LayerConvertWithLayerOptsFunc(esgzOptsPerLayer)
-		} else {
+			f = zstdchunkedconvert.LayerConvertWithLayerOptsFuncWithCompressionLevel(
+				zstd.EncoderLevelFromZstd(clicontext.Int("zstdchunked-compression-level")), esgzOptsPerLayer)
+		} else if !clicontext.Bool("estargz-external-toc") {
 			f = estargzconvert.LayerConvertWithLayerAndCommonOptsFunc(esgzOptsPerLayer,
-				estargz.WithCompressionLevel(clicontext.Int("estargz-compression-level")))
+				estargz.WithCompressionLevel(clicontext.Int("estargz-compression-level")),
+				estargz.WithChunkSize(clicontext.Int("estargz-chunk-size")),
+				estargz.WithMinChunkSize(clicontext.Int("estargz-min-chunk-size")))
+		} else {
+			if clicontext.Bool("reuse") {
+				// We require that the layer conversion is triggerd for each layer
+				// to make sure that "finalize" function has the information of all layers.
+				return fmt.Errorf("\"estargz-external-toc\" can't be used with \"reuse\" flag")
+			}
+			f, finalize = esgzexternaltocconvert.LayerConvertWithLayerAndCommonOptsFunc(esgzOptsPerLayer, []estargz.Option{
+				estargz.WithChunkSize(clicontext.Int("estargz-chunk-size")),
+				estargz.WithMinChunkSize(clicontext.Int("estargz-min-chunk-size")),
+			}, clicontext.Int("estargz-compression-level"))
 		}
 		if wrapper != nil {
 			f = wrapper(f)
 		}
 		layerConvertFunc := logWrapper(f)
+
+		sigCh := make(chan os.Signal, 1)
+		signal.Notify(sigCh, os.Interrupt)
+		go func() {
+			// Cleanly cancel conversion
+			select {
+			case s := <-sigCh:
+				log.G(ctx).Infof("Got %v", s)
+				cancel()
+			case <-ctx.Done():
+			}
+		}()
 		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc))
 		newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
 		if err != nil {
 			return err
 		}
+		if finalize != nil {
+			newI, err := finalize(ctx, client.ContentStore(), targetRef, &newImg.Target)
+			if err != nil {
+				return err
+			}
+			is := client.ImageService()
+			_ = is.Delete(ctx, newI.Name)
+			finimg, err := is.Create(ctx, *newI)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(clicontext.App.Writer, "extra image:", finimg.Name)
+		}
 		fmt.Fprintln(clicontext.App.Writer, newImg.Target.Digest.String())
 		return nil
 	},
@@ -194,7 +260,7 @@ func analyze(ctx context.Context, clicontext *cli.Context, client *containerd.Cl
 			for _, ps := range pss {
 				p, err := platforms.Parse(ps)
 				if err != nil {
-					return "", nil, nil, errors.Wrapf(err, "invalid platform %q", ps)
+					return "", nil, nil, fmt.Errorf("invalid platform %q: %w", ps, err)
 				}
 				if platforms.DefaultStrict().Match(p) {
 					containsDefault = true
@@ -218,7 +284,8 @@ func analyze(ctx context.Context, clicontext *cli.Context, client *containerd.Cl
 		aOpts = append(aOpts, analyzer.WithWaitOnSignal())
 	} else {
 		aOpts = append(aOpts,
-			analyzer.WithPeriod(time.Duration(clicontext.Int("period"))*time.Second))
+			analyzer.WithPeriod(time.Duration(clicontext.Int("period"))*time.Second),
+			analyzer.WithWaitLineOut(clicontext.String("wait-on-line")))
 	}
 	if clicontext.Bool("terminal") {
 		if !clicontext.Bool("i") {
@@ -322,7 +389,7 @@ func excludeWrapper(excludes []digest.Digest) func(converter.ConvertFunc) conver
 		return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
 			for _, e := range excludes {
 				if e == desc.Digest {
-					logrus.Warnf("reusing %q without conversion", e)
+					log.G(ctx).Warnf("reusing %q without conversion", e)
 					return nil, nil
 				}
 			}
@@ -333,7 +400,7 @@ func excludeWrapper(excludes []digest.Digest) func(converter.ConvertFunc) conver
 
 func logWrapper(convertFunc converter.ConvertFunc) converter.ConvertFunc {
 	return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
-		logrus.WithField("digest", desc.Digest).Infof("converting...")
+		log.G(ctx).WithField("digest", desc.Digest).Infof("converting...")
 		return convertFunc(ctx, cs, desc)
 	}
 }

cmd/ctr-remote/commands/rpull.go

@@ -20,18 +20,18 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/cmd/ctr/commands/content"
-	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/snapshots"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	ctdsnapshotters "github.com/containerd/containerd/v2/pkg/snapshotters"
+	"github.com/containerd/log"
 	fsconfig "github.com/containerd/stargz-snapshotter/fs/config"
 	"github.com/containerd/stargz-snapshotter/fs/source"
 	"github.com/containerd/stargz-snapshotter/ipfs"
-	httpapi "github.com/ipfs/go-ipfs-http-client"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -39,25 +39,29 @@ const (
 	skipContentVerifyOpt  = "skip-content-verify"
 )
 
-// RpullCommand is a subcommand to pull an image from a registry levaraging stargz snapshotter
-var RpullCommand = cli.Command{
+// RpullCommand is a subcommand to pull an image from a registry leveraging stargz snapshotter
+var RpullCommand = &cli.Command{
 	Name:      "rpull",
-	Usage:     "pull an image from a registry levaraging stargz snapshotter",
+	Usage:     "pull an image from a registry leveraging stargz snapshotter",
 	ArgsUsage: "[flags] <ref>",
-	Description: `Fetch and prepare an image for use in containerd levaraging stargz snapshotter.
+	Description: `Fetch and prepare an image for use in containerd leveraging stargz snapshotter.
 
 After pulling an image, it should be ready to use the same reference in a run
 command. 
 `,
 	Flags: append(append(commands.RegistryFlags, commands.LabelFlag,
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  skipContentVerifyOpt,
 			Usage: "Skip content verification for layers contained in this image.",
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "ipfs",
 			Usage: "Pull image from IPFS. Specify an IPFS CID as a reference. (experimental)",
 		},
+		&cli.BoolFlag{
+			Name:  "use-containerd-labels",
+			Usage: "Use labels defined in containerd project",
+		},
 	), commands.SnapshotterFlags...),
 	Action: func(context *cli.Context) error {
 		var (
@@ -85,17 +89,14 @@ command.
 			return err
 		}
 		config.FetchConfig = fc
+		config.containerdLabels = context.Bool("use-containerd-labels")
 
 		if context.Bool(skipContentVerifyOpt) {
 			config.skipVerify = true
 		}
 
 		if context.Bool("ipfs") {
-			ipfsClient, err := httpapi.NewLocalApi()
-			if err != nil {
-				return err
-			}
-			r, err := ipfs.NewResolver(ipfsClient, ipfs.ResolverOptions{
+			r, err := ipfs.NewResolver(ipfs.ResolverOptions{
 				Scheme: "ipfs",
 			})
 			if err != nil {
@@ -114,8 +115,9 @@ command.
 
 type rPullConfig struct {
 	*content.FetchConfig
-	skipVerify  bool
-	snapshotter string
+	skipVerify       bool
+	snapshotter      string
+	containerdLabels bool
 }
 
 func pull(ctx context.Context, client *containerd.Client, ref string, config *rPullConfig) error {
@@ -135,16 +137,23 @@ func pull(ctx context.Context, client *containerd.Client, ref string, config *rP
 		}))
 	}
 
+	var labelHandler func(h images.Handler) images.Handler
+	prefetchSize := int64(10 * 1024 * 1024)
+	if config.containerdLabels {
+		labelHandler = source.AppendExtraLabelsHandler(prefetchSize, ctdsnapshotters.AppendInfoHandlerWrapper(ref))
+	} else {
+		labelHandler = source.AppendDefaultLabelsHandlerWrapper(ref, prefetchSize)
+	}
+
 	log.G(pCtx).WithField("image", ref).Debug("fetching")
 	labels := commands.LabelArgs(config.Labels)
 	if _, err := client.Pull(pCtx, ref, []containerd.RemoteOpt{
 		containerd.WithPullLabels(labels),
 		containerd.WithResolver(config.Resolver),
 		containerd.WithImageHandler(h),
-		containerd.WithSchema1Conversion,
 		containerd.WithPullUnpack,
 		containerd.WithPullSnapshotter(config.snapshotter, snOpts...),
-		containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(ref, 10*1024*1024)),
+		containerd.WithImageHandlerWrapper(labelHandler),
 	}...); err != nil {
 		return err
 	}

cmd/ctr-remote/main.go

@@ -20,18 +20,13 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/containerd/containerd/cmd/ctr/app"
-	"github.com/containerd/containerd/pkg/seed"
+	"github.com/containerd/containerd/v2/cmd/ctr/app"
 	"github.com/containerd/stargz-snapshotter/cmd/ctr-remote/commands"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-func init() {
-	seed.WithTimeAndRand()
-}
-
 func main() {
-	customCommands := []cli.Command{
+	customCommands := []*cli.Command{
 		commands.RpullCommand,
 		commands.OptimizeCommand,
 		commands.ConvertCommand,
@@ -41,7 +36,7 @@ func main() {
 	app := app.New()
 	for i := range app.Commands {
 		if app.Commands[i].Name == "images" {
-			sc := map[string]cli.Command{}
+			sc := map[string]*cli.Command{}
 			for _, subcmd := range customCommands {
 				sc[subcmd.Name] = subcmd
 			}

cmd/go.mod

@@ -1,30 +1,154 @@
 module github.com/containerd/stargz-snapshotter/cmd
 
-go 1.16
+go 1.24.0
+
+toolchain go1.24.2
 
 require (
-	github.com/containerd/containerd v1.6.0-beta.1.0.20211101005050-f0d3ea96cf8c
-	github.com/containerd/containerd/api v1.6.0-beta.1.0.20211101005050-f0d3ea96cf8c
-	github.com/containerd/go-cni v1.1.0
-	github.com/containerd/stargz-snapshotter v0.10.0
-	github.com/containerd/stargz-snapshotter/estargz v0.10.0
-	github.com/containerd/stargz-snapshotter/ipfs v0.10.0
-	github.com/coreos/go-systemd/v22 v22.3.2
+	github.com/containerd/containerd/api v1.9.0
+	github.com/containerd/containerd/v2 v2.1.4
+	github.com/containerd/go-cni v1.1.13
+	github.com/containerd/log v0.1.0
+	github.com/containerd/platforms v1.0.0-rc.1
+	github.com/containerd/stargz-snapshotter v0.15.2-0.20240622031358-6405f362966d
+	github.com/containerd/stargz-snapshotter/estargz v0.17.0
+	github.com/containerd/stargz-snapshotter/ipfs v0.15.2-0.20240622031358-6405f362966d
+	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/docker/go-metrics v0.0.1
-	github.com/hashicorp/go-multierror v1.1.1
-	github.com/ipfs/go-ipfs-http-client v0.1.0
-	github.com/ipfs/interface-go-ipfs-core v0.5.2
+	github.com/goccy/go-json v0.10.5
+	github.com/klauspost/compress v1.18.0
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
-	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
-	github.com/pelletier/go-toml v1.9.4
-	github.com/pkg/errors v0.9.1
-	github.com/rs/xid v1.3.0
-	github.com/sirupsen/logrus v1.8.1
-	github.com/urfave/cli v1.22.4
-	golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
-	google.golang.org/grpc v1.42.0
-	k8s.io/cri-api v0.22.3
+	github.com/opencontainers/image-spec v1.1.1
+	github.com/opencontainers/runtime-spec v1.2.1
+	github.com/pelletier/go-toml v1.9.5
+	github.com/rs/xid v1.6.0
+	github.com/urfave/cli/v2 v2.27.7
+	go.etcd.io/bbolt v1.4.2
+	golang.org/x/sync v0.16.0
+	golang.org/x/sys v0.34.0
+	google.golang.org/grpc v1.74.2
+)
+
+require (
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.13.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cilium/ebpf v0.16.0 // indirect
+	github.com/containerd/cgroups/v3 v3.0.5 // indirect
+	github.com/containerd/console v1.0.5 // indirect
+	github.com/containerd/continuity v0.4.5 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/go-runc v1.1.0 // indirect
+	github.com/containerd/plugin v1.0.0 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
+	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/containernetworking/cni v1.3.0 // indirect
+	github.com/containernetworking/plugins v1.7.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/cli v28.3.3+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hanwen/go-fuse/v2 v2.8.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
+	github.com/intel/goresctrl v0.8.0 // indirect
+	github.com/ipfs/go-cid v0.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/mdlayher/vsock v1.2.1 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/symlink v0.3.0 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr v0.16.1 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opencontainers/selinux v1.12.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.23.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sasha-s/go-deadlock v0.3.5 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/vbatts/tar-split v0.12.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
+	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
+	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.33.3 // indirect
+	k8s.io/apimachinery v0.33.3 // indirect
+	k8s.io/client-go v0.33.3 // indirect
+	k8s.io/cri-api v0.33.3 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+	tags.cncf.io/container-device-interface v1.0.1 // indirect
+	tags.cncf.io/container-device-interface/specs-go v1.0.0 // indirect
 )
 
 replace (
@@ -32,7 +156,4 @@ replace (
 	github.com/containerd/stargz-snapshotter => ../
 	github.com/containerd/stargz-snapshotter/estargz => ../estargz
 	github.com/containerd/stargz-snapshotter/ipfs => ../ipfs
-
-	// Temporary fork for avoiding importing patent-protected code: https://github.com/hashicorp/golang-lru/issues/73
-	github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c
 )

cmd/stargz-fuse-manager/main.go

@@ -0,0 +1,97 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/log"
+
+	"github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/fsopts"
+	fusemanager "github.com/containerd/stargz-snapshotter/fusemanager"
+	"github.com/containerd/stargz-snapshotter/service"
+	"github.com/containerd/stargz-snapshotter/service/keychain/keychainconfig"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	fusemanager.RegisterConfigFunc(func(cc *fusemanager.ConfigContext) ([]service.Option, error) {
+		fsConfig := fsopts.Config{
+			EnableIpfs:    cc.Config.IPFS,
+			MetadataStore: cc.Config.MetadataStore,
+			OpenBoltDB:    cc.OpenBoltDB,
+		}
+		fsOpts, err := fsopts.ConfigFsOpts(cc.Ctx, cc.RootDir, &fsConfig)
+		if err != nil {
+			return nil, err
+		}
+		return []service.Option{service.WithFilesystemOptions(fsOpts...)}, nil
+	})
+
+	fusemanager.RegisterConfigFunc(func(cc *fusemanager.ConfigContext) ([]service.Option, error) {
+		keyChainConfig := keychainconfig.Config{
+			EnableKubeKeychain:         cc.Config.Config.KubeconfigKeychainConfig.EnableKeychain,
+			EnableCRIKeychain:          cc.Config.Config.CRIKeychainConfig.EnableKeychain,
+			KubeconfigPath:             cc.Config.Config.KubeconfigPath,
+			DefaultImageServiceAddress: cc.Config.DefaultImageServiceAddress,
+			ImageServicePath:           cc.Config.Config.ImageServicePath,
+		}
+		if cc.Config.Config.CRIKeychainConfig.EnableKeychain && cc.Config.Config.ListenPath == "" || cc.Config.Config.ListenPath == cc.Address {
+			return nil, fmt.Errorf("listen path of CRI server must be specified as a separated socket from FUSE manager server")
+		}
+		// For CRI keychain, if listening path is different from stargz-snapshotter's socket, prepare for the dedicated grpc server and the socket.
+		serveCRISocket := cc.Config.Config.CRIKeychainConfig.EnableKeychain && cc.Config.Config.ListenPath != "" && cc.Config.Config.ListenPath != cc.Address
+		if serveCRISocket {
+			cc.CRIServer = grpc.NewServer()
+		}
+		credsFuncs, err := keychainconfig.ConfigKeychain(cc.Ctx, cc.CRIServer, &keyChainConfig)
+		if err != nil {
+			return nil, err
+		}
+		if serveCRISocket {
+			addr := cc.Config.Config.ListenPath
+			// Prepare the directory for the socket
+			if err := os.MkdirAll(filepath.Dir(addr), 0700); err != nil {
+				return nil, fmt.Errorf("failed to create directory %q: %w", filepath.Dir(addr), err)
+			}
+
+			// Try to remove the socket file to avoid EADDRINUSE
+			if err := os.RemoveAll(addr); err != nil {
+				return nil, fmt.Errorf("failed to remove %q: %w", addr, err)
+			}
+
+			// Listen and serve
+			l, err := net.Listen("unix", addr)
+			if err != nil {
+				return nil, fmt.Errorf("error on listen socket %q: %w", addr, err)
+			}
+			go func() {
+				if err := cc.CRIServer.Serve(l); err != nil {
+					log.G(cc.Ctx).WithError(err).Errorf("error on serving CRI via socket %q", addr)
+				}
+			}()
+		}
+		return []service.Option{service.WithCredsFuncs(credsFuncs...)}, nil
+	})
+}
+
+func main() {
+	fusemanager.Run()
+}

cmd/stargz-store/helper/main.go

@@ -0,0 +1,66 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+	"time"
+
+	"github.com/containerd/containerd/v2/defaults"
+	"github.com/containerd/containerd/v2/pkg/dialer"
+	"github.com/containerd/stargz-snapshotter/store/pb"
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func main() {
+	var addr = "/var/lib/stargz-store/store.sock" // default
+	if len(os.Args) >= 2 {
+		addr = os.Args[1]
+	}
+	data, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		panic(err)
+	}
+
+	backoffConfig := backoff.DefaultConfig
+	backoffConfig.MaxDelay = 3 * time.Second
+	connParams := grpc.ConnectParams{
+		Backoff: backoffConfig,
+	}
+	gopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithConnectParams(connParams),
+		grpc.WithContextDialer(dialer.ContextDialer),
+		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
+		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
+	}
+	conn, err := grpc.NewClient(dialer.DialAddress(addr), gopts...)
+	if err != nil {
+		panic(err)
+	}
+	c := pb.NewControllerClient(conn)
+	_, err = c.AddCredential(context.Background(), &pb.AddCredentialRequest{
+		Data: data,
+	})
+	if err != nil {
+		panic(err)
+	}
+}

cmd/stargz-store/main.go

@@ -17,28 +17,41 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"errors"
 	"flag"
+	"fmt"
+	"io"
 	golog "log"
 	"math/rand"
+	"net"
 	"os"
 	"os/signal"
+	"path/filepath"
+	"sync"
 	"syscall"
 	"time"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/log"
+	dbmetadata "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/db"
 	"github.com/containerd/stargz-snapshotter/fs/config"
-	"github.com/containerd/stargz-snapshotter/service/keychain/dockerconfig"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
 	"github.com/containerd/stargz-snapshotter/service/keychain/kubeconfig"
 	"github.com/containerd/stargz-snapshotter/service/resolver"
 	"github.com/containerd/stargz-snapshotter/store"
+	"github.com/containerd/stargz-snapshotter/store/pb"
 	sddaemon "github.com/coreos/go-systemd/v22/daemon"
 	"github.com/pelletier/go-toml"
-	"github.com/sirupsen/logrus"
+	bolt "go.etcd.io/bbolt"
+	grpc "google.golang.org/grpc"
 )
 
 const (
-	defaultLogLevel   = logrus.InfoLevel
+	defaultLogLevel   = log.InfoLevel
 	defaultConfigPath = "/etc/stargz-store/config.toml"
 	defaultRootDir    = "/var/lib/stargz-store"
 )
@@ -47,6 +60,7 @@ var (
 	configPath = flag.String("config", defaultConfigPath, "path to the configuration file")
 	logLevel   = flag.String("log-level", defaultLogLevel.String(), "set the logging level [trace, debug, info, warn, error, fatal, panic]")
 	rootDir    = flag.String("root", defaultRootDir, "path to the root directory for this snapshotter")
+	listenaddr = flag.String("addr", filepath.Join(defaultRootDir, "store.sock"), "path to the socket listened by this snapshotter")
 )
 
 type Config struct {
@@ -57,6 +71,9 @@ type Config struct {
 
 	// ResolverConfig is config for resolving registries.
 	ResolverConfig `toml:"resolver"`
+
+	// MetadataStore is the type of the metadata store to use.
+	MetadataStore string `toml:"metadata_store" default:"memory"`
 }
 
 type KubeconfigKeychainConfig struct {
@@ -67,25 +84,22 @@ type KubeconfigKeychainConfig struct {
 type ResolverConfig resolver.Config
 
 func main() {
-	rand.Seed(time.Now().UnixNano())
+	rand.Seed(time.Now().UnixNano()) //nolint:staticcheck // Global math/rand seed is deprecated, but still used by external dependencies
 	flag.Parse()
 	mountPoint := flag.Arg(0)
-	lvl, err := logrus.ParseLevel(*logLevel)
+	err := log.SetLevel(*logLevel)
 	if err != nil {
 		log.L.WithError(err).Fatal("failed to prepare logger")
 	}
-	logrus.SetLevel(lvl)
-	logrus.SetFormatter(&logrus.JSONFormatter{
-		TimestampFormat: log.RFC3339NanoFixed,
-	})
+	log.SetFormat(log.JSONFormat)
 	var (
 		ctx    = log.WithLogger(context.Background(), log.L)
 		config Config
 	)
 	// Streams log of standard lib (go-fuse uses this) into debug log
-	// Snapshotter should use "github.com/containerd/containerd/log" otherwize
+	// Snapshotter should use "github.com/containerd/log" otherwise
 	// logs are always printed as "debug" mode.
-	golog.SetOutput(log.G(ctx).WriterLevel(logrus.DebugLevel))
+	golog.SetOutput(log.G(ctx).WriterLevel(log.DebugLevel))
 
 	if mountPoint == "" {
 		log.G(ctx).Fatalf("mount point must be specified")
@@ -94,7 +108,7 @@ func main() {
 	// Get configuration from specified file
 	if *configPath != "" {
 		tree, err := toml.LoadFile(*configPath)
-		if err != nil && !(os.IsNotExist(err) && *configPath == defaultConfigPath) {
+		if err != nil && (!os.IsNotExist(err) || *configPath != defaultConfigPath) {
 			log.G(ctx).WithError(err).Fatalf("failed to load config file %q", *configPath)
 		}
 		if err := tree.Unmarshal(&config); err != nil {
@@ -102,11 +116,15 @@ func main() {
 		}
 	}
 
+	sk := new(storeKeychain)
+
+	errCh := serveController(*listenaddr, sk)
+
 	// Prepare kubeconfig-based keychain if required
-	credsFuncs := []resolver.Credential{dockerconfig.NewDockerconfigKeychain(ctx)}
-	if config.KubeconfigKeychainConfig.EnableKeychain {
+	credsFuncs := []resolver.Credential{sk.credentials}
+	if config.EnableKeychain {
 		var opts []kubeconfig.Option
-		if kcp := config.KubeconfigKeychainConfig.KubeconfigPath; kcp != "" {
+		if kcp := config.KubeconfigPath; kcp != "" {
 			opts = append(opts, kubeconfig.WithKubeconfigPath(kcp))
 		}
 		credsFuncs = append(credsFuncs, kubeconfig.NewKubeconfigKeychain(ctx, opts...))
@@ -122,15 +140,18 @@ func main() {
 				Fatalf("failed to prepare mountpoint %q", mountPoint)
 		}
 	}
-	if !config.Config.DisableVerification {
-		log.G(ctx).Warnf("content verification is not supported; switching to non-verification mode")
-		config.Config.DisableVerification = true
+	if config.DisableVerification {
+		log.G(ctx).Fatalf("content verification can't be disabled")
 	}
-	layerManager, err := store.NewLayerManager(ctx, *rootDir, hosts, config.Config)
+	mt, err := getMetadataStore(*rootDir, config)
+	if err != nil {
+		log.G(ctx).WithError(err).Fatalf("failed to configure metadata store")
+	}
+	layerManager, err := store.NewLayerManager(ctx, *rootDir, hosts, mt, config.Config)
 	if err != nil {
 		log.G(ctx).WithError(err).Fatalf("failed to prepare pool")
 	}
-	if err := store.Mount(ctx, mountPoint, layerManager, config.Config.Debug); err != nil {
+	if err := store.Mount(ctx, mountPoint, layerManager, config.Debug); err != nil {
 		log.G(ctx).WithError(err).Fatalf("failed to mount fs at %q", mountPoint)
 	}
 	defer func() {
@@ -149,12 +170,125 @@ func main() {
 		}
 	}()
 
-	waitForSIGINT()
-	log.G(ctx).Info("Got SIGINT")
+	if err := waitForSignal(ctx, errCh); err != nil {
+		log.G(ctx).Errorf("error: %v", err)
+		os.Exit(1)
+	}
 }
 
-func waitForSIGINT() {
+func waitForSignal(ctx context.Context, errCh <-chan error) error {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt)
-	<-c
+	select {
+	case s := <-c:
+		log.G(ctx).Infof("Got %v", s)
+	case err := <-errCh:
+		return err
+	}
+	return nil
+}
+
+const (
+	memoryMetadataType = "memory"
+	dbMetadataType     = "db"
+)
+
+func getMetadataStore(rootDir string, config Config) (metadata.Store, error) {
+	switch config.MetadataStore {
+	case "", memoryMetadataType:
+		return memorymetadata.NewReader, nil
+	case dbMetadataType:
+		bOpts := bolt.Options{
+			NoFreelistSync:  true,
+			InitialMmapSize: 64 * 1024 * 1024,
+			FreelistType:    bolt.FreelistMapType,
+		}
+		db, err := bolt.Open(filepath.Join(rootDir, "metadata.db"), 0600, &bOpts)
+		if err != nil {
+			return nil, err
+		}
+		return func(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error) {
+			return dbmetadata.NewReader(db, sr, opts...)
+		}, nil
+	default:
+		return nil, fmt.Errorf("unknown metadata store type: %v; must be %v or %v",
+			config.MetadataStore, memoryMetadataType, dbMetadataType)
+	}
+}
+
+func newController(addCredentialFunc func(data []byte) error) *controller {
+	return &controller{
+		addCredentialFunc: addCredentialFunc,
+	}
+}
+
+type controller struct {
+	addCredentialFunc func(data []byte) error
+}
+
+func (c *controller) AddCredential(ctx context.Context, req *pb.AddCredentialRequest) (resp *pb.AddCredentialResponse, _ error) {
+	return &pb.AddCredentialResponse{}, c.addCredentialFunc(req.Data)
+}
+
+type authConfig struct {
+	Username      string `json:"username,omitempty"`
+	Password      string `json:"password,omitempty"`
+	IdentityToken string `json:"identityToken,omitempty"`
+}
+
+type storeKeychain struct {
+	config   map[string]authConfig
+	configMu sync.Mutex
+}
+
+func (sk *storeKeychain) add(data []byte) error {
+	conf := make(map[string]authConfig)
+	if err := json.NewDecoder(bytes.NewReader(data)).Decode(&conf); err != nil && !errors.Is(err, io.EOF) {
+		return err
+	}
+	sk.configMu.Lock()
+	if sk.config == nil {
+		sk.config = make(map[string]authConfig)
+	}
+	for k, c := range conf {
+		sk.config[k] = c
+	}
+	sk.configMu.Unlock()
+	return nil
+}
+
+func (sk *storeKeychain) credentials(host string, refspec reference.Spec) (string, string, error) {
+	if host != refspec.Hostname() {
+		return "", "", nil // Do not use creds for mirrors
+	}
+	sk.configMu.Lock()
+	defer sk.configMu.Unlock()
+	if acfg, ok := sk.config[refspec.String()]; ok {
+		if acfg.IdentityToken != "" {
+			return "", acfg.IdentityToken, nil
+		} else if acfg.Username != "" || acfg.Password != "" {
+			return acfg.Username, acfg.Password, nil
+		}
+	}
+	return "", "", nil
+}
+
+func serveController(addr string, sk *storeKeychain) <-chan error {
+	// Try to remove the socket file to avoid EADDRINUSE
+	os.Remove(addr)
+	rpc := grpc.NewServer()
+	c := newController(sk.add)
+	pb.RegisterControllerServer(rpc, c)
+	errCh := make(chan error, 1)
+	go func() {
+		l, err := net.Listen("unix", addr)
+		if err != nil {
+			errCh <- fmt.Errorf("error on listen socket %q: %w", addr, err)
+			return
+		}
+		if err := rpc.Serve(l); err != nil {
+			errCh <- fmt.Errorf("error on serving via socket %q: %w", addr, err)
+		}
+	}()
+	return errCh
 }

docs/INSTALL.md

@@ -44,6 +44,8 @@ We assume that you are using containerd (> v1.4.2) as a CRI runtime.
     [proxy_plugins.stargz]
       type = "snapshot"
       address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+    [proxy_plugins.stargz.exports]
+      root = "/var/lib/containerd-stargz-grpc/"
 
   ```
 
@@ -119,3 +121,64 @@ We assume that you are using CRI-O newer than https://github.com/cri-o/cri-o/pul
   systemctl enable --now stargz-store
   systemctl restart cri-o # if you are using CRI-O
   ```
+
+## Install Stargz Snapshotter for Docker(Moby) with Systemd
+
+- Docker(Moby) newer than [`5c1d6c957b97321c8577e10ddbffe6e01981617a`](https://github.com/moby/moby/commit/5c1d6c957b97321c8577e10ddbffe6e01981617a) is needed on your host. The commit is expected to be included in Docker v24.
+
+- Download stargz-snapshotter release tarball from [the release page](https://github.com/containerd/stargz-snapshotter/releases).
+
+- Enable `containerd-snapshotter` feature and `stargz` snapshotter in Docker. Add the following to docker's configuration file (typically: /etc/docker/daemon.json).
+  ```json
+  {
+    "features": {
+      "containerd-snapshotter": true
+    },
+    "storage-driver": "stargz"
+  }
+  ```
+
+- Enable stargz snapshotter in containerd. Add the following configuration to containerd's configuration file (typically: /etc/containerd/config.toml).
+  ```toml
+  version = 2
+
+  # Plug stargz snapshotter into containerd
+  [proxy_plugins]
+    [proxy_plugins.stargz]
+      type = "snapshot"
+      address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+    [proxy_plugins.stargz.exports]
+      root = "/var/lib/containerd-stargz-grpc/"
+  ```
+
+- Install fuse
+
+  ###### centos
+  ```
+  # centos 7
+  yum install fuse
+  # centos 8
+  dnf install fuse
+
+  modprobe fuse
+  ```
+
+  ###### ubuntu
+
+  ```
+  apt-get install fuse
+  modprobe fuse
+  ```
+
+- Start stargz-snapshotter and restart containerd and docker
+  ```
+  tar -C /usr/local/bin -xvf stargz-snapshotter-${version}-linux-${arch}.tar.gz containerd-stargz-grpc ctr-remote
+  wget -O /etc/systemd/system/stargz-snapshotter.service https://raw.githubusercontent.com/containerd/stargz-snapshotter/${version}/script/config/etc/systemd/system/stargz-snapshotter.service
+  systemctl enable --now stargz-snapshotter
+  systemctl restart containerd
+  systemctl restart docker
+  ```
+
+## Using stargz-snapshotter on Lima
+
+See [`./lima.md`](./lima.md)

docs/ctr-remote.md

@@ -20,7 +20,9 @@ This optimization is done by baking the information about files that are likely
 On runtime, Stargz Snapshotter prefetches these prioritized files before mounting the layer for making sure these files are locally accessible.
 This can avoid downloading chunks on every file read and mitigate the runtime performance drawbacks.
 
-For more details about eStargz and its optimization, refer also to [eStargz: Standard-Compatible Extensions to Tar.gz Layers for Lazy Pulling Container Images](/docs/stargz-estargz.md).
+:information_source: For more details about eStargz and its optimization, refer also to [eStargz: Standard-Compatible Extensions to Tar.gz Layers for Lazy Pulling Container Images](/docs/stargz-estargz.md).
+
+:information_source: Please see also [Creating smaller eStargz images](/docs/smaller-estargz.md) if you're interested in creating a smaller size of eStargz images.
 
 ## Requirements
 
@@ -72,10 +74,11 @@ You can enable host networking for the container using the `net-host` flag.
 # ctr-remote i optimize -t -i --oci --entrypoint='[ "/bin/bash", "-c" ]' --net-host --args='[ "ip a && curl example.com" ]' ghcr.io/stargz-containers/centos:8-test registry2:5000/centos:8-test-esgz
 ```
 
-You can optimize GPU-based images using the `gpu` flag. The flag expects a comma separated list of integers.
+You can optimize GPU-based images using the `gpu` flag. The flag expects a comma separated list of integers or 'all'.
 
 ```console
 # ctr-remote i optimize --oci --gpus "0" <src> <target>
+# ctr-remote i optimize --oci --gpus "all" <src> <target>
 ```
 
 `--oci` option is highly recommended to add when you create eStargz image.
@@ -266,3 +269,38 @@ ctr-remote image optimize --oci \
 By default, when the source image is a multi-platform image, `ctr-remote` converts the image corresponding to the platform where `ctr-remote` runs.
 
 Note that though the images specified by `--all-platform` and `--platform` are converted to eStargz, images that don't correspond to the current platform aren't *optimized*. That is, these images are lazily pulled but without prefetch.
+
+### Dump log of accessed files during optimization (`--record-out`)
+
+You can dump the information of which files are accesssed during optimization, using `--record-out` flag.
+
+For example, the following dumps logs of files accessed during running `ls` in `ubuntu:24.04`.
+
+```
+ctr-remote image pull docker.io/library/ubuntu:24.04
+ctr-remote image optimize --record-out=/tmp/log.json \
+  --entrypoint='[ "/bin/bash", "-c" ]' --args='[ "ls" ]' \
+  docker.io/library/ubuntu:24.04 registry2:5000/ubuntu:24.04
+```
+
+The following is the contents of the log (`/tmp/log.json`):
+
+```
+{"path":"usr/bin/bash","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/bin/bash","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/ld.so.cache","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libtinfo.so.6.4","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libc.so.6","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/nsswitch.conf","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/nsswitch.conf","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/passwd","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/bin/ls","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/ld.so.cache","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libselinux.so.1","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libc.so.6","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.11.2","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+```
+
+For creating an optimized eStargz using this log, you can input this log into [`--estargz-record-in` or `--zstdchunked-record-in` of `nerdctl image convert`](https://github.com/containerd/nerdctl/blob/8b814ca7fe29cb505a02a3d85ba22860e63d15bf/docs/command-reference.md#nerd_face-nerdctl-image-convert) or the same flags for `ctr-remote image convert` .

docs/estargz.md

@@ -177,6 +177,22 @@ Properties other than `chunkDigest` are inherited from [stargz](https://github.c
   TOCEntries of non-empty `reg` and `chunk` MUST set this property.
   This MAY be used for verifying the data of the chunk.
 
+- **`innerOffset`** *int64*
+
+  This OPTIONAL property indicates the uncompressed offset of the "reg" or "chunk" entry payload in a stream starts from `offset` field.
+
+#### Details about `innerOffset`
+
+`innerOffset` enables to put multiple "reg" or "chunk" payloads in one gzip stream starts from `offset`.
+This field allows the following structure.
+
+![The structure of eStargz with innerOffset](/docs/images/estargz-inneroffset.png)
+
+Use case of this field is `--estargz-min-chunk-size` flag of `ctr-remote`.
+The value of this flag is the minimal number of bytes of data must be written in one gzip stream.
+If it's > 0, multiple files and chunks can be written into one gzip stream.
+Smaller number of gzip header and smaller size of the result blob can be expected.
+
 ### Footer
 
 At the end of the blob, a *footer* MUST be appended.
@@ -198,7 +214,7 @@ Runtimes MAY first read and parse the footer to get the offset of TOC.
 
 Each file's metadata is recorded in the TOC so runtimes don't need to extract other parts of the archive as long as it only uses file metadata.
 If runtime needs to get a regular file's content, it can get the size and offset of that content from the TOC and extract that range without scanning the entire blob.
-By combining this with HTTP Range Request supported by [OCI Distribution Spec](https://github.com/opencontainers/distribution-spec/blob/main/detail.md#fetch-blob-part), runtimes can selectively download file entries from the registry.
+By combining this with HTTP Range Request supported by [OCI Distribution Spec](https://github.com/opencontainers/distribution-spec/blob/ef28f81727c3b5e98ab941ae050098ea664c0960/detail.md#fetch-blob-part), runtimes can selectively download file entries from the registry.
 
 ### Notes on compatibility with stargz
 
@@ -294,6 +310,61 @@ After the TOC is verified, the snapshotter mounts this layer using the metadata
 During runtime of the container, this snapshotter fetches chunks of regular file contents lazily.
 Before providing a chunk to the filesystem user, snapshotter recalculates the digest and checks it matches the one recorded in the corresponding TOCEntry.
 
+## eStargz image with an external TOC (OPTIONAL)
+
+This OPTIONAL feature allows separating TOC into another image called *TOC image*.
+This type of eStargz is the same as the normal eStargz but doesn't contain TOC JSON file (`stargz.index.json`) in the layer blob and has a special footer.
+This feature enables creating a smaller eStargz blob by avoiding including TOC JSON file in that blob.
+
+Footer has the following structure:
+
+```
+// The footer is an empty gzip stream with no compression and an Extra header.
+//
+// 46 comes from:
+//
+// 10 bytes  gzip header
+// 2  bytes  XLEN (length of Extra field) = 21 (4 bytes header + len("STARGZEXTERNALTOC"))
+// 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
+// 2  bytes  Extra: LEN = 17 (len("STARGZEXTERNALTOC"))
+// 17 bytes  Extra: subfield = "STARGZEXTERNALTOC"
+// 5  bytes  flate header
+// 8  bytes  gzip footer
+// (End of the eStargz blob)
+```
+
+TOC image is an OCI image containing TOC.
+Each layer contains a TOC JSON file (`stargz.index.json`) in the root directory.
+
+Layer descriptors in the manifest must contain an annotation `containerd.io/snapshot/stargz/layer.digest`.
+The value of this annotation is the digest of the eStargz layer blob corresponding to that TOC.
+
+The following is an example layer descriptor in the TOC image.
+This layer (`sha256:64dedefd539280a5578c8b94bae6f7b4ebdbd12cb7a7df0770c4887a53d9af70`) contains the TOC JSON file (`stargz.index.json`) in the root directory and can be used for eStargz layer blob that has the digest `sha256:5da5601c1f2024c07f580c11b2eccf490cd499473883a113c376d64b9b10558f`.
+
+```json
+{
+  "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+  "digest": "sha256:64dedefd539280a5578c8b94bae6f7b4ebdbd12cb7a7df0770c4887a53d9af70",
+  "size": 154425,
+  "annotations": {
+    "containerd.io/snapshot/stargz/layer.digest": "sha256:5da5601c1f2024c07f580c11b2eccf490cd499473883a113c376d64b9b10558f"
+  }
+}
+```
+
+### Example usecase: lazy pulling with Stargz Snapshotter
+
+Stargz Snapshotter supports eStargz with external TOC.
+If an eStargz blob's footer indicates that it requires the TOC image, stargz snapshotter also pulls it from the registry.
+
+Stargz snapshotter assumes the TOC image has the reference name same as the eStargz with `-esgztoc` suffix.
+For example, if an eStargz image is named `ghcr.io/stargz-containers/ubuntu:22.04-esgz`, stargz snapshotter acquires the TOC image from `ghcr.io/stargz-containers/ubuntu:22.04-esgz-esgztoc`.
+Note that future versions of stargz snapshotter will support more ways to search the TOC image (e.g. allowing custom suffix, using OCI Reference Type, etc.)
+
+Once stargz snapshotter acquires TOC image, it tries to find the TOC corresponding to the mounting eStargz blob, by looking `containerd.io/snapshot/stargz/layer.digest` annotations.
+As describe in the above, the acquired TOC JSON is validated using `containerd.io/snapshot/stargz/toc.digest` annotation.
+
 ## Example of TOC
 
 Here is an example TOC JSON:

docs/integration.md

@@ -0,0 +1,205 @@
+# Integration of eStargz with other tools
+
+This document lists links and information about integrations of stargz-snapshotter with tools in commuinty.
+
+You can refer to [issue #258 "Tracker issue for adoption status"](https://github.com/containerd/stargz-snapshotter/issues/258) for the list of the latest status of these integrations.
+
+## Kubernetes
+
+To use stargz snapshotter on Kubernetes nodes, you need to use containerd as the CRI runtime.
+You also need to run stargz snapshotter on the node.
+
+### Kind
+
+See [`/README.md#quick-start-with-kubernetes`](/README.md#quick-start-with-kubernetes).
+
+### k3s
+
+k3s >= v1.22 supports stagz-snapshotter as an experimental feature.
+`--snapshotter=stargz` for k3s server and agent enables this feature.
+
+```
+k3s server --snapshotter=stargz
+```
+
+Refer to [k3s docs](https://docs.k3s.io/advanced#enabling-lazy-pulling-of-estargz-experimental) for more details.
+
+The following is a quick demo using [k3d](https://github.com/k3d-io/k3d) (k3s in Docker).
+
+```console
+$ k3d cluster create mycluster --k3s-arg='--snapshotter=stargz@server:*;agent:*'
+$ cat <<'EOF' | kubectl --context=k3d-mycluster apply -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nodejs
+spec:
+  containers:
+  - name: nodejs-stargz
+    image: ghcr.io/stargz-containers/node:17.8.0-esgz
+    command: ["node"]
+    args:
+    - -e
+    - var http = require('http');
+      http.createServer(function(req, res) {
+        res.writeHead(200);
+        res.end('Hello World!\n');
+      }).listen(80);
+    ports:
+    - containerPort: 80
+EOF
+$ kubectl --context=k3d-mycluster get po nodejs -w
+$ kubectl --context=k3d-mycluster port-forward nodejs 8080:80 &
+$ curl 127.0.0.1:8080
+Hello World!
+$ k3d cluster delete mycluster
+```
+
+### Google Kubernetes Engine
+
+There is no node image includes stargz snapshotter by default as of now so you need to manually customize the nodes.
+
+A brief instrcution of enabling stargz snapshotter is the following:
+
+- Create a Kubernetes cluster using containerd-supported Linux node images like `ubuntu_containerd`. containerd must be >= v1.4.2.
+- SSH into each node and install stargz snapshotter following [`./INSTALL.md`](./INSTALL.md#install-stargz-snapshotter-for-containerd-with-systemd). You need this installation on all worker nodes.
+- Optionally apply configuration to allow stargz-snapshotter to access private registries following [`./overview.md`](./overview.md#authentication).
+
+### Amazon Elastic Kubernetes Service
+
+There is no AMI includes stargz snapshotter by default as of now so you need to manually customize the nodes.
+
+A brief instrcution of enabling stargz snapshotter is the following:
+
+- Create a Kubernetes cluster using containerd-supported Linux AMIs. containerd must be >= v1.4.2. e.g. Amazon EKS optimized Amazon Linux AMIs with [containerd runtime bootstrap flag](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html).
+- SSH into each node and install stargz snapshotter following [`./INSTALL.md`](./INSTALL.md#install-stargz-snapshotter-for-containerd-with-systemd). You need this installation on all worker nodes.
+- Optionally apply configuration to allow stargz-snapshotter to access private registries following [`./overview.md`](./overview.md#authentication).
+
+## CRI runtimes
+
+### containerd
+
+See [`./INSTALL.md`](./INSTALL.md#install-stargz-snapshotter-for-containerd-with-systemd)
+
+> :information_source: There is also a doc for [integration with firecracker-containerd](https://github.com/firecracker-microvm/firecracker-containerd/blob/24f1fcf99ebf6edcb94edd71a2affbcdae6b08e7/docs/remote-snapshotter-getting-started.md).
+
+### CRI-O
+
+See [`./INSTALL.md`](./INSTALL.md#install-stargz-store-for-cri-opodman-with-systemd).
+
+## High-level container engines
+
+### Docker
+
+#### Moby
+
+Moby supports lazy pulling of eStargz since [`5c1d6c957b97321c8577e10ddbffe6e01981617a`](https://github.com/moby/moby/commit/5c1d6c957b97321c8577e10ddbffe6e01981617a) .
+See [`./INSTALL.md`](./INSTALL.md#install-stargz-snapshotter-for-dockermoby-with-systemd) for details.
+
+#### Docker Desktop
+
+Docker Desktop 4.12.0 "Containerd Image Store (Beta)" uses stargz-snapshotter.
+Refer to [Docker documentation](https://docs.docker.com/desktop/containerd/).
+
+### nerdctl
+
+See the [docs in nerdctl](https://github.com/containerd/nerdctl/blob/main/docs/stargz.md).
+
+### Podman
+
+See [`./INSTALL.md`](./INSTALL.md#install-stargz-store-for-cri-opodman-with-systemd).
+
+## Image builders
+
+### BuildKit
+
+#### Building eStargz
+
+BuildKit >= v0.10 supports creating eStargz images.
+See [`README.md`](/README.md#building-estargz-images-using-buildkit) for details.
+
+#### Lazy pulling of eStargz
+
+BuildKit >= v0.8 supports stargz-snapshotter and can perform lazy pulling of eStargz-formatted base images during build.
+`--oci-worker-snapshotter=stargz` flag enables this feature.
+
+You can try this feature using Docker Buildx as the following.
+
+```
+$ docker buildx create --use --name lazy-builder --buildkitd-flags '--oci-worker-snapshotter=stargz'
+$ docker buildx inspect --bootstrap lazy-builder
+```
+
+The following is a sample Dockerfile that uses eStargz-formatted golang image (`ghcr.io/stargz-containers/golang:1.18-esgz`) as the base image.
+
+```Dockerfile
+FROM ghcr.io/stargz-containers/golang:1.18-esgz AS dev
+COPY ./hello.go /hello.go
+RUN go build -o /hello /hello.go
+
+FROM scratch
+COPY --from=dev /hello /
+ENTRYPOINT [ "/hello" ]
+```
+
+Put the following Go source code in the context directory with naming it `hello.go`.
+
+```golang
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Println("Hello, world!")
+}
+```
+
+The following build performs lazy pulling of the eStargz-formatted golang base image.
+
+```console
+$ docker buildx build --load -t hello /tmp/ctx/
+$ docker run --rm hello
+Hello, world!
+```
+
+### Kaniko
+
+#### Building eStargz
+
+Kaniko >= v1.5.0 creates eStargz images when `GGCR_EXPERIMENT_ESTARGZ=1` is specified.
+See [`README.md`](/README.md#building-estargz-images-using-kaniko) for details.
+
+### ko
+
+ko >= v0.7.0 creates eStargz images when `GGCR_EXPERIMENT_ESTARGZ=1` is specified.
+Please see also [the docs in ko](https://github.com/ko-build/ko/blob/f70e3cad38c3bbd232f51604d922b8baff31144e/docs/advanced/faq.md#can-i-optimize-images-for-estargz-support).
+
+## P2P image distribution
+
+### IPFS
+
+See [`./ipfs.md`](./ipfs.md)
+
+### Dragonfly
+
+Change the `/etc/containerd-stargz-grpc/config.toml` configuration to make dragonfly as registry mirror.
+`127.0.0.1:65001` is the proxy address of dragonfly peer,
+and the `X-Dragonfly-Registry` header is the address of origin registry,
+which is provided for dragonfly to download the images.
+
+```toml
+[[resolver.host."docker.io".mirrors]]
+  host = "127.0.0.1:65001"
+  insecure = true
+  [resolver.host."docker.io".mirrors.header]
+    X-Dragonfly-Registry = ["https://index.docker.io"]
+```
+
+For more details about dragonfly as registry mirror,
+refer to [How to use Dragonfly With eStargz](https://d7y.io/docs/setup/integration/stargz/).
+
+## Registry-side conversion of eStargz
+
+### Harbor
+
+See the docs in Harbor: https://github.com/goharbor/acceleration-service

docs/ipfs.md

@@ -1,5 +1,7 @@
 # Running containers on IPFS (experimental)
 
+:information_source: This document isn't for Kubernetes environemnt. For information about node-to-node image sharing on Kubernetes, please refer to [the docs in nerdctl project](https://github.com/containerd/nerdctl/tree/main/examples/nerdctl-ipfs-registry-kubernetes).
+
 You can run OCI-compatible container images on IPFS with lazy pulling.
 
 To enable this feature, add the following configuration to `config.toml` of Stargz Snapsohtter (typically located at `/etc/containerd-stargz-grpc/config.toml`).
@@ -8,6 +10,8 @@ To enable this feature, add the following configuration to `config.toml` of Star
 ipfs = true
 ```
 
+> NOTE: containerd-stargz-grpc tries to connect to IPFS API written in `~/.ipfs/api` (or the file under `$IPFS_PATH` if configured) via HTTP (not HTTPS).
+
 ## IPFS-enabled OCI Image
 
 For obtaining IPFS-enabled OCI Image, each descriptor in an OCI image must contain the following [IPFS URL](https://docs.ipfs.io/how-to/address-ipfs-on-web/#native-urls) in `urls` field.

docs/lima.md

@@ -0,0 +1,52 @@
+# Getting started with Stargz Snapshotter on Lima
+
+[Lima](https://github.com/lima-vm/lima) is a tool to manage Linux virtual machines on various hosts, including MacOS and Linux.
+Lima can be used as an easy way to get started with Stargz Snapshotter as Lima provides a default VM image bundling [containerd](https://github.com/containerd/containerd), [nerdctl](https://github.com/containerd/nerdctl)(Docker-compatible CLI of containerd) and Stargz Snapshotter.
+
+This document describes how to get started with Stargz Snapshotter on Lima.
+
+## Enable Stargz Snapshotter using `--snapshotter=stargz` flag
+
+nerdctl's `--snapshotter=stargz` flag enables stargz-snapshotter.
+
+```
+$ nerdctl.lima --snapshotter=stargz system info | grep stargz
+ Storage Driver: stargz
+```
+
+Using this flag, you can perform lazy pulling of a python eStargz image and run it.
+
+```
+$ nerdctl.lima --snapshotter=stargz run --rm -it --name python ghcr.io/stargz-containers/python:3.13-esgz
+Python 3.13.2 (main, Feb  6 2025, 22:37:13) [GCC 12.2.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>>
+```
+
+## Use Stargz Snapshotter as the default snapshotter
+
+nerdctl recognizes an environment variable `CONTAINERD_SNAPSHOTTER` for the snapshotter to use.
+You can add this environment variable to the VM by configuring Lima config as shown in the following:
+
+```
+$ cat <<EOF >> ~/.lima/_config/override.yaml
+env:
+  CONTAINERD_SNAPSHOTTER: stargz
+EOF
+$ limactl stop
+$ limactl start
+$ nerdctl.lima system info | grep Storage
+Storage Driver: stargz
+```
+
+> NOTE: `override.yaml` applies to all the instances of Lima
+
+
+You can perform lazy pulling of eStargz using nerdctl, without any extra flags.
+
+```
+$ nerdctl.lima run --rm -it --name python ghcr.io/stargz-containers/python:3.13-esgz
+Python 3.13.2 (main, Feb  6 2025, 22:37:13) [GCC 12.2.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>>
+```

docs/overview.md

@@ -1,22 +1,22 @@
 # Containerd Stargz Snapshotter Plugin Overview
 
-__Before get through this overview document, we recommend you to read [README](../README.md).__
+__Before reading this overview document, we recommend you read [README](../README.md).__
 
-Pulling image is one of the time-consuming steps in the container startup process.
-In containerd community, we have had a lot of discussions to address this issue as the following,
+Pulling images is one of the most time-consuming steps in the container startup process.
+In the containerd community, we have had a lot of discussions to address this issue at the following:
 
 - [#3731 Support remote snapshotter to speed up image pulling](https://github.com/containerd/containerd/issues/3731)
 - [#2968 Support `Prepare` for existing snapshots in Snapshotter interface](https://github.com/containerd/containerd/issues/2968)
 - [#2943 remote filesystem snapshotter](https://github.com/containerd/containerd/issues/2943)
 
-The solution for the fast image distribution is called *Remote Snapshotter* plugin.
-This prepares container's rootfs layers by directly mounting from remote stores instead of downloading and unpacking the entire image contents.
-The actual image contents can be fetched *lazily* so runtimes can startup containers before the entire image contents to be locally available.
-We call these remotely mounted layers as *remote snapshots*.
+The solution for fast image distribution is called *Remote Snapshotter* plugin.
+This prepares the container's rootfs layers by directly mounting from remote stores instead of downloading and unpacking the entire image contents.
+The actual image contents can be fetched *lazily* so runtimes can start containers before the entire image contents are locally available.
+We call these remotely mounted layers *remote snapshots*.
 
 *Stargz Snapshotter* is a remote snapshotter plugin implementation which supports standard compatible remote snapshots functionality.
 This snapshotter leverages [eStargz](/docs/stargz-estargz.md) image, which is lazily-pullable and still standard-compatible.
-Because of this compatibility, eStargz image can be pushed to and lazily pulled from [OCI](https://github.com/opencontainers/distribution-spec)/[Docker](https://docs.docker.com/registry/spec/api/) registries (e.g. ghcr.io).
+Because of this compatibility, eStargz images can be pushed to and lazily pulled from [OCI](https://github.com/opencontainers/distribution-spec)/[Docker](https://docs.docker.com/registry/spec/api/) registries (e.g. ghcr.io).
 Furthermore, images can run even on eStargz-agnostic runtimes (e.g. Docker).
 When you run a container image and it is formatted by eStargz, stargz snapshotter prepares container's rootfs layers as remote snapshots by mounting layers from the registry to the node, instead of pulling the entire image contents.
 
@@ -27,10 +27,10 @@ This document gives you a high-level overview of stargz snapshotter.
 ## Stargz Snapshotter proxy plugin
 
 Stargz snapshotter is implemented as a [proxy plugin](https://github.com/containerd/containerd/blob/04985039cede6aafbb7dfb3206c9c4d04e2f924d/PLUGINS.md#proxy-plugins) daemon (`containerd-stargz-grpc`) for containerd.
-When containerd starts a container, it queries the rootfs snapshots to stargz snapshotter daemon through an unix socket.
+When containerd starts a container, it queries the rootfs snapshots to stargz snapshotter daemon through a unix socket.
 This snapshotter remotely mounts queried eStargz layers from registries to the node and provides these mount points as remote snapshots to containerd.
 
-Containerd recognizes this plugin through an unix socket specified in the configuration file (e.g. `/etc/containerd/config.toml`).
+Containerd recognizes this plugin through a unix socket specified in the configuration file (e.g. `/etc/containerd/config.toml`).
 Stargz snapshotter can also be used through Kubernetes CRI by specifying the snapshotter name in the CRI plugin configuration.
 We assume that you are using containerd (> v1.4.2).
 
@@ -44,6 +44,8 @@ version = 2
   [proxy_plugins.stargz]
     type = "snapshot"
     address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
 
 # Use stargz snapshotter through CRI
 [plugins."io.containerd.grpc.v1.cri".containerd]
@@ -51,24 +53,26 @@ version = 2
   disable_snapshot_annotations = false
 ```
 
+> NOTE: `root` field of `proxy_plugins` is needed for the CRI plugin to recognize stargz snapshotter's root directory.
+
 This repo contains [a Dockerfile as a KinD node image](/Dockerfile) which includes the above configuration.
 
 ## State directory
 
 Stargz snapshotter mounts eStargz layers from registries to the node using FUSE.
-The all files metadata in the image are preserved on the filesystem and files contents are fetched from registries on demand.
+Metadata for all files in the image are preserved on the container filesystem and the file contents are fetched from registries on demand.
 
-At the root of the filesystem, there is a *state directory* (`/.stargz-snapshotter`) for status monitoring for the filesystem.
+At the root of the container filesystem, there is a *state directory* (`/.stargz-snapshotter`) for status monitoring for the filesystem.
 This directory is hidden from `getdents(2)` so you can't see this with `ls -a /`.
 Instead, you can directly access the directory by specifying the path (`/.stargz-snapshotter`).
 
-State directory contains JSON-formatted metadata files for each layer.
+The state directory contains JSON-formatted metadata files for each layer.
 In the following example, metadata JSON files for overlayed 7 layers are visible.
-In each metadata JSON file, the following fields are contained,
+In each metadata JSON file, the following fields are contained:
 
 - `digest` contains the layer digest. This is the same value as that in the image's manifest.
 - `size` is the size bytes of the layer.
-- `fetchedSize` and `fetchedPercent` indicate how many bytes have been fetched for this layer. Stargz snapshotter aggressively downloads this layer in the background - unless configured otherwise - so these values gradually increase. When `fetchedPercent` reaches to `100` percents, this layer has been fully downloaded on the node and no further access will occur for reading files.
+- `fetchedSize` and `fetchedPercent` indicate how many bytes have been fetched for this layer. Stargz snapshotter aggressively downloads this layer in the background - unless configured otherwise - so these values gradually increase. When `fetchedPercent` reaches `100` percent, this layer has been fully downloaded on the node and no further access will occur for reading files.
 
 Note that the state directory layout and the metadata JSON structure are subject to change.
 
@@ -95,6 +99,59 @@ root@1d43741b8d29:/go# cat /.stargz-snapshotter/*
 {"digest":"sha256:f077511be7d385c17ba88980379c5cd0aab7068844dffa7a1cefbf68cc3daea3","size":580,"fetchedSize":580,"fetchedPercent":100}
 ```
 
+## Fuse Manager
+
+The fuse manager is designed to maintain the availability of running containers by managing the lifecycle of FUSE mountpoints independently from the stargz snapshotter.
+
+### Fuse Manager Overview
+Remote snapshots are mounted using FUSE, and its filesystem processes are attached to the stargz snapshotter. If the stargz snapshotter restarts (due to configuration changes or crashes), all filesystem processes will be killed and restarted, which causes the remount of FUSE mountpoints, making running containers unavailable.
+
+To avoid this, we use a fuse daemon called the fuse manager to handle filesystem processes. The fuse manager is responsible for mounting and unmounting remote snapshotters. Its process is detached from the stargz snapshotter main process to an independent one in a shim-like way during the snapshotter's startup. This design ensures that the restart of the snapshotter won't affect the filesystem processes it manages, keeping mountpoints and running containers available during the restart. However, it is important to note that the restart of the fuse manager itself triggers a remount, so it is recommended to keep the fuse manager running in a good state.
+
+You can enable the fuse manager by adding the following configuration.
+
+```toml
+[fusem_anager]
+enable = true
+```
+
+## Killing and restarting Stargz Snapshotter
+
+Stargz Snapshotter works as a FUSE server for the snapshots.
+When you stop Stargz Sanpshotter on the node, it takes the following behaviour depending on the configuration.
+
+### FUSE manager mode is disabled
+
+killing containerd-stargz-grpc will result in unmounting all snapshot mounts managed by Stargz Snapshotter.
+When containerd-stargz-grpc is restarted, all those snapshots are mounted again by lazy pulling all layers.
+If the snapshotter fails to mount one of the snapshots (e.g. because of lazy pulling failure) during this step, the behaviour differs depending on `allow_invalid_mounts_on_restart` flag in the config TOML.
+
+- `allow_invalid_mounts_on_restart = true`: containerd-stargz-grpc leaves the failed snapshots as empty directories. The user needs to manually remove those snapshot via containerd (e.g. using `ctr snapshot rm` command). The name of those snapshots can be seen in the log with `failed to restore remote snapshot` message.
+
+- `allow_invalid_mounts_on_restart = false`: containerd-stargz-grpc doesn't start. The user needs to manually recover this (e.g. by wiping snapshotter and containerd state).
+
+### FUSE manager mode is enabled
+
+Killing containerd-stargz-grpc using non-SIGINT signal (e.g. using SIGTERM) doesn't affect the snapshot mounts because the FUSE manager process detached from containerd-stargz-grpc keeps on serving FUSE mounts to the kernel.
+This is useful when you reload the updated config TOML to Stargz Snapshotter without unmounting existing snapshots.
+
+FUSE manager serves FUSE mounts of the snapshots so if you kill this process, all snapshot mounts will be unavailable.
+When stopping FUSE manager for upgrading the binary or restarting the node, you can use SIGINT signal to trigger the graceful exit as shown in the following steps.
+
+1. Stop containers that use Stargz Snapshotter. Stopping FUSE manager makes all snapshot mounts unavailable so containers can't keep working.
+2. Stop containerd-stargz-grpc process using SIGINT. This signal triggers unmounting of all snapshots and cleaning up of the associated resources.
+3. Kill the FUSE manager process (`stargz-fuse-manager`)
+4. Restart the containerd-stargz-grpc process. This restores all snapshot mounts by lazy pulling them. `allow_invalid_mounts_on_restart` (described in the above) can still be used for controlling the behaviour of the error cases.
+5. Restart the containers.
+
+### Unexpected restart handling
+
+When Stargz Snapshotter is killed unexpectedly (e.g., by OOM killer or system crash), the process doesn't get a chance to perform graceful cleanup. In such cases, the snapshotter can successfully restart and restore remote snapshots, but this may lead to fscache duplicating cached data.
+
+**Recommended handling:**
+
+Since this scenario is caused by abnormal exit, users are expected to manually clean up the cache directory after an unexpected restart to avoid cache duplication issues. The cache cleanup should be performed before restarting the snapshotter service.
+
 ## Registry-related configuration
 
 You can configure stargz snapshotter for accessing registries with custom configurations.
@@ -123,12 +180,15 @@ Stargz snapshotter doesn't share credentials with containerd so credentials spec
 
 #### CRI-based authentication
 
-Following configuration enables stargz snapshotter to pull private images on Kubernetes.
+Following configuration (typically located at `/etc/containerd-stargz-grpc/config.toml`) enables stargz snapshotter to pull private images on Kubernetes.
 The snapshotter works as a proxy of CRI Image Service and exposes CRI Image Service API on the snapshotter's unix socket (i.e. `/run/containerd-stargz-grpc/containerd-stargz-grpc.sock`).
 The snapshotter acquires registry creds by scanning requests.
 
 You must specify `--image-service-endpoint=unix:///run/containerd-stargz-grpc/containerd-stargz-grpc.sock` option to kubelet.
 
+You can specify the backing image service's socket using `image_service_path`.
+The default is the containerd's socket (`/run/containerd/containerd.sock`).
+
 ```toml
 # Stargz Snapshotter proxies CRI Image Service into containerd socket.
 [cri_keychain]
@@ -136,11 +196,18 @@ enable_keychain = true
 image_service_path = "/run/containerd/containerd.sock"
 ```
 
+The default path where containerd-stargz-grpc serves the CRI Image Service API is `unix:///run/containerd-stargz-grpc/containerd-stargz-grpc.sock`.
+You can also change this path using `listen_path` field.
+
+> Note that if you enabled the FUSE manager and CRI-based authentication together, `listen_path` is a mandatory field with some caveats:
+> - This path must be different from the FUSE manager's socket path (`/run/containerd-stargz-grpc/fuse-manager.sock`) because they have different lifecycle. Specifically, the CRI socket is recreted on each reload of the configuration to the FUSE manager.
+> - containerd-stargz-grpc's socket path (`/run/containerd-stargz-grpc/containerd-stargz-grpc.sock`) can't be used as `listen_path` because the CRI socket is served by the FUSE manager process (not containerd-stargz-grpc process).
+
 #### kubeconfig-based authentication
 
 This is another way to enable lazy pulling of private images on Kubernetes.
 
-Following configuration enables stargz snapshotter to access to private registries using kubernetes secrets (type = `kubernetes.io/dockerconfigjson`) in the cluster using kubeconfig files.
+Following configuration (typically located at `/etc/containerd-stargz-grpc/config.toml`) enables stargz snapshotter to access to private registries using kubernetes secrets (type = `kubernetes.io/dockerconfigjson`) in the cluster using kubeconfig files.
 You can specify the path of kubeconfig file using `kubeconfig_path` option.
 It's no problem that the specified file doesn't exist when this snapshotter starts.
 In this case, snapsohtter polls the file until actually provided.
@@ -176,6 +243,17 @@ host = "exampleregistry.io"
 insecure = true
 ```
 
+`header` field allows to set headers to send to the server.
+
+```toml
+[[resolver.host."registry2:5000".mirrors]]
+  host = "registry2:5000"
+  [resolver.host."registry2:5000".mirrors.header]
+    x-custom-2 = ["value3", "value4"]
+```
+
+> NOTE: Headers aren't passed to the redirected location.
+
 The config file can be passed to stargz snapshotter using `containerd-stargz-grpc`'s `--config` option.
 
 ## Make your remote snapshotter

docs/passthrough.md

@@ -0,0 +1,60 @@
+# Introduction
+
+FUSE Passthrough has been introduced in the Linux kernel version 6.9 ([Linux Kernel Commit](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=6ce8b2ce0d7e3a621cdc9eb66d74436ca7d0e66e)). This feature has shown significant performance improvements, as detailed in the following articles:
+
+[Phoronix Article on FUSE Passthrough](https://www.phoronix.com/news/FUSE-Passthrough-In-6.9-Next)<br>
+
+FUSE Passthrough allows performing read and write (also via memory maps) on a backing file without incurring the overhead of roundtrips to userspace.
+
+![passhthrough feature](/docs/images/passthrough01.png)
+
+Additionally, the `go-fuse` package, which Stargz-Snapshotter depends on, has also added support for this passthrough feature:
+
+[go-fuse Commit 1](https://github.com/hanwen/go-fuse/commit/e0641a46c6cca7e5370fc135f78caf7cb7fc3aa8#diff-f830ac3db25844bf71102b09e4e02f7213e9cdb577b32745979d61d775462bd3R157)<br>
+[go-fuse Commit 2](https://github.com/hanwen/go-fuse/commit/e0a0b09ae8287249c38033a27fd69a3593c7e235#diff-1521152f1fc3600273bda897c669523dc1e9fc9cbe24046838f043a8040f0d67R749)<br>
+[go-fuse Commit 3](https://github.com/hanwen/go-fuse/commit/1a7d98b0360f945fca50ac79905332b7106c049f)
+
+When a user-defined file implements the `FilePassthroughFder` interface, `go-fuse` will attempt to register the file `fd` from the file with the kernel.
+
+# Configuration
+
+## Basic Configuration
+
+To enable FUSE passthrough mode, first verify that your host's kernel supports this feature. You can check this by running the following command:
+
+```bash
+$ cat /boot/config-$(uname -r) | grep "CONFIG_FUSE_PASSTHROUGH=y"
+CONFIG_FUSE_PASSTHROUGH=y
+```
+
+Once you have confirmed kernel support, you need to enable passthrough mode in your `config.toml` file with the following configuration:
+
+```toml
+[fuse]
+passthrough = true
+```
+
+After updating the configuration, specify the `config.toml` file when starting `containerd-stargz-grpc` and restart the service:
+
+```bash
+$ containerd-stargz-grpc -config config.toml
+```
+
+## Advanced Configuration
+
+In passthrough mode, the initial pull of an image requires merging chunks into a file. This process can be time-consuming, especially for large files.
+
+To optimize the time taken for the initial image pull, you can use the `merge_buffer_size` and `merge_worker_count` configuration options. The `merge_buffer_size` specifies the size of the buffer used for reading the image, with a default value of 400MB. The `merge_worker_count` determines the level of concurrency for reading the image, with a default value of 10.
+
+By concurrently reading chunks and caching them for batch writing, you can significantly enhance the performance of the initial image pull in passthrough mode.
+
+# Important Considerations
+
+When passthrough mode is enabled, the following configuration is applied by default, even if it is set to false in the configuration file:
+
+```toml
+[directory_cache]
+direct = true
+```
+
+This is because, in passthrough mode, read operations after opening a file are handled directly by the kernel.

docs/pre-converted-images.md

@@ -3,12 +3,16 @@
 We have several pre-converted stargz images on Github Container Registry (`ghcr.io/stargz-containers`), mainly for benchmarking purpose.
 This document lists them.
 
-:information_source: You can build your eStargz images optimized for your workload, using [`ctr-remote` command](/docs/ctr-remote.md).
+:information_source: You can build eStargz from Dockerfile using BuildKit, [using Docker Buildx](../README.md#building-estargz-images-using-buildkit) or [Kaniko](../README.md#building-estargz-images-using-kaniko).
+
+:information_source: You can convert arbitrary images into eStargz optimized for your workload, using [`ctr-remote` command](/docs/ctr-remote.md).
 
 :information_source: You can convert arbitrary images into eStargz on the registry-side, using [`estargz.kontain.me`](https://estargz.kontain.me).
 
 ## Pre-converted images
 
+:information_source: You can request new pre-converted images from our CI repository ([`github.com/stargz-containers/image-ci`](https://github.com/stargz-containers/image-ci)).
+
 In the following table, image names listed in `Image Name` contain the following suffixes based on the type of the image.
 
 - `org`: Legacy image copied from `docker.io/library` without optimization. Layers are normal tarballs.
@@ -18,61 +22,61 @@ In the following table, image names listed in `Image Name` contain the following
 
 |Image Name|Optimized Workload|
 ---|---
-|`ghcr.io/stargz-containers/alpine:3.10.2-org`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/alpine:3.10.2-esgz`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/drupal:8.7.6-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
-|`ghcr.io/stargz-containers/drupal:8.7.6-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
-|`ghcr.io/stargz-containers/fedora:30-org`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/fedora:30-esgz`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/gcc:10.2.0-org`|Compiling and executing a program which prints `hello`|
-|`ghcr.io/stargz-containers/gcc:10.2.0-esgz`|Compiling and executing a program which prints `hello`|
-|`ghcr.io/stargz-containers/golang:1.12.9-org`|Compiling and executing a program which prints `hello`|
-|`ghcr.io/stargz-containers/golang:1.12.9-esgz`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/alpine:3.15.3-org`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/alpine:3.15.3-esgz`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/drupal:9.3.9-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/drupal:9.3.9-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/fedora:35-org`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/fedora:35-esgz`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/gcc:11.2.0-org`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/gcc:11.2.0-esgz`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/golang:1.18-org`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/golang:1.18-esgz`|Compiling and executing a program which prints `hello`|
 |`ghcr.io/stargz-containers/jenkins:2.60.3-org`|Code execution until up and ready message (`Jenkins is fully up and running`) is printed|
 |`ghcr.io/stargz-containers/jenkins:2.60.3-esgz`|Code execution until up and ready message (`Jenkins is fully up and running`) is printed|
-|`ghcr.io/stargz-containers/jruby:9.2.8.0-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/jruby:9.2.8.0-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/node:13.13.0-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/node:13.13.0-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/perl:5.30-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/perl:5.30-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/php:7.3.8-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/php:7.3.8-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/pypy:3.5-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/pypy:3.5-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/python:3.9-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/python:3.9-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/r-base:3.6.1-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/r-base:3.6.1-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/redis:5.0.5-org`|Code execution until up and ready message (`Ready to accept connections`) is printed|
-|`ghcr.io/stargz-containers/redis:5.0.5-esgz`|Code execution until up and ready message (`Ready to accept connections`) is printed|
-|`ghcr.io/stargz-containers/rethinkdb:2.3.6-org`|Code execution until up and ready message (`Server ready`) is printed|
-|`ghcr.io/stargz-containers/rethinkdb:2.3.6-esgz`|Code execution until up and ready message (`Server ready`) is printed|
-|`ghcr.io/stargz-containers/tomcat:10.0.0-jdk15-openjdk-buster-org`|Code execution until up and ready message (`Server startup`) is printed|
-|`ghcr.io/stargz-containers/tomcat:10.0.0-jdk15-openjdk-buster-esgz`|Code execution until up and ready message (`Server startup`) is printed|
-|`ghcr.io/stargz-containers/postgres:13.1-org`|Code execution until up and ready message (`database system is ready to accept connections`) is printed|
-|`ghcr.io/stargz-containers/postgres:13.1-esgz`|Code execution until up and ready message (`database system is ready to accept connections`) is printed|
-|`ghcr.io/stargz-containers/wordpress:5.7-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
-|`ghcr.io/stargz-containers/wordpress:5.7-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
-|`ghcr.io/stargz-containers/mariadb:10.5-org`|Code execution until up and ready message (`mysqld: ready for connections`) is printed|
-|`ghcr.io/stargz-containers/mariadb:10.5-esgz`|Code execution until up and ready message (`mysqld: ready for connections`) is printed|
-|`ghcr.io/stargz-containers/php:8-apache-buster-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
-|`ghcr.io/stargz-containers/php:8-apache-buster-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/jruby:9.3.4-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/jruby:9.3.4-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/node:17.8.0-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/node:17.8.0-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/perl:5.34.1-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/perl:5.34.1-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/php:8.1.4-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/php:8.1.4-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/pypy:3.9-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/pypy:3.9-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/python:3.10-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/python:3.10-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/r-base:4.1.3-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/r-base:4.1.3-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/redis:6.2.6-org`|Code execution until up and ready message (`Ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/redis:6.2.6-esgz`|Code execution until up and ready message (`Ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/rethinkdb:2.4.1-org`|Code execution until up and ready message (`Server ready`) is printed|
+|`ghcr.io/stargz-containers/rethinkdb:2.4.1-esgz`|Code execution until up and ready message (`Server ready`) is printed|
+|`ghcr.io/stargz-containers/tomcat:10.1.0-jdk17-openjdk-bullseye-org`|Code execution until up and ready message (`Server startup`) is printed|
+|`ghcr.io/stargz-containers/tomcat:10.1.0-jdk17-openjdk-bullseye-esgz`|Code execution until up and ready message (`Server startup`) is printed|
+|`ghcr.io/stargz-containers/postgres:14.2-org`|Code execution until up and ready message (`database system is ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/postgres:14.2-esgz`|Code execution until up and ready message (`database system is ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/wordpress:5.9.2-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/wordpress:5.9.2-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/mariadb:10.7.3-org`|Code execution until up and ready message (`mysqld: ready for connections`) is printed|
+|`ghcr.io/stargz-containers/mariadb:10.7.3-esgz`|Code execution until up and ready message (`mysqld: ready for connections`) is printed|
+|`ghcr.io/stargz-containers/php:8.1.4-apache-bullseye-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/php:8.1.4-apache-bullseye-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/rabbitmq:3.9.14-org`|Code execution until up and ready message (`Server startup complete`) is printed|
+|`ghcr.io/stargz-containers/rabbitmq:3.9.14-esgz`|Code execution until up and ready message (`Server startup complete`) is printed|
+|`ghcr.io/stargz-containers/elasticsearch:8.1.1-org`|Code execution until up and ready message (`started`) is printed|
+|`ghcr.io/stargz-containers/elasticsearch:8.1.1-esgz`|Code execution until up and ready message (`started`) is printed|
+|`ghcr.io/stargz-containers/nixos/nix:2.3.12-org`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/nixos/nix:2.3.12-esgz`|Executing `echo hello` on the shell|
 
 ## lazy-pulling-enabled KinD node image
 
-You can enable lazy pulling of eStargz on [KinD](https://github.com/kubernetes-sigs/kind) using our [prebuilt node image](https://github.com/orgs/stargz-containers/packages/container/package/estargz-kind-node).
+You can enable lazy pulling of eStargz on [KinD](https://github.com/kubernetes-sigs/kind) using our prebuilt node image [`ghcr.io/containerd/stargz-snapshotter:${VERSION}-kind`](https://github.com/orgs/containerd/packages/container/package/stargz-snapshotter) namespace.
+
+Example:
 
 ```console
-$ kind create cluster --name stargz-demo --image ghcr.io/stargz-containers/estargz-kind-node:0.7.0
-```
-
-> kind binary v0.11.x or newer is recommended for `estargz-kind-node:0.7.0`.
-
-You can also build it on your own.
-
-```
-$ docker build -t estargz-kind-node https://github.com/containerd/stargz-snapshotter.git
+$ kind create cluster --name stargz-demo --image ghcr.io/containerd/stargz-snapshotter:0.12.1-kind
 ```
 
 Please refer to README for more details.

docs/rootless.md

@@ -0,0 +1,56 @@
+# Rootless execution of stargz snapshotter
+
+This document lists links and information about how to run Stargz Snapshotter and Stargz Store from the non-root user.
+
+## nerdctl (Stargz Snapshotter)
+
+Rootless Stargz Snapshotter for nerdctl can be installed via `containerd-rootless-setuptool.sh install-stargz` command.
+Please see [the doc in nerdctl repo](https://github.com/containerd/nerdctl/blob/v1.1.0/docs/rootless.md#stargz-snapshotter) for details.
+
+## Podman (Stargz Store)
+
+> NOTE: This is an experimental configuration leveraging [`podman unshare`](https://docs.podman.io/en/latest/markdown/podman-unshare.1.html). Limitation: `--uidmap` of `podman run` doesn't work.
+
+First, allow podman using Stargz Store by adding the following store configuration.
+Put the configuration file to [`/etc/containers/storage.conf` or `$HOME/.config/containers/storage.conf`](https://github.com/containers/podman/blob/v4.3.1/docs/tutorials/rootless_tutorial.md#storageconf).
+
+> NOTE: Replace `/path/to/home` to the actual home directory.
+
+```
+[storage]
+driver = "overlay"
+
+[storage.options]
+additionallayerstores = ["/path/to/homedir/.local/share/stargz-store/store:ref"]
+```
+
+Start Stargz Store in the namespace managed by podman via [`podman unshare`](https://docs.podman.io/en/latest/markdown/podman-unshare.1.html) command.
+
+```
+$ podman unshare stargz-store --root $HOME/.local/share/stargz-store/data $HOME/.local/share/stargz-store/store &
+```
+
+Podman performs lazy pulling when it pulls eStargz images.
+
+```
+$ podman pull ghcr.io/stargz-containers/python:3.9-esgz
+```
+
+<details>
+<summary>Creating systemd unit file for Stargz Store</summary>
+
+It's possible to create systemd unit file of Stargz Store for easily managing it.
+An example systemd unit file can be found [here](../script/podman/config/podman-rootless-stargz-store.service)
+
+After installing that file (e.g. to `$HOME/.config/systemd/user/`), start the service using `systemctl`.
+
+```
+$ systemctl --user start podman-rootless-stargz-store
+```
+
+</details>
+
+## BuildKit (Stargz Snapshotter)
+
+BuildKit supports running Stargz Snapshotter from the non-root user.
+Please see [the doc in BuildKit repo](https://github.com/moby/buildkit/blob/8b132188aa7af944c813d02da63c93308d83cf75/docs/stargz-estargz.md) (unmerged 2023/1/18) for details.

docs/smaller-estargz.md

@@ -0,0 +1,79 @@
+# Creating smaller eStargz images
+
+The following flags of `ctr-remote i convert` and `ctr-remote i optimize` allow users optionally creating smaller eStargz images.
+
+- `--estargz-external-toc`: Separate TOC JSON into another image (called "TOC image"). The result eStargz doesn't contain TOC so we can expect a smaller size than normal eStargz.
+
+- `--estargz-min-chunk-size`: The minimal number of bytes of data must be written in one gzip stream. If it's > 0, multiple files and chunks can be written into one gzip stream. Smaller number of gzip header and smaller size of the result blob can be expected. `--estargz-min-chunk-size=0` produces normal eStargz.
+
+## `--estargz-external-toc` usage
+
+convert:
+
+```console
+# ctr-remote i pull ghcr.io/stargz-containers/ubuntu:22.04
+# ctr-remote i convert --oci --estargz --estargz-external-toc ghcr.io/stargz-containers/ubuntu:22.04 registry2:5000/ubuntu:22.04-ex
+```
+
+Layers in eStargz (`registry2:5000/ubuntu:22.04-ex`) don't contain TOC JSON.
+TOC image (`registry2:5000/ubuntu:22.04-ex-esgztoc`) contains TOC of all layers of the eStargz image.
+Suffix `-esgztoc` is automatically added to the image name by `ctr-remote`.
+
+Then push eStargz(`registry2:5000/ubuntu:22.04-ex`) and TOC image(`registry2:5000/ubuntu:22.04-ex-esgztoc`) to the same registry:
+
+```console
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-ex
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-ex-esgztoc
+```
+
+Pull it lazily:
+
+```console
+# ctr-remote i rpull --plain-http registry2:5000/ubuntu:22.04-ex
+fetching sha256:14fb0ea2... application/vnd.oci.image.index.v1+json
+fetching sha256:24471b45... application/vnd.oci.image.manifest.v1+json
+fetching sha256:d2e4737e... application/vnd.oci.image.config.v1+json
+# mount | grep "stargz on"
+stargz on /var/lib/containerd-stargz-grpc/snapshotter/snapshots/1/fs type fuse.rawBridge (rw,nodev,relatime,user_id=0,group_id=0,allow_other)
+```
+
+Stargz Snapshotter automatically refers to the TOC image on the same registry.
+
+### optional `--estargz-keep-diff-id` flag for conversion without changing layer diffID
+
+`ctr-remote i convert` supports optional flag `--estargz-keep-diff-id` specified with `--estargz-external-toc`.
+This converts an image to eStargz without changing the diffID (uncompressed digest) so even eStargz-agnostic gzip decompressor (e.g. gunzip) can restore the original tar blob.
+
+```console
+# ctr-remote i pull ghcr.io/stargz-containers/ubuntu:22.04
+# ctr-remote i convert --oci --estargz --estargz-external-toc --estargz-keep-diff-id ghcr.io/stargz-containers/ubuntu:22.04 registry2:5000/ubuntu:22.04-ex-keepdiff
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-ex-keepdiff
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-ex-keepdiff-esgztoc
+# crane --insecure blob registry2:5000/ubuntu:22.04-ex-keepdiff@sha256:2dc39ba059dcd42ade30aae30147b5692777ba9ff0779a62ad93a74de02e3e1f | jq -r '.rootfs.diff_ids[]'
+sha256:7f5cbd8cc787c8d628630756bcc7240e6c96b876c2882e6fc980a8b60cdfa274
+# crane blob ghcr.io/stargz-containers/ubuntu:22.04@sha256:2dc39ba059dcd42ade30aae30147b5692777ba9ff0779a62ad93a74de02e3e1f | jq -r '.rootfs.diff_ids[]'
+sha256:7f5cbd8cc787c8d628630756bcc7240e6c96b876c2882e6fc980a8b60cdfa274
+```
+
+## `--estargz-min-chunk-size` usage
+
+conversion:
+
+```console
+# ctr-remote i pull ghcr.io/stargz-containers/ubuntu:22.04
+# ctr-remote i convert --oci --estargz --estargz-min-chunk-size=50000 ghcr.io/stargz-containers/ubuntu:22.04 registry2:5000/ubuntu:22.04-chunk50000
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-chunk50000
+```
+
+Pull it lazily:
+
+```console
+# ctr-remote i rpull --plain-http registry2:5000/ubuntu:22.04-chunk50000
+fetching sha256:5d1409a2... application/vnd.oci.image.index.v1+json
+fetching sha256:859e2b50... application/vnd.oci.image.manifest.v1+json
+fetching sha256:c07a44b9... application/vnd.oci.image.config.v1+json
+# mount | grep "stargz on"
+stargz on /var/lib/containerd-stargz-grpc/snapshotter/snapshots/1/fs type fuse.rawBridge (rw,nodev,relatime,user_id=0,group_id=0,allow_other)
+```
+
+> NOTE: This flag creates an eStargz image with newly-added `innerOffset` funtionality of eStargz. Stargz Snapshotter < v0.13.0 cannot perform lazy pulling for the images created with this flag.

docs/transfer.md

@@ -0,0 +1,99 @@
+# Enabling Stargz Snapshotter With Transfer Service
+
+Transfer Service is a containerd component which is used for image management in contianerd (e.g. pulling and pushing images).
+For details about Transfer Service, refer to [the official document in the containerd repo](https://github.com/containerd/containerd/blob/6af7c07905a317d4c343a49255e2392f4c8569f9/docs/transfer.md).
+
+To use Stargz Snapshotter on containerd with enabling Transfer Service, additional configurations is needed.
+
+## Availability of Transfer Service
+
+Transfer Service is available since v1.7.
+And this is enabled in different settings depending on the containerd version.
+
+|containerd version|`ctr`|CRI|
+---|---|---
+|containerd >= v1.7 and < v2.0|Disabled by default. Enabled by `--local=false`|Disabled|
+|containerd >= v2.0 and < v2.1|Enabled by default. Disabled by `--local`|Disabled|
+|containerd >= v2.1|Enabled by default. Disabled by `--local`|Enabled by default. Disabled when conditions described in [containerd's CRI document](https://github.com/containerd/containerd/blob/v2.1.0/docs/cri/config.md#image-pull-configuration-since-containerd-v21) are met|
+
+### Note about containerd v2.1
+
+Before containerd v2.1, `disable_snapshot_annotations = false` in containerd's config TOML was a mandatory field to enable Stargz Snapshotter in CRI.
+In containerd v2.1, `disable_snapshot_annotations = false` field can still be used to enable Stargz Snapshotter and containerd disables Transfer Service when this field is detected.
+If you want to enable Transfer Service, you need to remove `disable_snapshot_annotations = false` field and apply the configuration explaind in this document.
+
+## How to enable Stargz Snapshotter when Transfer Service is enabled?
+
+In containerd v2.1, Transfer Service added support for remote snapshotters like Stargz Snapshotter.
+
+### For ctr and other non-CRI clients
+
+To enable Stargz Snapshotter with Transfer Service, you need to start containerd-stargz-grpc on the node and add the following configuration to contianerd's config TOML file.
+Note that you need to add a field `enable_remote_snapshot_annotations = "true"` in `proxy_plugins.stargz.exports` so that containerd can correctly pass image-related information to Stargz Snapshotter.
+
+```toml
+version = 2
+
+# Enable Stargz Snapshotter in Transfer Service
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "stargz"
+
+# Plugin Stargz Snapshotter
+[proxy_plugins]
+  [proxy_plugins.stargz]
+    type = "snapshot"
+    address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
+    enable_remote_snapshot_annotations = "true"
+```
+
+#### Example client command
+
+When you enable Transfer Service with Stargz Snapshotter, you can perform lazy pulling using the normal `ctr` command. (of course, `ctr-remote` can still be used)
+
+```
+# ctr image pull --snapshotter=stargz ghcr.io/stargz-containers/ubuntu:24.04-esgz
+```
+
+Then `mount | grep stargz` prints stargz mounts on the node.
+
+### For CRI
+
+To enable Stargz Snapshotter with Transfer Service, you need to start containerd-stargz-grpc on the node and add the following configuration to contianerd's config TOML file.
+
+```toml
+version = 2
+
+# Basic CRI configuration with enabling Stargz Snapshotter
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  default_runtime_name = "runc"
+  snapshotter = "stargz"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+# Enable Stargz Snapshotter in Transfer Service
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "stargz"
+
+# Plugin Stargz Snapshotter
+[proxy_plugins]
+  [proxy_plugins.stargz]
+    type = "snapshot"
+    address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
+    enable_remote_snapshot_annotations = "true"
+```
+
+#### Example client command
+
+You can quickly check the behaviour using `crictl` command.
+
+```
+# crictl image pull ghcr.io/stargz-containers/ubuntu:24.04-esgz
+```
+
+Then `mount | grep stargz` prints stargz mounts on the node.

estargz/build.go

@@ -26,9 +26,10 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"context"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"runtime"
@@ -38,7 +39,6 @@ import (
 	"github.com/containerd/stargz-snapshotter/estargz/errorutil"
 	"github.com/klauspost/compress/zstd"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -48,6 +48,8 @@ type options struct {
 	prioritizedFiles       []string
 	missedPrioritizedFiles *[]string
 	compression            Compression
+	ctx                    context.Context
+	minChunkSize           int
 }
 
 type Option func(o *options) error
@@ -62,6 +64,7 @@ func WithChunkSize(chunkSize int) Option {
 
 // WithCompressionLevel option specifies the gzip compression level.
 // The default is gzip.BestCompression.
+// This option will be ignored if WithCompression option is used.
 // See also: https://godoc.org/compress/gzip#pkg-constants
 func WithCompressionLevel(level int) Option {
 	return func(o *options) error {
@@ -104,6 +107,26 @@ func WithCompression(compression Compression) Option {
 	}
 }
 
+// WithContext specifies a context that can be used for clean canceleration.
+func WithContext(ctx context.Context) Option {
+	return func(o *options) error {
+		o.ctx = ctx
+		return nil
+	}
+}
+
+// WithMinChunkSize option specifies the minimal number of bytes of data
+// must be written in one gzip stream.
+// By increasing this number, one gzip stream can contain multiple files
+// and it hopefully leads to smaller result blob.
+// NOTE: This adds a TOC property that old reader doesn't understand.
+func WithMinChunkSize(minChunkSize int) Option {
+	return func(o *options) error {
+		o.minChunkSize = minChunkSize
+		return nil
+	}
+}
+
 // Blob is an eStargz blob.
 type Blob struct {
 	io.ReadCloser
@@ -139,12 +162,29 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 		opts.compression = newGzipCompressionWithLevel(opts.compressionLevel)
 	}
 	layerFiles := newTempFiles()
+	ctx := opts.ctx
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-done:
+			// nop
+		case <-ctx.Done():
+			layerFiles.CleanupAll()
+		}
+	}()
 	defer func() {
 		if rErr != nil {
 			if err := layerFiles.CleanupAll(); err != nil {
-				rErr = errors.Wrapf(rErr, "failed to cleanup tmp files: %v", err)
+				rErr = fmt.Errorf("failed to cleanup tmp files: %v: %w", err, rErr)
 			}
 		}
+		if cErr := ctx.Err(); cErr != nil {
+			rErr = fmt.Errorf("error from context %q: %w", cErr, rErr)
+		}
 	}()
 	tarBlob, err := decompressBlob(tarBlob, layerFiles)
 	if err != nil {
@@ -154,7 +194,14 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 	if err != nil {
 		return nil, err
 	}
-	tarParts := divideEntries(entries, runtime.GOMAXPROCS(0))
+	var tarParts [][]*entry
+	if opts.minChunkSize > 0 {
+		// Each entry needs to know the size of the current gzip stream so they
+		// cannot be processed in parallel.
+		tarParts = [][]*entry{entries}
+	} else {
+		tarParts = divideEntries(entries, runtime.GOMAXPROCS(0))
+	}
 	writers := make([]*Writer, len(tarParts))
 	payloads := make([]*os.File, len(tarParts))
 	var mu sync.Mutex
@@ -169,6 +216,13 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 			}
 			sw := NewWriterWithCompressor(esgzFile, opts.compression)
 			sw.ChunkSize = opts.chunkSize
+			sw.MinChunkSize = opts.minChunkSize
+			if sw.needsOpenGzEntries == nil {
+				sw.needsOpenGzEntries = make(map[string]struct{})
+			}
+			for _, f := range []string{PrefetchLandmark, NoPrefetchLandmark} {
+				sw.needsOpenGzEntries[f] = struct{}{}
+			}
 			if err := sw.AppendTar(readerFromEntries(parts...)); err != nil {
 				return err
 			}
@@ -183,7 +237,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 		rErr = err
 		return nil, err
 	}
-	tocAndFooter, tocDgst, err := closeWithCombine(opts.compressionLevel, writers...)
+	tocAndFooter, tocDgst, err := closeWithCombine(writers...)
 	if err != nil {
 		rErr = err
 		return nil, err
@@ -226,7 +280,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 // Writers doesn't write TOC and footer to the underlying writers so they can be
 // combined into a single eStargz and tocAndFooter returned by this function can
 // be appended at the tail of that combined blob.
-func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) {
+func closeWithCombine(ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) {
 	if len(ws) == 0 {
 		return nil, "", fmt.Errorf("at least one writer must be passed")
 	}
@@ -307,7 +361,7 @@ func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]stri
 	// Import tar file.
 	intar, err := importTar(in)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to sort")
+		return nil, fmt.Errorf("failed to sort: %w", err)
 	}
 
 	// Sort the tar file respecting to the prioritized files list.
@@ -318,7 +372,7 @@ func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]stri
 				*missedPrioritized = append(*missedPrioritized, l)
 				continue // allow not found
 			}
-			return nil, errors.Wrap(err, "failed to sort tar entries")
+			return nil, fmt.Errorf("failed to sort tar entries: %w", err)
 		}
 	}
 	if len(prioritized) == 0 {
@@ -354,11 +408,11 @@ func readerFromEntries(entries ...*entry) io.Reader {
 		defer tw.Close()
 		for _, entry := range entries {
 			if err := tw.WriteHeader(entry.header); err != nil {
-				pw.CloseWithError(fmt.Errorf("Failed to write tar header: %v", err))
+				pw.CloseWithError(fmt.Errorf("failed to write tar header: %v", err))
 				return
 			}
 			if _, err := io.Copy(tw, entry.payload); err != nil {
-				pw.CloseWithError(fmt.Errorf("Failed to write tar payload: %v", err))
+				pw.CloseWithError(fmt.Errorf("failed to write tar payload: %v", err))
 				return
 			}
 		}
@@ -369,9 +423,9 @@ func readerFromEntries(entries ...*entry) io.Reader {
 
 func importTar(in io.ReaderAt) (*tarFile, error) {
 	tf := &tarFile{}
-	pw, err := newCountReader(in)
+	pw, err := newCountReadSeeker(in)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to make position watcher")
+		return nil, fmt.Errorf("failed to make position watcher: %w", err)
 	}
 	tr := tar.NewReader(pw)
 
@@ -382,9 +436,8 @@ func importTar(in io.ReaderAt) (*tarFile, error) {
 		if err != nil {
 			if err == io.EOF {
 				break
-			} else {
-				return nil, errors.Wrap(err, "failed to parse tar file")
 			}
+			return nil, fmt.Errorf("failed to parse tar file, %w", err)
 		}
 		switch cleanEntryName(h.Name) {
 		case PrefetchLandmark, NoPrefetchLandmark:
@@ -420,7 +473,7 @@ func moveRec(name string, in *tarFile, out *tarFile) error {
 	_, okIn := in.get(name)
 	_, okOut := out.get(name)
 	if !okIn && !okOut {
-		return errors.Wrapf(errNotFound, "file: %q", name)
+		return fmt.Errorf("file: %q: %w", name, errNotFound)
 	}
 
 	parent, _ := path.Split(strings.TrimSuffix(name, "/"))
@@ -506,12 +559,13 @@ func newTempFiles() *tempFiles {
 }
 
 type tempFiles struct {
-	files   []*os.File
-	filesMu sync.Mutex
+	files       []*os.File
+	filesMu     sync.Mutex
+	cleanupOnce sync.Once
 }
 
 func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) {
-	f, err := ioutil.TempFile(dir, pattern)
+	f, err := os.CreateTemp(dir, pattern)
 	if err != nil {
 		return nil, err
 	}
@@ -521,7 +575,14 @@ func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) {
 	return f, nil
 }
 
-func (tf *tempFiles) CleanupAll() error {
+func (tf *tempFiles) CleanupAll() (err error) {
+	tf.cleanupOnce.Do(func() {
+		err = tf.cleanupAll()
+	})
+	return
+}
+
+func (tf *tempFiles) cleanupAll() error {
 	tf.filesMu.Lock()
 	defer tf.filesMu.Unlock()
 	var allErr []error
@@ -537,19 +598,19 @@ func (tf *tempFiles) CleanupAll() error {
 	return errorutil.Aggregate(allErr)
 }
 
-func newCountReader(r io.ReaderAt) (*countReader, error) {
+func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) {
 	pos := int64(0)
-	return &countReader{r: r, cPos: &pos}, nil
+	return &countReadSeeker{r: r, cPos: &pos}, nil
 }
 
-type countReader struct {
+type countReadSeeker struct {
 	r    io.ReaderAt
 	cPos *int64
 
 	mu sync.Mutex
 }
 
-func (cr *countReader) Read(p []byte) (int, error) {
+func (cr *countReadSeeker) Read(p []byte) (int, error) {
 	cr.mu.Lock()
 	defer cr.mu.Unlock()
 
@@ -560,18 +621,18 @@ func (cr *countReader) Read(p []byte) (int, error) {
 	return n, err
 }
 
-func (cr *countReader) Seek(offset int64, whence int) (int64, error) {
+func (cr *countReadSeeker) Seek(offset int64, whence int) (int64, error) {
 	cr.mu.Lock()
 	defer cr.mu.Unlock()
 
 	switch whence {
 	default:
-		return 0, fmt.Errorf("Unknown whence: %v", whence)
+		return 0, fmt.Errorf("unknown whence: %v", whence)
 	case io.SeekStart:
 	case io.SeekCurrent:
 		offset += *cr.cPos
 	case io.SeekEnd:
-		return 0, fmt.Errorf("Unsupported whence: %v", whence)
+		return 0, fmt.Errorf("unsupported whence: %v", whence)
 	}
 
 	if offset < 0 {
@@ -581,7 +642,7 @@ func (cr *countReader) Seek(offset int64, whence int) (int64, error) {
 	return offset, nil
 }
 
-func (cr *countReader) currentPos() int64 {
+func (cr *countReadSeeker) currentPos() int64 {
 	cr.mu.Lock()
 	defer cr.mu.Unlock()
 

estargz/build_test.go

@@ -28,7 +28,6 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"reflect"
 	"testing"
 )
@@ -413,9 +412,8 @@ func TestSort(t *testing.T) {
 							if err != nil {
 								if err == io.EOF {
 									break
-								} else {
-									t.Fatalf("Failed to parse tar file: %v", err)
 								}
+								t.Fatalf("Failed to parse tar file: %v", err)
 							}
 
 							if !reflect.DeepEqual(gotH, wantH) {
@@ -425,11 +423,11 @@ func TestSort(t *testing.T) {
 
 							}
 
-							got, err := ioutil.ReadAll(gotTar)
+							got, err := io.ReadAll(gotTar)
 							if err != nil {
 								t.Fatal("failed to read got tar payload")
 							}
-							want, err := ioutil.ReadAll(wantTar)
+							want, err := io.ReadAll(wantTar)
 							if err != nil {
 								t.Fatal("failed to read want tar payload")
 							}
@@ -504,19 +502,19 @@ func longstring(size int) (str string) {
 func TestCountReader(t *testing.T) {
 	tests := []struct {
 		name    string
-		ops     func(*countReader) error
+		ops     func(*countReadSeeker) error
 		wantPos int64
 	}{
 		{
 			name: "nop",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				return nil
 			},
 			wantPos: 0,
 		},
 		{
 			name: "read",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				size := 5
 				if _, err := pw.Read(make([]byte, size)); err != nil {
 					return err
@@ -527,7 +525,7 @@ func TestCountReader(t *testing.T) {
 		},
 		{
 			name: "readtwice",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				size1, size2 := 5, 3
 				if _, err := pw.Read(make([]byte, size1)); err != nil {
 					if err != io.EOF {
@@ -545,7 +543,7 @@ func TestCountReader(t *testing.T) {
 		},
 		{
 			name: "seek_start",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				size := int64(5)
 				if _, err := pw.Seek(size, io.SeekStart); err != nil {
 					if err != io.EOF {
@@ -558,7 +556,7 @@ func TestCountReader(t *testing.T) {
 		},
 		{
 			name: "seek_start_twice",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				size1, size2 := int64(5), int64(3)
 				if _, err := pw.Seek(size1, io.SeekStart); err != nil {
 					if err != io.EOF {
@@ -576,7 +574,7 @@ func TestCountReader(t *testing.T) {
 		},
 		{
 			name: "seek_current",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				size := int64(5)
 				if _, err := pw.Seek(size, io.SeekCurrent); err != nil {
 					if err != io.EOF {
@@ -589,7 +587,7 @@ func TestCountReader(t *testing.T) {
 		},
 		{
 			name: "seek_current_twice",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				size1, size2 := int64(5), int64(3)
 				if _, err := pw.Seek(size1, io.SeekCurrent); err != nil {
 					if err != io.EOF {
@@ -607,7 +605,7 @@ func TestCountReader(t *testing.T) {
 		},
 		{
 			name: "seek_current_twice_negative",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				size1, size2 := int64(5), int64(-3)
 				if _, err := pw.Seek(size1, io.SeekCurrent); err != nil {
 					if err != io.EOF {
@@ -625,7 +623,7 @@ func TestCountReader(t *testing.T) {
 		},
 		{
 			name: "mixed",
-			ops: func(pw *countReader) error {
+			ops: func(pw *countReadSeeker) error {
 				size1, size2, size3, size4, size5 := int64(5), int64(-3), int64(4), int64(-1), int64(6)
 				if _, err := pw.Read(make([]byte, size1)); err != nil {
 					if err != io.EOF {
@@ -660,7 +658,7 @@ func TestCountReader(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			pw, err := newCountReader(bytes.NewReader(make([]byte, 100)))
+			pw, err := newCountReadSeeker(bytes.NewReader(make([]byte, 100)))
 			if err != nil {
 				t.Fatalf("failed to make position watcher: %q", err)
 			}

estargz/errorutil/errors_test.go

@@ -17,9 +17,8 @@
 package errorutil
 
 import (
+	"errors"
 	"testing"
-
-	"github.com/pkg/errors"
 )
 
 func TestNoError(t *testing.T) {

estargz/estargz.go

@@ -27,10 +27,10 @@ import (
 	"bytes"
 	"compress/gzip"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"hash"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"sort"
@@ -40,7 +40,6 @@ import (
 
 	"github.com/containerd/stargz-snapshotter/estargz/errorutil"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
 	"github.com/vbatts/tar-split/archive/tar"
 )
 
@@ -107,7 +106,7 @@ type Telemetry struct {
 }
 
 // Open opens a stargz file for reading.
-// The behaviour is configurable using options.
+// The behavior is configurable using options.
 //
 // Note that each entry name is normalized as the path that is relative to root.
 func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) {
@@ -151,10 +150,10 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) {
 			allErr = append(allErr, err)
 			continue
 		}
-		if tocSize <= 0 {
+		if tocOffset >= 0 && tocSize <= 0 {
 			tocSize = sr.Size() - tocOffset - fSize
 		}
-		if tocSize < int64(len(maybeTocBytes)) {
+		if tocOffset >= 0 && tocSize < int64(len(maybeTocBytes)) {
 			maybeTocBytes = maybeTocBytes[:tocSize]
 		}
 		r, err = parseTOC(d, sr, tocOffset, tocSize, maybeTocBytes, opts)
@@ -208,8 +207,16 @@ func (r *Reader) initFields() error {
 	uname := map[int]string{}
 	gname := map[int]string{}
 	var lastRegEnt *TOCEntry
-	for _, ent := range r.toc.Entries {
+	var chunkTopIndex int
+	for i, ent := range r.toc.Entries {
 		ent.Name = cleanEntryName(ent.Name)
+		switch ent.Type {
+		case "reg", "chunk":
+			if ent.Offset != r.toc.Entries[chunkTopIndex].Offset {
+				chunkTopIndex = i
+			}
+			ent.chunkTopIndex = chunkTopIndex
+		}
 		if ent.Type == "reg" {
 			lastRegEnt = ent
 		}
@@ -295,7 +302,7 @@ func (r *Reader) initFields() error {
 		if e.isDataType() {
 			e.nextOffset = lastOffset
 		}
-		if e.Offset != 0 {
+		if e.Offset != 0 && e.InnerOffset == 0 {
 			lastOffset = e.Offset
 		}
 	}
@@ -385,8 +392,7 @@ func (r *Reader) Verifiers() (TOCEntryVerifier, error) {
 			if e.Digest != "" {
 				d, err := digest.Parse(e.Digest)
 				if err != nil {
-					return nil, errors.Wrapf(err,
-						"failed to parse regular file digest %q", e.Digest)
+					return nil, fmt.Errorf("failed to parse regular file digest %q: %w", e.Digest, err)
 				}
 				regDigestMap[e.Offset] = d
 			} else {
@@ -401,8 +407,7 @@ func (r *Reader) Verifiers() (TOCEntryVerifier, error) {
 		if e.ChunkDigest != "" {
 			d, err := digest.Parse(e.ChunkDigest)
 			if err != nil {
-				return nil, errors.Wrapf(err,
-					"failed to parse chunk digest %q", e.ChunkDigest)
+				return nil, fmt.Errorf("failed to parse chunk digest %q: %w", e.ChunkDigest, err)
 			}
 			chunkDigestMap[e.Offset] = d
 		} else {
@@ -491,6 +496,14 @@ func (r *Reader) Lookup(path string) (e *TOCEntry, ok bool) {
 //
 // Name must be absolute path or one that is relative to root.
 func (r *Reader) OpenFile(name string) (*io.SectionReader, error) {
+	fr, err := r.newFileReader(name)
+	if err != nil {
+		return nil, err
+	}
+	return io.NewSectionReader(fr, 0, fr.size), nil
+}
+
+func (r *Reader) newFileReader(name string) (*fileReader, error) {
 	name = cleanEntryName(name)
 	ent, ok := r.Lookup(name)
 	if !ok {
@@ -508,11 +521,19 @@ func (r *Reader) OpenFile(name string) (*io.SectionReader, error) {
 			Err:  errors.New("not a regular file"),
 		}
 	}
-	fr := &fileReader{
+	return &fileReader{
 		r:    r,
 		size: ent.Size,
 		ents: r.getChunks(ent),
+	}, nil
+}
+
+func (r *Reader) OpenFileWithPreReader(name string, preRead func(*TOCEntry, io.Reader) error) (*io.SectionReader, error) {
+	fr, err := r.newFileReader(name)
+	if err != nil {
+		return nil, err
 	}
+	fr.preRead = preRead
 	return io.NewSectionReader(fr, 0, fr.size), nil
 }
 
@@ -524,9 +545,10 @@ func (r *Reader) getChunks(ent *TOCEntry) []*TOCEntry {
 }
 
 type fileReader struct {
-	r    *Reader
-	size int64
-	ents []*TOCEntry // 1 or more reg/chunk entries
+	r       *Reader
+	size    int64
+	ents    []*TOCEntry // 1 or more reg/chunk entries
+	preRead func(*TOCEntry, io.Reader) error
 }
 
 func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) {
@@ -581,10 +603,48 @@ func (fr *fileReader) ReadAt(p []byte, off int64) (n int, err error) {
 		return 0, fmt.Errorf("fileReader.ReadAt.decompressor.Reader: %v", err)
 	}
 	defer dr.Close()
-	if n, err := io.CopyN(ioutil.Discard, dr, off); n != off || err != nil {
-		return 0, fmt.Errorf("discard of %d bytes = %v, %v", off, n, err)
+
+	if fr.preRead == nil {
+		if n, err := io.CopyN(io.Discard, dr, ent.InnerOffset+off); n != ent.InnerOffset+off || err != nil {
+			return 0, fmt.Errorf("discard of %d bytes != %v, %v", ent.InnerOffset+off, n, err)
+		}
+		return io.ReadFull(dr, p)
 	}
-	return io.ReadFull(dr, p)
+
+	var retN int
+	var retErr error
+	var found bool
+	var nr int64
+	for _, e := range fr.r.toc.Entries[ent.chunkTopIndex:] {
+		if !e.isDataType() {
+			continue
+		}
+		if e.Offset != fr.r.toc.Entries[ent.chunkTopIndex].Offset {
+			break
+		}
+		if in, err := io.CopyN(io.Discard, dr, e.InnerOffset-nr); err != nil || in != e.InnerOffset-nr {
+			return 0, fmt.Errorf("discard of remaining %d bytes != %v, %v", e.InnerOffset-nr, in, err)
+		}
+		nr = e.InnerOffset
+		if e == ent {
+			found = true
+			if n, err := io.CopyN(io.Discard, dr, off); n != off || err != nil {
+				return 0, fmt.Errorf("discard of offset %d bytes != %v, %v", off, n, err)
+			}
+			retN, retErr = io.ReadFull(dr, p)
+			nr += off + int64(retN)
+			continue
+		}
+		cr := &countReader{r: io.LimitReader(dr, e.ChunkSize)}
+		if err := fr.preRead(e, cr); err != nil {
+			return 0, fmt.Errorf("failed to pre read: %w", err)
+		}
+		nr += cr.n
+	}
+	if !found {
+		return 0, fmt.Errorf("fileReader.ReadAt: target entry not found")
+	}
+	return retN, retErr
 }
 
 // A Writer writes stargz files.
@@ -602,11 +662,20 @@ type Writer struct {
 	lastGroupname map[int]string
 	compressor    Compressor
 
+	uncompressedCounter *countWriteFlusher
+
 	// ChunkSize optionally controls the maximum number of bytes
 	// of data of a regular file that can be written in one gzip
 	// stream before a new gzip stream is started.
 	// Zero means to use a default, currently 4 MiB.
 	ChunkSize int
+
+	// MinChunkSize optionally controls the minimum number of bytes
+	// of data must be written in one gzip stream before a new gzip
+	// NOTE: This adds a TOC property that stargz snapshotter < v0.13.0 doesn't understand.
+	MinChunkSize int
+
+	needsOpenGzEntries map[string]struct{}
 }
 
 // currentCompressionWriter writes to the current w.gz field, which can
@@ -647,7 +716,10 @@ func Unpack(sr *io.SectionReader, c Decompressor) (io.ReadCloser, error) {
 	}
 	blobPayloadSize, _, _, err := c.ParseFooter(footer)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse footer")
+		return nil, fmt.Errorf("failed to parse footer: %w", err)
+	}
+	if blobPayloadSize < 0 {
+		blobPayloadSize = sr.Size()
 	}
 	return c.Reader(io.LimitReader(sr, blobPayloadSize))
 }
@@ -675,11 +747,12 @@ func NewWriterWithCompressor(w io.Writer, c Compressor) *Writer {
 	bw := bufio.NewWriter(w)
 	cw := &countWriter{w: bw}
 	return &Writer{
-		bw:         bw,
-		cw:         cw,
-		toc:        &JTOC{Version: 1},
-		diffHash:   sha256.New(),
-		compressor: c,
+		bw:                  bw,
+		cw:                  cw,
+		toc:                 &JTOC{Version: 1},
+		diffHash:            sha256.New(),
+		compressor:          c,
+		uncompressedCounter: &countWriteFlusher{},
 	}
 }
 
@@ -720,6 +793,20 @@ func (w *Writer) closeGz() error {
 	return nil
 }
 
+func (w *Writer) flushGz() error {
+	if w.closed {
+		return errors.New("flush on closed Writer")
+	}
+	if w.gz != nil {
+		if f, ok := w.gz.(interface {
+			Flush() error
+		}); ok {
+			return f.Flush()
+		}
+	}
+	return nil
+}
+
 // nameIfChanged returns name, unless it was the already the value of (*mp)[id],
 // in which case it returns the empty string.
 func (w *Writer) nameIfChanged(mp *map[int]string, id int, name string) string {
@@ -739,6 +826,9 @@ func (w *Writer) nameIfChanged(mp *map[int]string, id int, name string) string {
 func (w *Writer) condOpenGz() (err error) {
 	if w.gz == nil {
 		w.gz, err = w.compressor.Writer(w.cw)
+		if w.gz != nil {
+			w.gz = w.uncompressedCounter.register(w.gz)
+		}
 	}
 	return
 }
@@ -787,6 +877,8 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
 	if lossless {
 		tr.RawAccounting = true
 	}
+	prevOffset := w.cw.n
+	var prevOffsetUncompressed int64
 	for {
 		h, err := tr.Next()
 		if err == io.EOF {
@@ -886,10 +978,6 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
 			totalSize := ent.Size // save it before we destroy ent
 			tee := io.TeeReader(tr, payloadDigest.Hash())
 			for written < totalSize {
-				if err := w.closeGz(); err != nil {
-					return err
-				}
-
 				chunkSize := int64(w.chunkSize())
 				remain := totalSize - written
 				if remain < chunkSize {
@@ -897,7 +985,23 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
 				} else {
 					ent.ChunkSize = chunkSize
 				}
-				ent.Offset = w.cw.n
+
+				// We flush the underlying compression writer here to correctly calculate "w.cw.n".
+				if err := w.flushGz(); err != nil {
+					return err
+				}
+				if w.needsOpenGz(ent) || w.cw.n-prevOffset >= int64(w.MinChunkSize) {
+					if err := w.closeGz(); err != nil {
+						return err
+					}
+					ent.Offset = w.cw.n
+					prevOffset = ent.Offset
+					prevOffsetUncompressed = w.uncompressedCounter.n
+				} else {
+					ent.Offset = prevOffset
+					ent.InnerOffset = w.uncompressedCounter.n - prevOffsetUncompressed
+				}
+
 				ent.ChunkOffset = written
 				chunkDigest := digest.Canonical.Digester()
 
@@ -935,7 +1039,7 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
 			}
 		}
 	}
-	remainDest := ioutil.Discard
+	remainDest := io.Discard
 	if lossless {
 		remainDest = dst // Preserve the remaining bytes in lossless mode
 	}
@@ -943,6 +1047,17 @@ func (w *Writer) appendTar(r io.Reader, lossless bool) error {
 	return err
 }
 
+func (w *Writer) needsOpenGz(ent *TOCEntry) bool {
+	if ent.Type != "reg" {
+		return false
+	}
+	if w.needsOpenGzEntries == nil {
+		return false
+	}
+	_, ok := w.needsOpenGzEntries[ent.Name]
+	return ok
+}
+
 // DiffID returns the SHA-256 of the uncompressed tar bytes.
 // It is only valid to call DiffID after Close.
 func (w *Writer) DiffID() string {
@@ -959,6 +1074,28 @@ func maxFooterSize(blobSize int64, decompressors ...Decompressor) (res int64) {
 }
 
 func parseTOC(d Decompressor, sr *io.SectionReader, tocOff, tocSize int64, tocBytes []byte, opts openOpts) (*Reader, error) {
+	if tocOff < 0 {
+		// This means that TOC isn't contained in the blob.
+		// We pass nil reader to ParseTOC and expect that ParseTOC acquire TOC from
+		// the external location.
+		start := time.Now()
+		toc, tocDgst, err := d.ParseTOC(nil)
+		if err != nil {
+			return nil, err
+		}
+		if opts.telemetry != nil && opts.telemetry.GetTocLatency != nil {
+			opts.telemetry.GetTocLatency(start)
+		}
+		if opts.telemetry != nil && opts.telemetry.DeserializeTocLatency != nil {
+			opts.telemetry.DeserializeTocLatency(start)
+		}
+		return &Reader{
+			sr:           sr,
+			toc:          toc,
+			tocDigest:    tocDgst,
+			decompressor: d,
+		}, nil
+	}
 	if len(tocBytes) > 0 {
 		start := time.Now()
 		toc, tocDgst, err := d.ParseTOC(bytes.NewReader(tocBytes))
@@ -1024,6 +1161,37 @@ func (cw *countWriter) Write(p []byte) (n int, err error) {
 	return
 }
 
+type countWriteFlusher struct {
+	io.WriteCloser
+	n int64
+}
+
+func (wc *countWriteFlusher) register(w io.WriteCloser) io.WriteCloser {
+	wc.WriteCloser = w
+	return wc
+}
+
+func (wc *countWriteFlusher) Write(p []byte) (n int, err error) {
+	n, err = wc.WriteCloser.Write(p)
+	wc.n += int64(n)
+	return
+}
+
+func (wc *countWriteFlusher) Flush() error {
+	if f, ok := wc.WriteCloser.(interface {
+		Flush() error
+	}); ok {
+		return f.Flush()
+	}
+	return nil
+}
+
+func (wc *countWriteFlusher) Close() error {
+	err := wc.WriteCloser.Close()
+	wc.WriteCloser = nil
+	return err
+}
+
 // isGzip reports whether br is positioned right before an upcoming gzip stream.
 // It does not consume any bytes from br.
 func isGzip(br *bufio.Reader) bool {
@@ -1042,3 +1210,14 @@ func positive(n int64) int64 {
 	}
 	return n
 }
+
+type countReader struct {
+	r io.Reader
+	n int64
+}
+
+func (cr *countReader) Read(p []byte) (n int, err error) {
+	n, err = cr.r.Read(p)
+	cr.n += int64(n)
+	return
+}

estargz/estargz_test.go

@@ -81,7 +81,7 @@ func TestChunkEntryForOffset(t *testing.T) {
 			if ok != te.wantOk {
 				t.Errorf("ok = %v; want (%v)", ok, te.wantOk)
 			} else if ok {
-				if !(ce.ChunkOffset == te.wantChunkOffset && ce.ChunkSize == te.wantChunkSize) {
+				if ce.ChunkOffset != te.wantChunkOffset || ce.ChunkSize != te.wantChunkSize {
 					t.Errorf("chunkOffset = %d, ChunkSize = %d; want (chunkOffset = %d, chunkSize = %d)",
 						ce.ChunkOffset, ce.ChunkSize, te.wantChunkOffset, te.wantChunkSize)
 				}

estargz/externaltoc/externaltoc.go

@@ -0,0 +1,278 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright 2019 The Go Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style
+   license that can be found in the LICENSE file.
+*/
+
+package externaltoc
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"hash"
+	"io"
+	"sync"
+
+	"github.com/containerd/stargz-snapshotter/estargz"
+	digest "github.com/opencontainers/go-digest"
+)
+
+type GzipCompression struct {
+	*GzipCompressor
+	*GzipDecompressor
+}
+
+func NewGzipCompressionWithLevel(provideTOC func() ([]byte, error), level int) estargz.Compression {
+	return &GzipCompression{
+		NewGzipCompressorWithLevel(level),
+		NewGzipDecompressor(provideTOC),
+	}
+}
+
+func NewGzipCompressor() *GzipCompressor {
+	return &GzipCompressor{compressionLevel: gzip.BestCompression}
+}
+
+func NewGzipCompressorWithLevel(level int) *GzipCompressor {
+	return &GzipCompressor{compressionLevel: level}
+}
+
+type GzipCompressor struct {
+	compressionLevel int
+	buf              *bytes.Buffer
+}
+
+func (gc *GzipCompressor) WriteTOCTo(w io.Writer) (int, error) {
+	if len(gc.buf.Bytes()) == 0 {
+		return 0, fmt.Errorf("TOC hasn't been registered")
+	}
+	return w.Write(gc.buf.Bytes())
+}
+
+func (gc *GzipCompressor) Writer(w io.Writer) (estargz.WriteFlushCloser, error) {
+	return gzip.NewWriterLevel(w, gc.compressionLevel)
+}
+
+func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *estargz.JTOC, diffHash hash.Hash) (digest.Digest, error) {
+	tocJSON, err := json.MarshalIndent(toc, "", "\t")
+	if err != nil {
+		return "", err
+	}
+	buf := new(bytes.Buffer)
+	gz, _ := gzip.NewWriterLevel(buf, gc.compressionLevel)
+	// TOC isn't written to layer so no effect to diff ID
+	tw := tar.NewWriter(gz)
+	if err := tw.WriteHeader(&tar.Header{
+		Typeflag: tar.TypeReg,
+		Name:     estargz.TOCTarName,
+		Size:     int64(len(tocJSON)),
+	}); err != nil {
+		return "", err
+	}
+	if _, err := tw.Write(tocJSON); err != nil {
+		return "", err
+	}
+
+	if err := tw.Close(); err != nil {
+		return "", err
+	}
+	if err := gz.Close(); err != nil {
+		return "", err
+	}
+	gc.buf = buf
+	footerBytes, err := gzipFooterBytes()
+	if err != nil {
+		return "", err
+	}
+	if _, err := w.Write(footerBytes); err != nil {
+		return "", err
+	}
+	return digest.FromBytes(tocJSON), nil
+}
+
+// The footer is an empty gzip stream with no compression and an Extra header.
+//
+// 46 comes from:
+//
+// 10 bytes  gzip header
+// 2  bytes  XLEN (length of Extra field) = 21 (4 bytes header + len("STARGZEXTERNALTOC"))
+// 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
+// 2  bytes  Extra: LEN = 17 (len("STARGZEXTERNALTOC"))
+// 17 bytes  Extra: subfield = "STARGZEXTERNALTOC"
+// 5  bytes  flate header
+// 8  bytes  gzip footer
+// (End of the eStargz blob)
+const FooterSize = 46
+
+// gzipFooterBytes returns the 104 bytes footer.
+func gzipFooterBytes() ([]byte, error) {
+	buf := bytes.NewBuffer(make([]byte, 0, FooterSize))
+	gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes
+
+	// Extra header indicating the offset of TOCJSON
+	// https://tools.ietf.org/html/rfc1952#section-2.3.1.1
+	header := make([]byte, 4)
+	header[0], header[1] = 'S', 'G'
+	subfield := "STARGZEXTERNALTOC"                                   // len("STARGZEXTERNALTOC") = 17
+	binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952
+	gz.Extra = append(header, []byte(subfield)...)
+	if err := gz.Close(); err != nil {
+		return nil, err
+	}
+	if buf.Len() != FooterSize {
+		panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize))
+	}
+	return buf.Bytes(), nil
+}
+
+func NewGzipDecompressor(provideTOCFunc func() ([]byte, error)) *GzipDecompressor {
+	return &GzipDecompressor{provideTOCFunc: provideTOCFunc}
+}
+
+type GzipDecompressor struct {
+	provideTOCFunc func() ([]byte, error)
+	rawTOC         []byte // Do not access this field directly. Get this through getTOC() method.
+	getTOCOnce     sync.Once
+}
+
+func (gz *GzipDecompressor) getTOC() ([]byte, error) {
+	if len(gz.rawTOC) == 0 {
+		var retErr error
+		gz.getTOCOnce.Do(func() {
+			if gz.provideTOCFunc == nil {
+				retErr = fmt.Errorf("TOC hasn't been provided")
+				return
+			}
+			rawTOC, err := gz.provideTOCFunc()
+			if err != nil {
+				retErr = err
+				return
+			}
+			gz.rawTOC = rawTOC
+		})
+		if retErr != nil {
+			return nil, retErr
+		}
+		if len(gz.rawTOC) == 0 {
+			return nil, fmt.Errorf("no TOC is provided")
+		}
+	}
+	return gz.rawTOC, nil
+}
+
+func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
+	return gzip.NewReader(r)
+}
+
+func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *estargz.JTOC, tocDgst digest.Digest, err error) {
+	if r != nil {
+		return nil, "", fmt.Errorf("TOC must be provided externally but got internal one")
+	}
+	rawTOC, err := gz.getTOC()
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to get TOC: %v", err)
+	}
+	return parseTOCEStargz(bytes.NewReader(rawTOC))
+}
+
+func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
+	if len(p) != FooterSize {
+		return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p))
+	}
+	zr, err := gzip.NewReader(bytes.NewReader(p))
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	defer zr.Close()
+	extra := zr.Extra
+	si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
+	if si1 != 'S' || si2 != 'G' {
+		return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
+	}
+	if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(len("STARGZEXTERNALTOC")) {
+		return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ"))
+	}
+	if string(subfield) != "STARGZEXTERNALTOC" {
+		return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield")
+	}
+	// tocOffset < 0 indicates external TOC.
+	// blobPayloadSize < 0 indicates the entire blob size.
+	return -1, -1, 0, nil
+}
+
+func (gz *GzipDecompressor) FooterSize() int64 {
+	return FooterSize
+}
+
+func (gz *GzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
+	if r != nil {
+		return nil, fmt.Errorf("TOC must be provided externally but got internal one")
+	}
+	rawTOC, err := gz.getTOC()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get TOC: %v", err)
+	}
+	return decompressTOCEStargz(bytes.NewReader(rawTOC))
+}
+
+func parseTOCEStargz(r io.Reader) (toc *estargz.JTOC, tocDgst digest.Digest, err error) {
+	tr, err := decompressTOCEStargz(r)
+	if err != nil {
+		return nil, "", err
+	}
+	dgstr := digest.Canonical.Digester()
+	toc = new(estargz.JTOC)
+	if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil {
+		return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err)
+	}
+	if err := tr.Close(); err != nil {
+		return nil, "", err
+	}
+	return toc, dgstr.Digest(), nil
+}
+
+func decompressTOCEStargz(r io.Reader) (tocJSON io.ReadCloser, err error) {
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("malformed TOC gzip header: %v", err)
+	}
+	zr.Multistream(false)
+	tr := tar.NewReader(zr)
+	h, err := tr.Next()
+	if err != nil {
+		return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err)
+	}
+	if h.Name != estargz.TOCTarName {
+		return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, estargz.TOCTarName)
+	}
+	return readCloser{tr, zr.Close}, nil
+}
+
+type readCloser struct {
+	io.Reader
+	closeFunc func() error
+}
+
+func (rc readCloser) Close() error {
+	return rc.closeFunc()
+}

estargz/externaltoc/externaltoc_test.go

@@ -0,0 +1,102 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package externaltoc
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"testing"
+
+	"github.com/containerd/stargz-snapshotter/estargz"
+)
+
+// TestGzipEStargz tests gzip-based external TOC eStargz
+func TestGzipEStargz(t *testing.T) {
+	testRunner := &estargz.TestRunner{
+		TestingT: t,
+		Runner: func(testingT estargz.TestingT, name string, run func(t estargz.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	estargz.CompressionTestSuite(testRunner,
+		gzipControllerWithLevel(gzip.NoCompression),
+		gzipControllerWithLevel(gzip.BestSpeed),
+		gzipControllerWithLevel(gzip.BestCompression),
+		gzipControllerWithLevel(gzip.DefaultCompression),
+		gzipControllerWithLevel(gzip.HuffmanOnly),
+	)
+}
+
+func gzipControllerWithLevel(compressionLevel int) estargz.TestingControllerFactory {
+	return func() estargz.TestingController {
+		compressor := NewGzipCompressorWithLevel(compressionLevel)
+		decompressor := NewGzipDecompressor(func() ([]byte, error) {
+			buf := new(bytes.Buffer)
+			if _, err := compressor.WriteTOCTo(buf); err != nil {
+				return nil, err
+			}
+			return buf.Bytes(), nil
+		})
+		return &gzipController{compressor, decompressor}
+	}
+}
+
+type gzipController struct {
+	*GzipCompressor
+	*GzipDecompressor
+}
+
+func (gc *gzipController) String() string {
+	return fmt.Sprintf("externaltoc_gzip_compression_level=%v", gc.compressionLevel)
+}
+
+// TestStream tests the passed estargz blob contains the specified list of streams.
+func (gc *gzipController) TestStreams(t estargz.TestingT, b []byte, streams []int64) {
+	estargz.CheckGzipHasStreams(t, b, streams)
+}
+
+func (gc *gzipController) DiffIDOf(t estargz.TestingT, b []byte) string {
+	return estargz.GzipDiffIDOf(t, b)
+}
+
+// Tests footer encoding, size, and parsing of gzip-based eStargz.
+func TestGzipFooter(t *testing.T) {
+	footer, err := gzipFooterBytes()
+	if err != nil {
+		t.Fatalf("failed gzipFooterBytes: %v", err)
+	}
+	if len(footer) != FooterSize {
+		t.Fatalf("footer length was %d, not expected %d. got bytes: %q", len(footer), FooterSize, footer)
+	}
+	_, gotTOCOffset, _, err := (&GzipDecompressor{}).ParseFooter(footer)
+	if err != nil {
+		t.Fatalf("failed to parse footer, footer: %x: err: %v", footer, err)
+	}
+	if gotTOCOffset != -1 {
+		t.Fatalf("ParseFooter(footerBytes) must return -1 for external toc but got %d", gotTOCOffset)
+	}
+}

estargz/go.mod

@@ -1,11 +1,10 @@
 module github.com/containerd/stargz-snapshotter/estargz
 
-go 1.16
+go 1.23.0
 
 require (
-	github.com/klauspost/compress v1.13.6
+	github.com/klauspost/compress v1.18.0
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/pkg/errors v0.9.1
-	github.com/vbatts/tar-split v0.11.2
-	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
+	github.com/vbatts/tar-split v0.12.1
+	golang.org/x/sync v0.16.0
 )

estargz/go.sum

@@ -1,22 +1,8 @@
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
-github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
-github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo=
+github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=

estargz/gzip.go

@@ -34,7 +34,6 @@ import (
 	"strconv"
 
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
 )
 
 type gzipCompression struct {
@@ -61,7 +60,7 @@ type GzipCompressor struct {
 	compressionLevel int
 }
 
-func (gc *GzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) {
+func (gc *GzipCompressor) Writer(w io.Writer) (WriteFlushCloser, error) {
 	return gzip.NewWriterLevel(w, gc.compressionLevel)
 }
 
@@ -110,7 +109,7 @@ func gzipFooterBytes(tocOff int64) []byte {
 	header[0], header[1] = 'S', 'G'
 	subfield := fmt.Sprintf("%016xSTARGZ", tocOff)
 	binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952
-	gz.Header.Extra = append(header, []byte(subfield)...)
+	gz.Extra = append(header, []byte(subfield)...)
 	gz.Close()
 	if buf.Len() != FooterSize {
 		panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize))
@@ -137,7 +136,7 @@ func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, t
 		return 0, 0, 0, err
 	}
 	defer zr.Close()
-	extra := zr.Header.Extra
+	extra := zr.Extra
 	si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
 	if si1 != 'S' || si2 != 'G' {
 		return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
@@ -150,7 +149,7 @@ func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, t
 	}
 	tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64)
 	if err != nil {
-		return 0, 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset")
+		return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err)
 	}
 	return tocOffset, tocOffset, 0, nil
 }
@@ -179,10 +178,10 @@ func (gz *LegacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOff
 	}
 	zr, err := gzip.NewReader(bytes.NewReader(p))
 	if err != nil {
-		return 0, 0, 0, errors.Wrapf(err, "legacy: failed to get footer gzip reader")
+		return 0, 0, 0, fmt.Errorf("legacy: failed to get footer gzip reader: %w", err)
 	}
 	defer zr.Close()
-	extra := zr.Header.Extra
+	extra := zr.Extra
 	if len(extra) != 16+len("STARGZ") {
 		return 0, 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size")
 	}
@@ -191,7 +190,7 @@ func (gz *LegacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOff
 	}
 	tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64)
 	if err != nil {
-		return 0, 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset")
+		return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err)
 	}
 	return tocOffset, tocOffset, 0, nil
 }

estargz/gzip_test.go

@@ -25,16 +25,28 @@ package estargz
 import (
 	"bytes"
 	"compress/gzip"
-	"crypto/sha256"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"testing"
 )
 
 // TestGzipEStargz tests gzip-based eStargz
 func TestGzipEStargz(t *testing.T) {
-	CompressionTestSuite(t,
+	testRunner := &TestRunner{
+		TestingT: t,
+		Runner: func(testingT TestingT, name string, run func(t TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	CompressionTestSuite(testRunner,
 		gzipControllerWithLevel(gzip.NoCompression),
 		gzipControllerWithLevel(gzip.BestSpeed),
 		gzipControllerWithLevel(gzip.BestCompression),
@@ -43,8 +55,10 @@ func TestGzipEStargz(t *testing.T) {
 	)
 }
 
-func gzipControllerWithLevel(compressionLevel int) TestingController {
-	return &gzipController{&GzipCompressor{compressionLevel}, &GzipDecompressor{}}
+func gzipControllerWithLevel(compressionLevel int) TestingControllerFactory {
+	return func() TestingController {
+		return &gzipController{&GzipCompressor{compressionLevel}, &GzipDecompressor{}}
+	}
 }
 
 type gzipController struct {
@@ -53,47 +67,16 @@ type gzipController struct {
 }
 
 func (gc *gzipController) String() string {
-	return fmt.Sprintf("gzip_compression_level=%v", gc.GzipCompressor.compressionLevel)
+	return fmt.Sprintf("gzip_compression_level=%v", gc.compressionLevel)
 }
 
-func (gc *gzipController) CountStreams(t *testing.T, b []byte) (numStreams int) {
-	len0 := len(b)
-	br := bytes.NewReader(b)
-	zr := new(gzip.Reader)
-	t.Logf("got gzip streams:")
-	for {
-		zoff := len0 - br.Len()
-		if err := zr.Reset(br); err != nil {
-			if err == io.EOF {
-				return
-			}
-			t.Fatalf("countStreams(gzip), Reset: %v", err)
-		}
-		zr.Multistream(false)
-		n, err := io.Copy(ioutil.Discard, zr)
-		if err != nil {
-			t.Fatalf("countStreams(gzip), Copy: %v", err)
-		}
-		var extra string
-		if len(zr.Header.Extra) > 0 {
-			extra = fmt.Sprintf("; extra=%q", zr.Header.Extra)
-		}
-		t.Logf("  [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra)
-		numStreams++
-	}
+// TestStream tests the passed estargz blob contains the specified list of streams.
+func (gc *gzipController) TestStreams(t TestingT, b []byte, streams []int64) {
+	CheckGzipHasStreams(t, b, streams)
 }
 
-func (gc *gzipController) DiffIDOf(t *testing.T, b []byte) string {
-	h := sha256.New()
-	zr, err := gzip.NewReader(bytes.NewReader(b))
-	if err != nil {
-		t.Fatalf("diffIDOf(gzip): %v", err)
-	}
-	defer zr.Close()
-	if _, err := io.Copy(h, zr); err != nil {
-		t.Fatalf("diffIDOf(gzip).Copy: %v", err)
-	}
-	return fmt.Sprintf("sha256:%x", h.Sum(nil))
+func (gc *gzipController) DiffIDOf(t TestingT, b []byte) string {
+	return GzipDiffIDOf(t, b)
 }
 
 // Tests footer encoding, size, and parsing of gzip-based eStargz.
@@ -138,7 +121,7 @@ func checkLegacyFooter(t *testing.T, off int64) {
 func legacyFooterBytes(tocOff int64) []byte {
 	buf := bytes.NewBuffer(make([]byte, 0, legacyFooterSize))
 	gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression)
-	gz.Header.Extra = []byte(fmt.Sprintf("%016xSTARGZ", tocOff))
+	gz.Extra = []byte(fmt.Sprintf("%016xSTARGZ", tocOff))
 	gz.Close()
 	if buf.Len() != legacyFooterSize {
 		panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), legacyFooterSize))

estargz/types.go

@@ -149,6 +149,12 @@ type TOCEntry struct {
 	// ChunkSize.
 	Offset int64 `json:"offset,omitempty"`
 
+	// InnerOffset is an optional field indicates uncompressed offset
+	// of this "reg" or "chunk" payload in a stream starts from Offset.
+	// This field enables to put multiple "reg" or "chunk" payloads
+	// in one chunk with having the same Offset but different InnerOffset.
+	InnerOffset int64 `json:"innerOffset,omitempty"`
+
 	nextOffset int64 // the Offset of the next entry with a non-zero Offset
 
 	// DevMajor is the major device number for "char" and "block" types.
@@ -159,7 +165,8 @@ type TOCEntry struct {
 
 	// NumLink is the number of entry names pointing to this entry.
 	// Zero means one name references this entry.
-	NumLink int
+	// This field is calculated during runtime and not recorded in TOC JSON.
+	NumLink int `json:"-"`
 
 	// Xattrs are the extended attribute for the entry.
 	Xattrs map[string][]byte `json:"xattrs,omitempty"`
@@ -185,6 +192,9 @@ type TOCEntry struct {
 	ChunkDigest string `json:"chunkDigest,omitempty"`
 
 	children map[string]*TOCEntry
+
+	// chunkTopIndex is index of the entry where Offset starts in the blob.
+	chunkTopIndex int
 }
 
 // ModTime returns the entry's modification time.
@@ -278,7 +288,10 @@ type Compressor interface {
 	// Writer returns WriteCloser to be used for writing a chunk to eStargz.
 	// Everytime a chunk is written, the WriteCloser is closed and Writer is
 	// called again for writing the next chunk.
-	Writer(w io.Writer) (io.WriteCloser, error)
+	//
+	// The returned writer should implement "Flush() error" function that flushes
+	// any pending compressed data to the underlying writer.
+	Writer(w io.Writer) (WriteFlushCloser, error)
 
 	// WriteTOCAndFooter is called to write JTOC to the passed Writer.
 	// diffHash calculates the DiffID (uncompressed sha256 hash) of the blob
@@ -302,8 +315,12 @@ type Decompressor interface {
 	// payloadBlobSize is the (compressed) size of the blob payload (i.e. the size between
 	// the top until the TOC JSON).
 	//
-	// Here, tocSize is optional. If tocSize <= 0, it's by default the size of the range
-	// from tocOffset until the beginning of the footer (blob size - tocOff - FooterSize).
+	// If tocOffset < 0, we assume that TOC isn't contained in the blob and pass nil reader
+	// to ParseTOC. We expect that ParseTOC acquire TOC from the external location and return it.
+	//
+	// tocSize is optional. If tocSize <= 0, it's by default the size of the range from tocOffset until the beginning of the
+	// footer (blob size - tocOff - FooterSize).
+	// If blobPayloadSize < 0, blobPayloadSize become the blob size.
 	ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error)
 
 	// ParseTOC parses TOC from the passed reader. The reader provides the partial contents
@@ -312,5 +329,14 @@ type Decompressor interface {
 	// This function returns tocDgst that represents the digest of TOC that will be used
 	// to verify this blob. This must match to the value returned from
 	// Compressor.WriteTOCAndFooter that is used when creating this blob.
+	//
+	// If tocOffset returned by ParseFooter is < 0, we assume that TOC isn't contained in the blob.
+	// Pass nil reader to ParseTOC then we expect that ParseTOC acquire TOC from the external location
+	// and return it.
 	ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error)
 }
+
+type WriteFlushCloser interface {
+	io.WriteCloser
+	Flush() error
+}

estargz/zstdchunked/zstdchunked.go

@@ -29,7 +29,6 @@ import (
 	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/klauspost/compress/zstd"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -70,7 +69,7 @@ func (zz *Decompressor) ParseTOC(r io.Reader) (toc *estargz.JTOC, tocDgst digest
 	dgstr := digest.Canonical.Digester()
 	toc = new(estargz.JTOC)
 	if err := json.NewDecoder(io.TeeReader(zr, dgstr.Hash())).Decode(&toc); err != nil {
-		return nil, "", errors.Wrap(err, "error decoding TOC JSON")
+		return nil, "", fmt.Errorf("error decoding TOC JSON: %w", err)
 	}
 	return toc, dgstr.Digest(), nil
 }
@@ -122,7 +121,7 @@ type Compressor struct {
 	pool sync.Pool
 }
 
-func (zc *Compressor) Writer(w io.Writer) (io.WriteCloser, error) {
+func (zc *Compressor) Writer(w io.Writer) (estargz.WriteFlushCloser, error) {
 	if wc := zc.pool.Get(); wc != nil {
 		ec := wc.(*zstd.Encoder)
 		ec.Reset(w)

estargz/zstdchunked/zstdchunked_test.go

@@ -19,10 +19,9 @@ package zstdchunked
 import (
 	"bytes"
 	"crypto/sha256"
-	"encoding/binary"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"sort"
 	"testing"
 
 	"github.com/containerd/stargz-snapshotter/estargz"
@@ -31,7 +30,22 @@ import (
 
 // TestZstdChunked tests zstd:chunked
 func TestZstdChunked(t *testing.T) {
-	estargz.CompressionTestSuite(t,
+	testRunner := &estargz.TestRunner{
+		TestingT: t,
+		Runner: func(testingT estargz.TestingT, name string, run func(t estargz.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	estargz.CompressionTestSuite(testRunner,
 		zstdControllerWithLevel(zstd.SpeedFastest),
 		zstdControllerWithLevel(zstd.SpeedDefault),
 		zstdControllerWithLevel(zstd.SpeedBetterCompression),
@@ -39,8 +53,10 @@ func TestZstdChunked(t *testing.T) {
 	)
 }
 
-func zstdControllerWithLevel(compressionLevel zstd.EncoderLevel) estargz.TestingController {
-	return &zstdController{&Compressor{CompressionLevel: compressionLevel}, &Decompressor{}}
+func zstdControllerWithLevel(compressionLevel zstd.EncoderLevel) estargz.TestingControllerFactory {
+	return func() estargz.TestingController {
+		return &zstdController{&Compressor{CompressionLevel: compressionLevel}, &Decompressor{}}
+	}
 }
 
 type zstdController struct {
@@ -49,20 +65,39 @@ type zstdController struct {
 }
 
 func (zc *zstdController) String() string {
-	return fmt.Sprintf("zstd_compression_level=%v", zc.Compressor.CompressionLevel)
+	return fmt.Sprintf("zstd_compression_level=%v", zc.CompressionLevel)
 }
 
-func (zc *zstdController) CountStreams(t *testing.T, b []byte) (numStreams int) {
+// TestStream tests the passed zstdchunked blob contains the specified list of streams.
+// The last entry of streams must be the offset of footer (len(b) - footerSize).
+func (zc *zstdController) TestStreams(t estargz.TestingT, b []byte, streams []int64) {
 	t.Logf("got zstd streams (compressed size: %d):", len(b))
-	zh := new(zstd.Header)
+
+	if len(streams) == 0 {
+		return // nop
+	}
+
+	// We expect the last offset is footer offset.
+	// 8 is the size of the zstd skippable frame header + the frame size (see WriteTOCAndFooter)
+	sort.Slice(streams, func(i, j int) bool {
+		return streams[i] < streams[j]
+	})
+	streams[len(streams)-1] = streams[len(streams)-1] - 8
+	wants := map[int64]struct{}{}
+	for _, s := range streams {
+		wants[s] = struct{}{}
+	}
+
 	magicLen := 4 // length of magic bytes and skippable frame magic bytes
 	zoff := 0
+	numStreams := 0
 	for {
 		if len(b) <= zoff {
 			break
 		} else if len(b)-zoff <= magicLen {
 			t.Fatalf("invalid frame size %d is too small", len(b)-zoff)
 		}
+		delete(wants, int64(zoff)) // offset found
 		remainingFrames := b[zoff:]
 
 		// Check if zoff points to the beginning of a frame
@@ -71,73 +106,24 @@ func (zc *zstdController) CountStreams(t *testing.T, b []byte) (numStreams int)
 				t.Fatalf("frame must start from magic bytes; but %x",
 					remainingFrames[:magicLen])
 			}
-
-			// This is a skippable frame
-			size := binary.LittleEndian.Uint32(remainingFrames[magicLen : magicLen+4])
-			t.Logf("  [%d] at %d in stargz, SKIPPABLE FRAME (nextFrame: %d/%d)",
-				numStreams, zoff, zoff+(magicLen+4+int(size)), len(b))
-			zoff += (magicLen + 4 + int(size))
-			numStreams++
-			continue
 		}
-
-		// Parse header and get uncompressed size of this frame
-		if err := zh.Decode(remainingFrames); err != nil {
-			t.Fatalf("countStreams(zstd), *Header.Decode: %v", err)
-		}
-		uncompressedFrameSize := zh.FrameContentSize
-		if uncompressedFrameSize == 0 {
-			// FrameContentSize is optional so it's possible we cannot get size info from
-			// this field. If this frame contains only one block, we can get the decompressed
-			// size from that block header.
-			if zh.FirstBlock.OK && zh.FirstBlock.Last && !zh.FirstBlock.Compressed {
-				uncompressedFrameSize = uint64(zh.FirstBlock.DecompressedSize)
-			} else {
-				t.Fatalf("countStreams(zstd), failed to get uncompressed frame size")
+		searchBase := magicLen
+		nextMagicIdx := nextIndex(remainingFrames[searchBase:], zstdFrameMagic)
+		nextSkippableIdx := nextIndex(remainingFrames[searchBase:], skippableFrameMagic)
+		nextFrame := len(remainingFrames)
+		for _, i := range []int{nextMagicIdx, nextSkippableIdx} {
+			if 0 < i && searchBase+i < nextFrame {
+				nextFrame = searchBase + i
 			}
 		}
-
-		// Identify the offset of the next frame
-		nextFrame := magicLen // ignore the magic bytes of this frame
-		for {
-			// search for the beginning magic bytes of the next frame
-			searchBase := nextFrame
-			nextMagicIdx := nextIndex(remainingFrames[searchBase:], zstdFrameMagic)
-			nextSkippableIdx := nextIndex(remainingFrames[searchBase:], skippableFrameMagic)
-			nextFrame = len(remainingFrames)
-			for _, i := range []int{nextMagicIdx, nextSkippableIdx} {
-				if 0 < i && searchBase+i < nextFrame {
-					nextFrame = searchBase + i
-				}
-			}
-
-			// "nextFrame" seems the offset of the next frame. Verify it by checking if
-			// the decompressed size of this frame is the same value as set in the header.
-			zr, err := zstd.NewReader(bytes.NewReader(remainingFrames[:nextFrame]))
-			if err != nil {
-				t.Logf("  [%d] invalid frame candidate: %v", numStreams, err)
-				continue
-			}
-			defer zr.Close()
-			res, err := ioutil.ReadAll(zr)
-			if err != nil && err != io.ErrUnexpectedEOF {
-				t.Fatalf("countStreams(zstd), ReadAll: %v", err)
-			}
-			if uint64(len(res)) == uncompressedFrameSize {
-				break
-			}
-
-			// Try the next magic byte candidate until end
-			if uint64(len(res)) > uncompressedFrameSize || nextFrame > len(remainingFrames) {
-				t.Fatalf("countStreams(zstd), cannot identify frame (off:%d)", zoff)
-			}
-		}
-		t.Logf("  [%d] at %d in stargz, uncompressed length %d (nextFrame: %d/%d)",
-			numStreams, zoff, uncompressedFrameSize, zoff+nextFrame, len(b))
+		t.Logf("  [%d] at %d in stargz (nextFrame: %d/%d): %v, %v",
+			numStreams, zoff, zoff+nextFrame, len(b), nextMagicIdx, nextSkippableIdx)
 		zoff += nextFrame
 		numStreams++
 	}
-	return numStreams
+	if len(wants) != 0 {
+		t.Fatalf("some stream offsets not found in the blob: %v", wants)
+	}
 }
 
 func nextIndex(s1, sub []byte) int {
@@ -151,7 +137,7 @@ func nextIndex(s1, sub []byte) int {
 	return -1
 }
 
-func (zc *zstdController) DiffIDOf(t *testing.T, b []byte) string {
+func (zc *zstdController) DiffIDOf(t estargz.TestingT, b []byte) string {
 	h := sha256.New()
 	zr, err := zstd.NewReader(bytes.NewReader(b))
 	if err != nil {

fs/config/config.go

@@ -33,60 +33,131 @@ const (
 	TargetPrefetchSizeLabel = "containerd.io/snapshot/remote/stargz.prefetch"
 )
 
+// Config is configuration for stargz snapshotter filesystem.
 type Config struct {
-	HTTPCacheType       string `toml:"http_cache_type"`
-	FSCacheType         string `toml:"filesystem_cache_type"`
-	ResolveResultEntry  int    `toml:"resolve_result_entry"`
-	PrefetchSize        int64  `toml:"prefetch_size"`
-	PrefetchTimeoutSec  int64  `toml:"prefetch_timeout_sec"`
-	NoPrefetch          bool   `toml:"noprefetch"`
-	NoBackgroundFetch   bool   `toml:"no_background_fetch"`
-	Debug               bool   `toml:"debug"`
-	AllowNoVerification bool   `toml:"allow_no_verification"`
-	DisableVerification bool   `toml:"disable_verification"`
-	MaxConcurrency      int64  `toml:"max_concurrency"`
-	NoPrometheus        bool   `toml:"no_prometheus"`
-	MetadataStore       string `toml:"metadata_store" default:"memory"`
+	// Type of cache for compressed contents fetched from the registry. "memory" stores them on memory.
+	// Other values default to cache them on disk.
+	HTTPCacheType string `toml:"http_cache_type" json:"http_cache_type"`
+
+	// Type of cache for uncompressed files contents. "memory" stores them on memory. Other values
+	// default to cache them on disk.
+	FSCacheType string `toml:"filesystem_cache_type" json:"filesystem_cache_type"`
+
+	// ResolveResultEntryTTLSec is TTL (in sec) to cache resolved layers for
+	// future use. (default 120s)
+	ResolveResultEntryTTLSec int `toml:"resolve_result_entry_ttl_sec" json:"resolve_result_entry_ttl_sec"`
+
+	// PrefetchSize is the default size (in bytes) to prefetch when mounting a layer. Default is 0. Stargz-snapshotter still
+	// uses the value specified by the image using "containerd.io/snapshot/remote/stargz.prefetch" or the landmark file.
+	PrefetchSize int64 `toml:"prefetch_size" json:"prefetch_size"`
+
+	// PrefetchTimeoutSec is the default timeout (in seconds) when the prefetching takes long. Default is 10s.
+	PrefetchTimeoutSec int64 `toml:"prefetch_timeout_sec" json:"prefetch_timeout_sec"`
+
+	// NoPrefetch disables prefetching. Default is false.
+	NoPrefetch bool `toml:"noprefetch" json:"noprefetch"`
+
+	// NoBackgroundFetch disables the behaviour of fetching the entire layer contents in background. Default is false.
+	NoBackgroundFetch bool `toml:"no_background_fetch" json:"no_background_fetch"`
+
+	// Debug enables filesystem debug log.
+	Debug bool `toml:"debug" json:"debug"`
+
+	// AllowNoVerification allows mouting images without verification. Default is false.
+	AllowNoVerification bool `toml:"allow_no_verification" json:"allow_no_verification"`
+
+	// DisableVerification disables verifying layer contents. Default is false.
+	DisableVerification bool `toml:"disable_verification" json:"disable_verification"`
+
+	// MaxConcurrency is max number of concurrent background tasks for fetching layer contents. Default is 2.
+	MaxConcurrency int64 `toml:"max_concurrency" json:"max_concurrency"`
+
+	// NoPrometheus disables exposing filesystem-related metrics. Default is false.
+	NoPrometheus bool `toml:"no_prometheus" json:"no_prometheus"`
 
 	// BlobConfig is config for layer blob management.
-	BlobConfig `toml:"blob"`
+	BlobConfig `toml:"blob" json:"blob"`
 
 	// DirectoryCacheConfig is config for directory-based cache.
-	DirectoryCacheConfig `toml:"directory_cache"`
+	DirectoryCacheConfig `toml:"directory_cache" json:"directory_cache"`
 
-	FuseConfig `toml:"fuse"`
+	// FuseConfig is configurations for FUSE fs.
+	FuseConfig `toml:"fuse" json:"fuse"`
+
+	// ResolveResultEntry is a deprecated field.
+	ResolveResultEntry int `toml:"resolve_result_entry" json:"resolve_result_entry"` // deprecated
 }
 
+// BlobConfig is configuration for the logic to fetching blobs.
 type BlobConfig struct {
-	ValidInterval int64 `toml:"valid_interval"`
-	CheckAlways   bool  `toml:"check_always"`
-	// ChunkSize is the granularity at which background fetch and on-demand reads
-	// are fetched from the remote registry.
-	ChunkSize            int64 `toml:"chunk_size"`
-	FetchTimeoutSec      int64 `toml:"fetching_timeout_sec"`
-	ForceSingleRangeMode bool  `toml:"force_single_range_mode"`
+	// ValidInterval specifies a duration (in seconds) during which the layer can be reused without
+	// checking the connection to the registry. Default is 60.
+	ValidInterval int64 `toml:"valid_interval" json:"valid_interval"`
+
+	// CheckAlways overwrites ValidInterval to 0 if it's true. Default is false.
+	CheckAlways bool `toml:"check_always" json:"check_always"`
+
+	// ChunkSize is the granularity (in bytes) at which background fetch and on-demand reads
+	// are fetched from the remote registry. Default is 50000.
+	ChunkSize int64 `toml:"chunk_size" json:"chunk_size"`
+
+	// FetchTimeoutSec is a timeout duration (in seconds) for fetching chunks from the registry. Default is 300.
+	FetchTimeoutSec int64 `toml:"fetching_timeout_sec" json:"fetching_tieout_sec"`
+
+	// ForceSingleRangeMode disables using of multiple ranges in a Range Request and always specifies one larger
+	// region that covers them. Default is false.
+	ForceSingleRangeMode bool `toml:"force_single_range_mode" json:"force_single_range_mode"`
+
 	// PrefetchChunkSize is the maximum bytes transferred per http GET from remote registry
 	// during prefetch. It is recommended to have PrefetchChunkSize > ChunkSize.
 	// If PrefetchChunkSize < ChunkSize prefetch bytes will be fetched as a single http GET,
 	// else total GET requests for prefetch = ceil(PrefetchSize / PrefetchChunkSize).
-	PrefetchChunkSize int64 `toml:"prefetch_chunk_size"`
+	// Default is 0.
+	PrefetchChunkSize int64 `toml:"prefetch_chunk_size" json:"prefetch_chunk_size"`
 
-	MaxRetries  int `toml:"max_retries"`
-	MinWaitMSec int `toml:"min_wait_msec"`
-	MaxWaitMSec int `toml:"max_wait_msec"`
+	// MaxRetries is a max number of reries of a HTTP request. Default is 5.
+	MaxRetries int `toml:"max_retries" json:"max_retries"`
+
+	// MinWaitMSec is minimal delay (in seconds) for the next retrying after a request failure. Default is 30.
+	MinWaitMSec int `toml:"min_wait_msec" json:"min_wait_msec"`
+
+	// MinWaitMSec is maximum delay (in seconds) for the next retrying after a request failure. Default is 30.
+	MaxWaitMSec int `toml:"max_wait_msec" json:"max_wait_msec"`
 }
 
+// DirectoryCacheConfig is configuration for the disk-based cache.
 type DirectoryCacheConfig struct {
-	MaxLRUCacheEntry int  `toml:"max_lru_cache_entry"`
-	MaxCacheFds      int  `toml:"max_cache_fds"`
-	SyncAdd          bool `toml:"sync_add"`
-	Direct           bool `toml:"direct" default:"true"`
+	// MaxLRUCacheEntry is the number of entries of LRU cache to cache data on memory. Default is 10.
+	MaxLRUCacheEntry int `toml:"max_lru_cache_entry" json:"max_lru_cache_entry"`
+
+	// MaxCacheFds is the number of entries of LRU cache to hold fds of files of cached contents. Default is 10.
+	MaxCacheFds int `toml:"max_cache_fds" json:"max_cache_fds"`
+
+	// SyncAdd being true means that each adding of data to the cache blocks until the data is fully written to the
+	// cache directory. Default is false.
+	SyncAdd bool `toml:"sync_add" json:"sync_add"`
+
+	// Direct disables on-memory data cache. Default is true for saving memory usage.
+	Direct bool `toml:"direct" default:"true" json:"direct"`
+
+	// FadvDontNeed forcefully clean fscache pagecache for saving memory. Default is false.
+	FadvDontNeed bool `toml:"fadv_dontneed" json:"fadv_dontneed"`
 }
 
+// FuseConfig is configuration for FUSE fs.
 type FuseConfig struct {
 	// AttrTimeout defines overall timeout attribute for a file system in seconds.
-	AttrTimeout int64 `toml:"attr_timeout"`
+	AttrTimeout int64 `toml:"attr_timeout" json:"attr_timeout"`
 
 	// EntryTimeout defines TTL for directory, name lookup in seconds.
-	EntryTimeout int64 `toml:"entry_timeout"`
+	EntryTimeout int64 `toml:"entry_timeout" json:"entry_timeout"`
+
+	// PassThrough indicates whether to enable FUSE passthrough mode to improve local file read performance. Default is false.
+	PassThrough bool `toml:"passthrough" default:"false" json:"passthrough"`
+
+	// MergeBufferSize is the size of the buffer to merge chunks (in bytes) for passthrough mode. Default is 400MB.
+	MergeBufferSize int64 `toml:"merge_buffer_size" default:"419430400" json:"merge_buffer_size"`
+
+	// MergeWorkerCount is the number of workers to merge chunks for passthrough mode. Default is 10.
+	MergeWorkerCount int `toml:"merge_worker_count" default:"10" json:"merge_worker_count"`
 }

fs/fs.go

@@ -42,12 +42,11 @@ import (
 	"os/exec"
 	"strconv"
 	"sync"
-	"syscall"
 	"time"
 
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/log"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containerd/stargz-snapshotter/fs/config"
 	"github.com/containerd/stargz-snapshotter/fs/layer"
@@ -55,6 +54,8 @@ import (
 	layermetrics "github.com/containerd/stargz-snapshotter/fs/metrics/layer"
 	"github.com/containerd/stargz-snapshotter/fs/remote"
 	"github.com/containerd/stargz-snapshotter/fs/source"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
 	"github.com/containerd/stargz-snapshotter/snapshot"
 	"github.com/containerd/stargz-snapshotter/task"
 	metrics "github.com/docker/go-metrics"
@@ -62,20 +63,31 @@ import (
 	"github.com/hanwen/go-fuse/v2/fuse"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 )
 
 const (
 	defaultFuseTimeout    = time.Second
 	defaultMaxConcurrency = 2
-	fusermountBin         = "fusermount"
+)
+
+var fusermountBin = []string{"fusermount", "fusermount3"}
+var (
+	nsLock = sync.Mutex{}
+
+	ns         *metrics.Namespace
+	metricsCtr *layermetrics.Controller
 )
 
 type Option func(*options)
 
 type options struct {
-	getSources      source.GetSources
-	resolveHandlers map[string]remote.Handler
+	getSources              source.GetSources
+	resolveHandlers         map[string]remote.Handler
+	metadataStore           metadata.Store
+	metricsLogLevel         *log.Level
+	overlayOpaqueType       layer.OverlayOpaqueType
+	additionalDecompressors func(context.Context, source.RegistryHosts, reference.Spec, ocispec.Descriptor) []metadata.Decompressor
 }
 
 func WithGetSources(s source.GetSources) Option {
@@ -93,6 +105,30 @@ func WithResolveHandler(name string, handler remote.Handler) Option {
 	}
 }
 
+func WithMetadataStore(metadataStore metadata.Store) Option {
+	return func(opts *options) {
+		opts.metadataStore = metadataStore
+	}
+}
+
+func WithMetricsLogLevel(logLevel log.Level) Option {
+	return func(opts *options) {
+		opts.metricsLogLevel = &logLevel
+	}
+}
+
+func WithOverlayOpaqueType(overlayOpaqueType layer.OverlayOpaqueType) Option {
+	return func(opts *options) {
+		opts.overlayOpaqueType = overlayOpaqueType
+	}
+}
+
+func WithAdditionalDecompressors(d func(context.Context, source.RegistryHosts, reference.Spec, ocispec.Descriptor) []metadata.Decompressor) Option {
+	return func(opts *options) {
+		opts.additionalDecompressors = d
+	}
+}
+
 func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snapshot.FileSystem, err error) {
 	var fsOpts options
 	for _, o := range opts {
@@ -103,16 +139,21 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snapshot.F
 		maxConcurrency = defaultMaxConcurrency
 	}
 
-	attrTimeout := time.Duration(cfg.FuseConfig.AttrTimeout) * time.Second
+	attrTimeout := time.Duration(cfg.AttrTimeout) * time.Second
 	if attrTimeout == 0 {
 		attrTimeout = defaultFuseTimeout
 	}
 
-	entryTimeout := time.Duration(cfg.FuseConfig.EntryTimeout) * time.Second
+	entryTimeout := time.Duration(cfg.EntryTimeout) * time.Second
 	if entryTimeout == 0 {
 		entryTimeout = defaultFuseTimeout
 	}
 
+	metadataStore := fsOpts.metadataStore
+	if metadataStore == nil {
+		metadataStore = memorymetadata.NewReader
+	}
+
 	getSources := fsOpts.getSources
 	if getSources == nil {
 		getSources = source.FromDefaultLabels(func(refspec reference.Spec) (hosts []docker.RegistryHost, _ error) {
@@ -120,19 +161,25 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snapshot.F
 		})
 	}
 	tm := task.NewBackgroundTaskManager(maxConcurrency, 5*time.Second)
-	r, err := layer.NewResolver(root, tm, cfg, fsOpts.resolveHandlers)
+	r, err := layer.NewResolver(root, tm, cfg, fsOpts.resolveHandlers, metadataStore, fsOpts.overlayOpaqueType, fsOpts.additionalDecompressors)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to setup resolver")
+		return nil, fmt.Errorf("failed to setup resolver: %w", err)
 	}
 
-	var ns *metrics.Namespace
-	if !cfg.NoPrometheus {
+	nsLock.Lock()
+	defer nsLock.Unlock()
+
+	if !cfg.NoPrometheus && ns == nil {
 		ns = metrics.NewNamespace("stargz", "fs", nil)
-		commonmetrics.Register() // Register common metrics. This will happen only once.
+		logLevel := log.DebugLevel
+		if fsOpts.metricsLogLevel != nil {
+			logLevel = *fsOpts.metricsLogLevel
+		}
+		commonmetrics.Register(logLevel) // Register common metrics. This will happen only once.
+		metrics.Register(ns)             // Register layer metrics.
 	}
-	c := layermetrics.NewLayerMetrics(ns)
-	if ns != nil {
-		metrics.Register(ns) // Register layer metrics.
+	if metricsCtr == nil {
+		metricsCtr = layermetrics.NewLayerMetrics(ns)
 	}
 
 	return &filesystem{
@@ -146,7 +193,7 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snapshot.F
 		backgroundTaskManager: tm,
 		allowNoVerification:   cfg.AllowNoVerification,
 		disableVerification:   cfg.DisableVerification,
-		metricsController:     c,
+		metricsController:     metricsCtr,
 		attrTimeout:           attrTimeout,
 		entryTimeout:          entryTimeout,
 	}, nil
@@ -209,7 +256,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 				fs.prefetch(ctx, l, defaultPrefetchSize, start)
 				return
 			}
-			rErr = errors.Wrapf(rErr, "failed to resolve layer %q from %q: %v", s.Target.Digest, s.Name, err)
+			rErr = fmt.Errorf("failed to resolve layer %q from %q: %v: %w", s.Target.Digest, s.Name, err, rErr)
 		}
 		errChan <- rErr
 	}()
@@ -240,7 +287,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 	case l = <-resultChan:
 	case err := <-errChan:
 		log.G(ctx).WithError(err).Debug("failed to resolve layer")
-		return errors.Wrapf(err, "failed to resolve layer")
+		return fmt.Errorf("failed to resolve layer: %w", err)
 	case <-time.After(30 * time.Second):
 		log.G(ctx).Debug("failed to resolve layer (timeout)")
 		return fmt.Errorf("failed to resolve layer (timeout)")
@@ -261,11 +308,11 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 		dgst, err := digest.Parse(tocDigest)
 		if err != nil {
 			log.G(ctx).WithError(err).Debugf("failed to parse passed TOC digest %q", dgst)
-			return errors.Wrapf(err, "invalid TOC digest: %v", tocDigest)
+			return fmt.Errorf("invalid TOC digest: %v: %w", tocDigest, err)
 		}
 		if err := l.Verify(dgst); err != nil {
 			log.G(ctx).WithError(err).Debugf("invalid layer")
-			return errors.Wrapf(err, "invalid stargz layer")
+			return fmt.Errorf("invalid stargz layer: %w", err)
 		}
 		log.G(ctx).Debugf("verified")
 	} else if _, ok := labels[config.TargetSkipVerifyLabel]; ok && fs.allowNoVerification {
@@ -281,7 +328,7 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 	node, err := l.RootNode(0)
 	if err != nil {
 		log.G(ctx).WithError(err).Warnf("Failed to get root node")
-		return errors.Wrapf(err, "failed to get root node")
+		return fmt.Errorf("failed to get root node: %w", err)
 	}
 
 	// Measuring duration of Mount operation for resolved layer.
@@ -306,7 +353,8 @@ func (fs *filesystem) Mount(ctx context.Context, mountpoint string, labels map[s
 		FsName:     "stargz", // name this filesystem as "stargz"
 		Debug:      fs.debug,
 	}
-	if _, err := exec.LookPath(fusermountBin); err == nil {
+	if isFusermountBinExist() {
+		log.G(ctx).Infof("fusermount detected")
 		mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container
 	} else {
 		log.G(ctx).WithError(err).Infof("%s not installed; trying direct mount", fusermountBin)
@@ -341,10 +389,13 @@ func (fs *filesystem) Check(ctx context.Context, mountpoint string, labels map[s
 		return fmt.Errorf("layer not registered")
 	}
 
-	// Check the blob connectivity and try to refresh the connection on failure
-	if err := fs.check(ctx, l, labels); err != nil {
-		log.G(ctx).WithError(err).Warn("check failed")
-		return err
+	if l.Info().FetchedSize < l.Info().Size {
+		// Image contents hasn't fully cached yet.
+		// Check the blob connectivity and try to refresh the connection on failure
+		if err := fs.check(ctx, l, labels); err != nil {
+			log.G(ctx).WithError(err).Warn("check failed")
+			return err
+		}
 	}
 
 	// Wait for prefetch compeletion
@@ -381,10 +432,8 @@ func (fs *filesystem) check(ctx context.Context, l layer.Layer, labels map[strin
 				log.G(ctx).Debug("Successfully refreshed connection")
 				return nil
 			}
-			log.G(ctx).WithError(err).Warnf("failed to refresh the layer %q from %q",
-				s.Target.Digest, s.Name)
-			rErr = errors.Wrapf(rErr, "failed(layer:%q, ref:%q): %v",
-				s.Target.Digest, s.Name, err)
+			log.G(ctx).WithError(err).Warnf("failed to refresh the layer %q from %q", s.Target.Digest, s.Name)
+			rErr = fmt.Errorf("failed(layer:%q, ref:%q): %v: %w", s.Target.Digest, s.Name, err, rErr)
 		}
 	}
 
@@ -392,22 +441,42 @@ func (fs *filesystem) check(ctx context.Context, l layer.Layer, labels map[strin
 }
 
 func (fs *filesystem) Unmount(ctx context.Context, mountpoint string) error {
+	if mountpoint == "" {
+		return fmt.Errorf("mount point must be specified")
+	}
 	fs.layerMu.Lock()
 	l, ok := fs.layer[mountpoint]
 	if !ok {
 		fs.layerMu.Unlock()
 		return fmt.Errorf("specified path %q isn't a mountpoint", mountpoint)
 	}
-	delete(fs.layer, mountpoint) // unregisters the corresponding layer
-	l.Done()
+	delete(fs.layer, mountpoint)      // unregisters the corresponding layer
+	if err := l.Close(); err != nil { // Cleanup associated resources
+		log.G(ctx).WithError(err).Warn("failed to release resources of the layer")
+	}
 	fs.layerMu.Unlock()
 	fs.metricsController.Remove(mountpoint)
-	// The goroutine which serving the mountpoint possibly becomes not responding.
-	// In case of such situations, we use MNT_FORCE here and abort the connection.
-	// In the future, we might be able to consider to kill that specific hanging
-	// goroutine using channel, etc.
-	// See also: https://www.kernel.org/doc/html/latest/filesystems/fuse.html#aborting-a-filesystem-connection
-	return syscall.Unmount(mountpoint, syscall.MNT_FORCE)
+
+	if err := unmount(mountpoint, 0); err != nil {
+		if err != unix.EBUSY {
+			return err
+		}
+		// Try force unmount
+		log.G(ctx).WithError(err).Debugf("trying force unmount %q", mountpoint)
+		if err := unmount(mountpoint, unix.MNT_FORCE); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func unmount(target string, flags int) error {
+	for {
+		if err := unix.Unmount(target, flags); err != unix.EINTR {
+			return err
+		}
+	}
 }
 
 func (fs *filesystem) prefetch(ctx context.Context, l layer.Layer, defaultPrefetchSize int64, start time.Time) {
@@ -436,3 +505,12 @@ func neighboringLayers(manifest ocispec.Manifest, target ocispec.Descriptor) (de
 	}
 	return
 }
+
+func isFusermountBinExist() bool {
+	for _, b := range fusermountBin {
+		if _, err := exec.LookPath(b); err == nil {
+			return true
+		}
+	}
+	return false
+}

fs/fs_test.go

@@ -28,8 +28,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/reference"
 	"github.com/containerd/stargz-snapshotter/fs/layer"
 	"github.com/containerd/stargz-snapshotter/fs/remote"
 	"github.com/containerd/stargz-snapshotter/fs/source"
@@ -65,14 +65,20 @@ type breakableLayer struct {
 	success bool
 }
 
-func (l *breakableLayer) Info() layer.Info                                    { return layer.Info{} }
-func (l *breakableLayer) RootNode(uint32) (fusefs.InodeEmbedder, error)       { return nil, nil }
-func (l *breakableLayer) Verify(tocDigest digest.Digest) error                { return nil }
-func (l *breakableLayer) SkipVerify()                                         {}
-func (l *breakableLayer) Prefetch(prefetchSize int64) error                   { return fmt.Errorf("fail") }
-func (l *breakableLayer) ReadAt([]byte, int64, ...remote.Option) (int, error) { return 0, nil }
-func (l *breakableLayer) WaitForPrefetchCompletion() error                    { return fmt.Errorf("fail") }
-func (l *breakableLayer) BackgroundFetch() error                              { return fmt.Errorf("fail") }
+func (l *breakableLayer) Info() layer.Info {
+	return layer.Info{
+		Size: 1,
+	}
+}
+func (l *breakableLayer) RootNode(uint32) (fusefs.InodeEmbedder, error) { return nil, nil }
+func (l *breakableLayer) Verify(tocDigest digest.Digest) error          { return nil }
+func (l *breakableLayer) SkipVerify()                                   {}
+func (l *breakableLayer) Prefetch(prefetchSize int64) error             { return fmt.Errorf("fail") }
+func (l *breakableLayer) ReadAt([]byte, int64, ...remote.Option) (int, error) {
+	return 0, fmt.Errorf("fail")
+}
+func (l *breakableLayer) WaitForPrefetchCompletion() error { return fmt.Errorf("fail") }
+func (l *breakableLayer) BackgroundFetch() error           { return fmt.Errorf("fail") }
 func (l *breakableLayer) Check() error {
 	if !l.success {
 		return fmt.Errorf("failed")
@@ -85,4 +91,5 @@ func (l *breakableLayer) Refresh(ctx context.Context, hosts source.RegistryHosts
 	}
 	return nil
 }
-func (l *breakableLayer) Done() {}
+func (l *breakableLayer) Done()        {}
+func (l *breakableLayer) Close() error { return nil }

fs/layer/layer.go

@@ -27,14 +27,13 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"
 
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/log"
 	"github.com/containerd/stargz-snapshotter/cache"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/containerd/stargz-snapshotter/estargz/zstdchunked"
@@ -44,29 +43,34 @@ import (
 	"github.com/containerd/stargz-snapshotter/fs/remote"
 	"github.com/containerd/stargz-snapshotter/fs/source"
 	"github.com/containerd/stargz-snapshotter/metadata"
-	dbmetadata "github.com/containerd/stargz-snapshotter/metadata/db"
-	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
 	"github.com/containerd/stargz-snapshotter/task"
-	"github.com/containerd/stargz-snapshotter/util/lrucache"
+	"github.com/containerd/stargz-snapshotter/util/cacheutil"
 	"github.com/containerd/stargz-snapshotter/util/namedmutex"
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	bolt "go.etcd.io/bbolt"
 )
 
 const (
-	defaultResolveResultEntry = 30
-	defaultMaxLRUCacheEntry   = 10
-	defaultMaxCacheFds        = 10
-	defaultPrefetchTimeoutSec = 10
-	memoryCacheType           = "memory"
-	memoryMetadataType        = "memory"
-	dbMetadataType            = "db"
+	defaultResolveResultEntryTTLSec = 120
+	defaultMaxLRUCacheEntry         = 10
+	defaultMaxCacheFds              = 10
+	defaultPrefetchTimeoutSec       = 10
+	memoryCacheType                 = "memory"
 )
 
+// passThroughConfig contains configuration for FUSE passthrough mode
+type passThroughConfig struct {
+	// enable indicates whether to enable FUSE passthrough mode
+	enable bool
+
+	// mergeBufferSize is the size of the buffer to merge chunks (in bytes)
+	mergeBufferSize int64
+
+	// mergeWorkerCount is the number of workers to merge chunks
+	mergeWorkerCount int
+}
+
 // Layer represents a layer.
 type Layer interface {
 	// Info returns the information of this layer.
@@ -106,6 +110,10 @@ type Layer interface {
 	// Done releases the reference to this layer. The resources related to this layer will be
 	// discarded sooner or later. Queries after calling this function won't be serviced.
 	Done()
+
+	// Close is the same as Done. But this evicts the resources related to this Layer immediately.
+	// This can be used for cleaning up resources on unmount.
+	Close() error
 }
 
 // Info is the current status of a layer.
@@ -115,28 +123,31 @@ type Info struct {
 	FetchedSize  int64     // layer fetched size in bytes
 	PrefetchSize int64     // layer prefetch size in bytes
 	ReadTime     time.Time // last time the layer was read
+	TOCDigest    digest.Digest
 }
 
 // Resolver resolves the layer location and provieds the handler of that layer.
 type Resolver struct {
-	rootDir               string
-	resolver              *remote.Resolver
-	prefetchTimeout       time.Duration
-	layerCache            *lrucache.Cache
-	layerCacheMu          sync.Mutex
-	blobCache             *lrucache.Cache
-	blobCacheMu           sync.Mutex
-	backgroundTaskManager *task.BackgroundTaskManager
-	resolveLock           *namedmutex.NamedMutex
-	config                config.Config
-	newMetadataReader     func(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error)
+	rootDir                 string
+	resolver                *remote.Resolver
+	prefetchTimeout         time.Duration
+	layerCache              *cacheutil.TTLCache
+	layerCacheMu            sync.Mutex
+	blobCache               *cacheutil.TTLCache
+	blobCacheMu             sync.Mutex
+	backgroundTaskManager   *task.BackgroundTaskManager
+	resolveLock             *namedmutex.NamedMutex
+	config                  config.Config
+	metadataStore           metadata.Store
+	overlayOpaqueType       OverlayOpaqueType
+	additionalDecompressors func(context.Context, source.RegistryHosts, reference.Spec, ocispec.Descriptor) []metadata.Decompressor
 }
 
 // NewResolver returns a new layer resolver.
-func NewResolver(root string, backgroundTaskManager *task.BackgroundTaskManager, cfg config.Config, resolveHandlers map[string]remote.Handler) (*Resolver, error) {
-	resolveResultEntry := cfg.ResolveResultEntry
-	if resolveResultEntry == 0 {
-		resolveResultEntry = defaultResolveResultEntry
+func NewResolver(root string, backgroundTaskManager *task.BackgroundTaskManager, cfg config.Config, resolveHandlers map[string]remote.Handler, metadataStore metadata.Store, overlayOpaqueType OverlayOpaqueType, additionalDecompressors func(context.Context, source.RegistryHosts, reference.Spec, ocispec.Descriptor) []metadata.Decompressor) (*Resolver, error) {
+	resolveResultEntryTTL := time.Duration(cfg.ResolveResultEntryTTLSec) * time.Second
+	if resolveResultEntryTTL == 0 {
+		resolveResultEntryTTL = defaultResolveResultEntryTTLSec * time.Second
 	}
 	prefetchTimeout := time.Duration(cfg.PrefetchTimeoutSec) * time.Second
 	if prefetchTimeout == 0 {
@@ -146,62 +157,42 @@ func NewResolver(root string, backgroundTaskManager *task.BackgroundTaskManager,
 	// layerCache caches resolved layers for future use. This is useful in a use-case where
 	// the filesystem resolves and caches all layers in an image (not only queried one) in parallel,
 	// before they are actually queried.
-	layerCache := lrucache.New(resolveResultEntry)
+	layerCache := cacheutil.NewTTLCache(resolveResultEntryTTL)
 	layerCache.OnEvicted = func(key string, value interface{}) {
 		if err := value.(*layer).close(); err != nil {
-			logrus.WithField("key", key).WithError(err).Warnf("failed to clean up layer")
+			log.L.WithField("key", key).WithError(err).Warnf("failed to clean up layer")
 			return
 		}
-		logrus.WithField("key", key).Debugf("cleaned up layer")
+		log.L.WithField("key", key).Debugf("cleaned up layer")
 	}
 
 	// blobCache caches resolved blobs for futural use. This is especially useful when a layer
 	// isn't eStargz/stargz (the *layer object won't be created/cached in this case).
-	blobCache := lrucache.New(resolveResultEntry)
+	blobCache := cacheutil.NewTTLCache(resolveResultEntryTTL)
 	blobCache.OnEvicted = func(key string, value interface{}) {
 		if err := value.(remote.Blob).Close(); err != nil {
-			logrus.WithField("key", key).WithError(err).Warnf("failed to clean up blob")
+			log.L.WithField("key", key).WithError(err).Warnf("failed to clean up blob")
 			return
 		}
-		logrus.WithField("key", key).Debugf("cleaned up blob")
+		log.L.WithField("key", key).Debugf("cleaned up blob")
 	}
 
 	if err := os.MkdirAll(root, 0700); err != nil {
 		return nil, err
 	}
 
-	var newReader func(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error)
-	switch cfg.MetadataStore {
-	case "", memoryMetadataType:
-		newReader = memorymetadata.NewReader
-	case dbMetadataType:
-		bOpts := bolt.Options{
-			NoFreelistSync:  true,
-			InitialMmapSize: 64 * 1024 * 1024,
-			FreelistType:    bolt.FreelistMapType,
-		}
-		db, err := bolt.Open(filepath.Join(root, "metadata.db"), 0600, &bOpts)
-		if err != nil {
-			return nil, err
-		}
-		newReader = func(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error) {
-			return dbmetadata.NewReader(db, sr, opts...)
-		}
-	default:
-		return nil, fmt.Errorf("unknown metadata store type: %v; must be %v or %v",
-			cfg.MetadataStore, memoryMetadataType, dbMetadataType)
-	}
-
 	return &Resolver{
-		rootDir:               root,
-		resolver:              remote.NewResolver(cfg.BlobConfig, resolveHandlers),
-		layerCache:            layerCache,
-		blobCache:             blobCache,
-		prefetchTimeout:       prefetchTimeout,
-		backgroundTaskManager: backgroundTaskManager,
-		config:                cfg,
-		resolveLock:           new(namedmutex.NamedMutex),
-		newMetadataReader:     newReader,
+		rootDir:                 root,
+		resolver:                remote.NewResolver(cfg.BlobConfig, resolveHandlers),
+		layerCache:              layerCache,
+		blobCache:               blobCache,
+		prefetchTimeout:         prefetchTimeout,
+		backgroundTaskManager:   backgroundTaskManager,
+		config:                  cfg,
+		resolveLock:             new(namedmutex.NamedMutex),
+		metadataStore:           metadataStore,
+		overlayOpaqueType:       overlayOpaqueType,
+		additionalDecompressors: additionalDecompressors,
 	}, nil
 }
 
@@ -225,7 +216,7 @@ func newCache(root string, cacheType string, cfg config.Config) (cache.BlobCache
 			return new(bytes.Buffer)
 		},
 	}
-	dCache, fCache := lrucache.New(maxDataEntry), lrucache.New(maxFdEntry)
+	dCache, fCache := cacheutil.NewLRUCache(maxDataEntry), cacheutil.NewLRUCache(maxFdEntry)
 	dCache.OnEvicted = func(key string, value interface{}) {
 		value.(*bytes.Buffer).Reset()
 		bufPool.Put(value)
@@ -237,18 +228,19 @@ func newCache(root string, cacheType string, cfg config.Config) (cache.BlobCache
 	if err := os.MkdirAll(root, 0700); err != nil {
 		return nil, err
 	}
-	cachePath, err := ioutil.TempDir(root, "")
+	cachePath, err := os.MkdirTemp(root, "")
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to initialize directory cache")
+		return nil, fmt.Errorf("failed to initialize directory cache: %w", err)
 	}
 	return cache.NewDirectoryCache(
 		cachePath,
 		cache.DirectoryCacheConfig{
-			SyncAdd:   dcc.SyncAdd,
-			DataCache: dCache,
-			FdCache:   fCache,
-			BufPool:   bufPool,
-			Direct:    dcc.Direct,
+			SyncAdd:      dcc.SyncAdd,
+			DataCache:    dCache,
+			FdCache:      fCache,
+			BufPool:      bufPool,
+			Direct:       dcc.Direct,
+			FadvDontNeed: dcc.FadvDontNeed,
 		},
 	)
 }
@@ -258,13 +250,13 @@ func (r *Resolver) Resolve(ctx context.Context, hosts source.RegistryHosts, refs
 	name := refspec.String() + "/" + desc.Digest.String()
 
 	// Wait if resolving this layer is already running. The result
-	// can hopefully get from the LRU cache.
+	// can hopefully get from the cache.
 	r.resolveLock.Lock(name)
 	defer r.resolveLock.Unlock(name)
 
 	ctx = log.WithLogger(ctx, log.G(ctx).WithField("src", name))
 
-	// First, try to retrieve this layer from the underlying LRU cache.
+	// First, try to retrieve this layer from the underlying cache.
 	r.layerCacheMu.Lock()
 	c, done, ok := r.layerCache.Get(name)
 	r.layerCacheMu.Unlock()
@@ -274,7 +266,7 @@ func (r *Resolver) Resolve(ctx context.Context, hosts source.RegistryHosts, refs
 			return &layerRef{l, done}, nil
 		}
 		// Cached layer is invalid
-		done()
+		done(true)
 		r.layerCacheMu.Lock()
 		r.layerCache.Remove(name)
 		r.layerCacheMu.Unlock()
@@ -285,17 +277,17 @@ func (r *Resolver) Resolve(ctx context.Context, hosts source.RegistryHosts, refs
 	// Resolve the blob.
 	blobR, err := r.resolveBlob(ctx, hosts, refspec, desc)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to resolve the blob")
+		return nil, fmt.Errorf("failed to resolve the blob: %w", err)
 	}
 	defer func() {
 		if retErr != nil {
-			blobR.done()
+			blobR.done(true)
 		}
 	}()
 
 	fsCache, err := newCache(filepath.Join(r.rootDir, "fscache"), r.config.FSCacheType, r.config)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to create fs cache")
+		return nil, fmt.Errorf("failed to create fs cache: %w", err)
 	}
 	defer func() {
 		if retErr != nil {
@@ -324,18 +316,27 @@ func (r *Resolver) Resolve(ctx context.Context, hosts source.RegistryHosts, refs
 			commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.DeserializeTocJSON, desc.Digest, start)
 		},
 	}
-	meta, err := r.newMetadataReader(sr,
-		append(esgzOpts, metadata.WithTelemetry(&telemetry), metadata.WithDecompressors(new(zstdchunked.Decompressor)))...)
+
+	additionalDecompressors := []metadata.Decompressor{new(zstdchunked.Decompressor)}
+	if r.additionalDecompressors != nil {
+		additionalDecompressors = append(additionalDecompressors, r.additionalDecompressors(ctx, hosts, refspec, desc)...)
+	}
+	meta, err := r.metadataStore(sr,
+		append(esgzOpts, metadata.WithTelemetry(&telemetry), metadata.WithDecompressors(additionalDecompressors...))...)
 	if err != nil {
 		return nil, err
 	}
 	vr, err := reader.NewReader(meta, fsCache, desc.Digest)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to read layer")
+		return nil, fmt.Errorf("failed to read layer: %w", err)
 	}
 
 	// Combine layer information together and cache it.
-	l := newLayer(r, desc, blobR, vr)
+	l := newLayer(r, desc, blobR, vr, passThroughConfig{
+		enable:           r.config.PassThrough,
+		mergeBufferSize:  r.config.MergeBufferSize,
+		mergeWorkerCount: r.config.MergeWorkerCount,
+	})
 	r.layerCacheMu.Lock()
 	cachedL, done2, added := r.layerCache.Add(name, l)
 	r.layerCacheMu.Unlock()
@@ -351,7 +352,7 @@ func (r *Resolver) Resolve(ctx context.Context, hosts source.RegistryHosts, refs
 func (r *Resolver) resolveBlob(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) (_ *blobRef, retErr error) {
 	name := refspec.String() + "/" + desc.Digest.String()
 
-	// Try to retrieve the blob from the underlying LRU cache.
+	// Try to retrieve the blob from the underlying cache.
 	r.blobCacheMu.Lock()
 	c, done, ok := r.blobCache.Get(name)
 	r.blobCacheMu.Unlock()
@@ -360,7 +361,7 @@ func (r *Resolver) resolveBlob(ctx context.Context, hosts source.RegistryHosts,
 			return &blobRef{blob, done}, nil
 		}
 		// invalid blob. discard this.
-		done()
+		done(true)
 		r.blobCacheMu.Lock()
 		r.blobCache.Remove(name)
 		r.blobCacheMu.Unlock()
@@ -368,7 +369,7 @@ func (r *Resolver) resolveBlob(ctx context.Context, hosts source.RegistryHosts,
 
 	httpCache, err := newCache(filepath.Join(r.rootDir, "httpcache"), r.config.HTTPCacheType, r.config)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to create http cache")
+		return nil, fmt.Errorf("failed to create http cache: %w", err)
 	}
 	defer func() {
 		if retErr != nil {
@@ -379,7 +380,7 @@ func (r *Resolver) resolveBlob(ctx context.Context, hosts source.RegistryHosts,
 	// Resolve the blob and cache the result.
 	b, err := r.resolver.Resolve(ctx, hosts, refspec, desc, httpCache)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to resolve the source")
+		return nil, fmt.Errorf("failed to resolve the source: %w", err)
 	}
 	r.blobCacheMu.Lock()
 	cachedB, done, added := r.blobCache.Add(name, b)
@@ -395,6 +396,7 @@ func newLayer(
 	desc ocispec.Descriptor,
 	blob *blobRef,
 	vr *reader.VerifiableReader,
+	pth passThroughConfig,
 ) *layer {
 	return &layer{
 		resolver:         resolver,
@@ -402,6 +404,7 @@ func newLayer(
 		blob:             blob,
 		verifiableReader: vr,
 		prefetchWaiter:   newWaiter(),
+		passThrough:      pth,
 	}
 }
 
@@ -422,6 +425,7 @@ type layer struct {
 
 	prefetchOnce        sync.Once
 	backgroundFetchOnce sync.Once
+	passThrough         passThroughConfig
 }
 
 func (l *layer) Info() Info {
@@ -435,6 +439,7 @@ func (l *layer) Info() Info {
 		FetchedSize:  l.blob.FetchedSize(),
 		PrefetchSize: l.prefetchedSize(),
 		ReadTime:     readTime,
+		TOCDigest:    l.verifiableReader.Metadata().TOCDigest(),
 	}
 }
 
@@ -510,7 +515,7 @@ func (l *layer) prefetch(ctx context.Context, prefetchSize int64) error {
 	} else if id, _, err := l.verifiableReader.Metadata().GetChild(rootID, estargz.PrefetchLandmark); err == nil {
 		offset, err := l.verifiableReader.Metadata().GetOffset(id)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get offset of prefetch landmark")
+			return fmt.Errorf("failed to get offset of prefetch landmark: %w", err)
 		}
 		// override the prefetch size with optimized value
 		prefetchSize = offset
@@ -525,7 +530,7 @@ func (l *layer) prefetch(ctx context.Context, prefetchSize int64) error {
 	commonmetrics.WriteLatencyLogValue(ctx, l.desc.Digest, commonmetrics.PrefetchDownload, downloadStart) // time to download prefetch data
 
 	if err != nil {
-		return errors.Wrap(err, "failed to prefetch layer")
+		return fmt.Errorf("failed to prefetch layer: %w", err)
 	}
 
 	// Set prefetch size for metrics after prefetch completed
@@ -540,7 +545,7 @@ func (l *layer) prefetch(ctx context.Context, prefetchSize int64) error {
 	}))
 	commonmetrics.WriteLatencyLogValue(ctx, l.desc.Digest, commonmetrics.PrefetchDecompress, decompressStart) // time to decompress prefetch data
 	if err != nil {
-		return errors.Wrap(err, "failed to cache prefetched layer")
+		return fmt.Errorf("failed to cache prefetched layer: %w", err)
 	}
 
 	return nil
@@ -592,7 +597,12 @@ func (l *layer) backgroundFetch(ctx context.Context) error {
 }
 
 func (l *layerRef) Done() {
-	l.done()
+	l.done(false) // leave chances to reuse this
+}
+
+func (l *layerRef) Close() error {
+	l.done(true) // evict this from the cache
+	return nil
 }
 
 func (l *layer) RootNode(baseInode uint32) (fusefs.InodeEmbedder, error) {
@@ -602,7 +612,7 @@ func (l *layer) RootNode(baseInode uint32) (fusefs.InodeEmbedder, error) {
 	if l.r == nil {
 		return nil, fmt.Errorf("layer hasn't been verified yet")
 	}
-	return newNode(l.desc.Digest, l.r, l.blob, baseInode)
+	return newNode(l.desc.Digest, l.r, l.blob, baseInode, l.resolver.overlayOpaqueType, l.passThrough)
 }
 
 func (l *layer) ReadAt(p []byte, offset int64, opts ...remote.Option) (int, error) {
@@ -616,7 +626,7 @@ func (l *layer) close() error {
 		return nil
 	}
 	l.closed = true
-	defer l.blob.done() // Close reader first, then close the blob
+	defer l.blob.done(true) // Close reader first, then close the blob
 	l.verifiableReader.Close()
 	if l.r != nil {
 		return l.r.Close()
@@ -636,7 +646,7 @@ func (l *layer) isClosed() bool {
 // to this blob will be discarded.
 type blobRef struct {
 	remote.Blob
-	done func()
+	done func(bool)
 }
 
 // layerRef is a reference to the layer in the cache. Calling `Done` or `done` decreases the
@@ -644,7 +654,7 @@ type blobRef struct {
 // cache, resources bound to this layer will be discarded.
 type layerRef struct {
 	*layer
-	done func()
+	done func(bool)
 }
 
 func newWaiter() *waiter {

fs/layer/layer_test.go

@@ -23,235 +23,31 @@
 package layer
 
 import (
-	"context"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"path"
-	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/stargz-snapshotter/cache"
-	"github.com/containerd/stargz-snapshotter/estargz"
-	"github.com/containerd/stargz-snapshotter/fs/reader"
-	"github.com/containerd/stargz-snapshotter/fs/remote"
-	"github.com/containerd/stargz-snapshotter/fs/source"
-	"github.com/containerd/stargz-snapshotter/metadata"
-	"github.com/containerd/stargz-snapshotter/task"
-	"github.com/containerd/stargz-snapshotter/util/testutil"
-	digest "github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
 )
 
-const (
-	sampleChunkSize = 3
-	sampleData1     = "0123456789"
-	sampleData2     = "abcdefghij"
-)
-
-var testStateLayerDigest = digest.FromString("dummy")
-
-// Tests prefetch method of each stargz file.
-func TestPrefetch(t *testing.T) {
-	testPrefetch(t, "db", newDBReader)
-	testPrefetch(t, "mem", newMemoryReader)
-}
-
-func testPrefetch(t *testing.T, name string, factory readerFactory) {
-	defaultPrefetchSize := int64(10000)
-	landmarkPosition := func(t *testing.T, l *layer) int64 {
-		if l.r == nil {
-			t.Fatalf("layer hasn't been verified yet")
-		}
-		if id, _, err := l.r.Metadata().GetChild(l.r.Metadata().RootID(), estargz.PrefetchLandmark); err == nil {
-			offset, err := l.r.Metadata().GetOffset(id)
-			if err != nil {
-				t.Fatalf("failed to get offset of prefetch landmark")
-			}
-			return offset
-		}
-		return defaultPrefetchSize
-	}
-	tests := []struct {
-		name             string
-		in               []testutil.TarEntry
-		wantNum          int      // number of chunks wanted in the cache
-		wants            []string // filenames to compare
-		prefetchSize     func(*testing.T, *layer) int64
-		prioritizedFiles []string
-	}{
-		{
-			name: "no_prefetch",
-			in: []testutil.TarEntry{
-				testutil.File("foo.txt", sampleData1),
-			},
-			wantNum:          0,
-			prioritizedFiles: nil,
-		},
-		{
-			name: "prefetch",
-			in: []testutil.TarEntry{
-				testutil.File("foo.txt", sampleData1),
-				testutil.File("bar.txt", sampleData2),
-			},
-			wantNum:          chunkNum(sampleData1),
-			wants:            []string{"foo.txt"},
-			prefetchSize:     landmarkPosition,
-			prioritizedFiles: []string{"foo.txt"},
-		},
-		{
-			name: "with_dir",
-			in: []testutil.TarEntry{
-				testutil.Dir("foo/"),
-				testutil.File("foo/bar.txt", sampleData1),
-				testutil.Dir("buz/"),
-				testutil.File("buz/buzbuz.txt", sampleData2),
-			},
-			wantNum:          chunkNum(sampleData1),
-			wants:            []string{"foo/bar.txt"},
-			prefetchSize:     landmarkPosition,
-			prioritizedFiles: []string{"foo/", "foo/bar.txt"},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name+"-"+name, func(t *testing.T) {
-			sr, dgst, err := testutil.BuildEStargz(tt.in,
-				testutil.WithEStargzOptions(
-					estargz.WithChunkSize(sampleChunkSize),
-					estargz.WithPrioritizedFiles(tt.prioritizedFiles),
-				))
-			if err != nil {
-				t.Fatalf("failed to build eStargz: %v", err)
-			}
-			blob := newBlob(sr)
-			mcache := cache.NewMemoryCache()
-			mr, closeFn, err := factory(sr)
-			if err != nil {
-				t.Fatalf("failed to create metadata reader: %v", err)
-			}
-			defer mr.Close()
-			defer closeFn()
-			vr, err := reader.NewReader(mr, mcache, digest.FromString(""))
-			if err != nil {
-				t.Fatalf("failed to create reader: %v", err)
-			}
-			l := newLayer(
-				&Resolver{
-					prefetchTimeout:       time.Second,
-					backgroundTaskManager: task.NewBackgroundTaskManager(10, 5*time.Second),
-				},
-				ocispec.Descriptor{Digest: testStateLayerDigest},
-				&blobRef{blob, func() {}},
-				vr,
-			)
-			if err := l.Verify(dgst); err != nil {
-				t.Errorf("failed to verify reader: %v", err)
-				return
-			}
-			prefetchSize := int64(0)
-			if tt.prefetchSize != nil {
-				prefetchSize = tt.prefetchSize(t, l)
-			}
-			if err := l.Prefetch(defaultPrefetchSize); err != nil {
-				t.Errorf("failed to prefetch: %v", err)
-				return
-			}
-			if blob.calledPrefetchOffset != 0 {
-				t.Errorf("invalid prefetch offset %d; want %d",
-					blob.calledPrefetchOffset, 0)
-			}
-			if blob.calledPrefetchSize != prefetchSize {
-				t.Errorf("invalid prefetch size %d; want %d",
-					blob.calledPrefetchSize, prefetchSize)
-			}
-			if cLen := len(mcache.(*cache.MemoryCache).Membuf); tt.wantNum != cLen {
-				t.Errorf("number of chunks in the cache %d; want %d: %v", cLen, tt.wantNum, err)
+func TestLayer(t *testing.T) {
+	testRunner := &TestRunner{
+		TestingT: t,
+		Runner: func(testingT TestingT, name string, run func(t TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
 				return
 			}
 
-			lr := l.r
-			if lr == nil {
-				t.Fatalf("failed to get reader from layer: %v", err)
-			}
-			for _, file := range tt.wants {
-				id, err := lookup(lr.Metadata(), file)
-				if err != nil {
-					t.Fatalf("failed to lookup %q: %v", file, err)
-				}
-				e, err := lr.Metadata().GetAttr(id)
-				if err != nil {
-					t.Fatalf("failed to get attr of %q: %v", file, err)
-				}
-				wantFile, err := lr.OpenFile(id)
-				if err != nil {
-					t.Fatalf("failed to open file %q", file)
-				}
-				blob.readCalled = false
-				if _, err := io.Copy(ioutil.Discard, io.NewSectionReader(wantFile, 0, e.Size)); err != nil {
-					t.Fatalf("failed to read file %q", file)
-				}
-				if blob.readCalled {
-					t.Errorf("chunks of file %q aren't cached", file)
-					return
-				}
-			}
-		})
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
 	}
-}
 
-func lookup(r metadata.Reader, name string) (uint32, error) {
-	name = strings.TrimPrefix(path.Clean("/"+name), "/")
-	if name == "" {
-		return r.RootID(), nil
-	}
-	dir, base := filepath.Split(name)
-	pid, err := lookup(r, dir)
-	if err != nil {
-		return 0, err
-	}
-	id, _, err := r.GetChild(pid, base)
-	return id, err
+	TestSuiteLayer(testRunner, memorymetadata.NewReader)
 }
 
-func chunkNum(data string) int {
-	return (len(data)-1)/sampleChunkSize + 1
-}
-
-func newBlob(sr *io.SectionReader) *sampleBlob {
-	return &sampleBlob{
-		r: sr,
-	}
-}
-
-type sampleBlob struct {
-	r                    *io.SectionReader
-	readCalled           bool
-	calledPrefetchOffset int64
-	calledPrefetchSize   int64
-}
-
-func (sb *sampleBlob) Authn(tr http.RoundTripper) (http.RoundTripper, error) { return nil, nil }
-func (sb *sampleBlob) Check() error                                          { return nil }
-func (sb *sampleBlob) Size() int64                                           { return sb.r.Size() }
-func (sb *sampleBlob) FetchedSize() int64                                    { return 0 }
-func (sb *sampleBlob) ReadAt(p []byte, offset int64, opts ...remote.Option) (int, error) {
-	sb.readCalled = true
-	return sb.r.ReadAt(p, offset)
-}
-func (sb *sampleBlob) Cache(offset int64, size int64, option ...remote.Option) error {
-	sb.calledPrefetchOffset = offset
-	sb.calledPrefetchSize = size
-	return nil
-}
-func (sb *sampleBlob) Refresh(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) error {
-	return nil
-}
-func (sb *sampleBlob) Close() error { return nil }
-
 func TestWaiter(t *testing.T) {
 	var (
 		w         = newWaiter()

fs/layer/node.go

@@ -36,7 +36,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	commonmetrics "github.com/containerd/stargz-snapshotter/fs/metrics/common"
 	"github.com/containerd/stargz-snapshotter/fs/reader"
@@ -45,33 +45,54 @@ import (
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
 	"github.com/hanwen/go-fuse/v2/fuse"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
 const (
 	blockSize         = 4096
-	whiteoutPrefix    = ".wh."
-	whiteoutOpaqueDir = whiteoutPrefix + whiteoutPrefix + ".opq"
-	opaqueXattrValue  = "y"
-	stateDirName      = ".stargz-snapshotter"
-	statFileMode      = syscall.S_IFREG | 0400 // -r--------
-	stateDirMode      = syscall.S_IFDIR | 0500 // dr-x------
+	physicalBlockSize = 512
+	// physicalBlockRatio is the ratio of blockSize to physicalBlockSize.
+	// It can be used to convert from # blockSize-byte blocks to # physicalBlockSize-byte blocks
+	physicalBlockRatio = blockSize / physicalBlockSize
+	whiteoutPrefix     = ".wh."
+	whiteoutOpaqueDir  = whiteoutPrefix + whiteoutPrefix + ".opq"
+	opaqueXattrValue   = "y"
+	stateDirName       = ".stargz-snapshotter"
+	statFileMode       = syscall.S_IFREG | 0400 // -r--------
+	stateDirMode       = syscall.S_IFDIR | 0500 // dr-x------
 )
 
-var opaqueXattrs = []string{"trusted.overlay.opaque", "user.overlay.opaque"}
+type OverlayOpaqueType int
 
-func newNode(layerDgst digest.Digest, r reader.Reader, blob remote.Blob, baseInode uint32) (fusefs.InodeEmbedder, error) {
+const (
+	OverlayOpaqueAll OverlayOpaqueType = iota
+	OverlayOpaqueTrusted
+	OverlayOpaqueUser
+)
+
+var opaqueXattrs = map[OverlayOpaqueType][]string{
+	OverlayOpaqueAll:     {"trusted.overlay.opaque", "user.overlay.opaque"},
+	OverlayOpaqueTrusted: {"trusted.overlay.opaque"},
+	OverlayOpaqueUser:    {"user.overlay.opaque"},
+}
+
+func newNode(layerDgst digest.Digest, r reader.Reader, blob remote.Blob, baseInode uint32, opaque OverlayOpaqueType, pth passThroughConfig) (fusefs.InodeEmbedder, error) {
 	rootID := r.Metadata().RootID()
 	rootAttr, err := r.Metadata().GetAttr(rootID)
 	if err != nil {
 		return nil, err
 	}
+	opq, ok := opaqueXattrs[opaque]
+	if !ok {
+		return nil, fmt.Errorf("unknown overlay opaque type")
+	}
 	ffs := &fs{
-		r:           r,
-		layerDigest: layerDgst,
-		baseInode:   baseInode,
-		rootID:      rootID,
+		r:            r,
+		layerDigest:  layerDgst,
+		baseInode:    baseInode,
+		rootID:       rootID,
+		opaqueXattrs: opq,
+		passThrough:  pth,
 	}
 	ffs.s = ffs.newState(layerDgst, blob)
 	return &node{
@@ -83,11 +104,13 @@ func newNode(layerDgst digest.Digest, r reader.Reader, blob remote.Blob, baseIno
 
 // fs contains global metadata used by nodes
 type fs struct {
-	r           reader.Reader
-	s           *state
-	layerDigest digest.Digest
-	baseInode   uint32
-	rootID      uint32
+	r            reader.Reader
+	s            *state
+	layerDigest  digest.Digest
+	baseInode    uint32
+	rootID       uint32
+	opaqueXattrs []string
+	passThrough  passThroughConfig
 }
 
 func (fs *fs) inodeOfState() uint64 {
@@ -109,11 +132,13 @@ func (fs *fs) inodeOfID(id uint32) (uint64, error) {
 // node is a filesystem inode abstraction.
 type node struct {
 	fusefs.Inode
-	fs         *fs
-	id         uint32
-	attr       metadata.Attr
+	fs   *fs
+	id   uint32
+	attr metadata.Attr
+
 	ents       []fuse.DirEntry
 	entsCached bool
+	entsMu     sync.Mutex
 }
 
 func (n *node) isRootNode() bool {
@@ -144,9 +169,13 @@ func (n *node) readdir() ([]fuse.DirEntry, syscall.Errno) {
 	start := time.Now() // set start time
 	defer commonmetrics.MeasureLatencyInMicroseconds(commonmetrics.NodeReaddir, n.fs.layerDigest, start)
 
+	n.entsMu.Lock()
 	if n.entsCached {
-		return n.ents, 0
+		ents := n.ents
+		n.entsMu.Unlock()
+		return ents, 0
 	}
+	n.entsMu.Unlock()
 
 	isRoot := n.isRootNode()
 
@@ -210,6 +239,8 @@ func (n *node) readdir() ([]fuse.DirEntry, syscall.Errno) {
 	sort.Slice(ents, func(i, j int) bool {
 		return ents[i].Name < ents[j].Name
 	})
+	n.entsMu.Lock()
+	defer n.entsMu.Unlock()
 	n.ents, n.entsCached = ents, true // cache it
 
 	return ents, 0
@@ -261,6 +292,7 @@ func (n *node) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fu
 	}
 
 	// early return if this entry doesn't exist
+	n.entsMu.Lock()
 	if n.entsCached {
 		var found bool
 		for _, e := range n.ents {
@@ -269,9 +301,11 @@ func (n *node) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fu
 			}
 		}
 		if !found {
+			n.entsMu.Unlock()
 			return nil, syscall.ENOENT
 		}
 	}
+	n.entsMu.Unlock()
 
 	id, ce, err := n.fs.r.Metadata().GetChild(n.id, name)
 	if err != nil {
@@ -312,10 +346,26 @@ func (n *node) Open(ctx context.Context, flags uint32) (fh fusefs.FileHandle, fu
 		n.fs.s.report(fmt.Errorf("node.Open: %v", err))
 		return nil, 0, syscall.EIO
 	}
-	return &file{
+
+	f := &file{
 		n:  n,
 		ra: ra,
-	}, fuse.FOPEN_KEEP_CACHE, 0
+		fd: -1,
+	}
+
+	if n.fs.passThrough.enable {
+		if getter, ok := ra.(reader.PassthroughFdGetter); ok {
+			fd, err := getter.GetPassthroughFd(n.fs.passThrough.mergeBufferSize, n.fs.passThrough.mergeWorkerCount)
+			if err != nil {
+				n.fs.s.report(fmt.Errorf("passThrough model failed due to node.Open: %v", err))
+				n.fs.passThrough.enable = false
+			} else {
+				f.InitFd(int(fd))
+			}
+		}
+	}
+
+	return f, fuse.FOPEN_KEEP_CACHE, 0
 }
 
 var _ = (fusefs.NodeGetattrer)((*node)(nil))
@@ -335,7 +385,7 @@ var _ = (fusefs.NodeGetxattrer)((*node)(nil))
 func (n *node) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) {
 	ent := n.attr
 	opq := n.isOpaque()
-	for _, opaqueXattr := range opaqueXattrs {
+	for _, opaqueXattr := range n.fs.opaqueXattrs {
 		if attr == opaqueXattr && opq {
 			// This node is an opaque directory so give overlayfs-compliant indicator.
 			if len(dest) < len(opaqueXattrValue) {
@@ -361,7 +411,7 @@ func (n *node) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errn
 	var attrs []byte
 	if opq {
 		// This node is an opaque directory so add overlayfs-compliant indicator.
-		for _, opaqueXattr := range opaqueXattrs {
+		for _, opaqueXattr := range n.fs.opaqueXattrs {
 			attrs = append(attrs, []byte(opaqueXattr+"\x00")...)
 		}
 	}
@@ -392,6 +442,7 @@ func (n *node) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno {
 type file struct {
 	n  *node
 	ra io.ReaderAt
+	fd int
 }
 
 var _ = (fusefs.FileReader)((*file)(nil))
@@ -419,6 +470,20 @@ func (f *file) Getattr(ctx context.Context, out *fuse.AttrOut) syscall.Errno {
 	return 0
 }
 
+// Implement PassthroughFd to enable go-fuse passthrough
+var _ = (fusefs.FilePassthroughFder)((*file)(nil))
+
+func (f *file) PassthroughFd() (int, bool) {
+	if f.fd <= 0 {
+		return -1, false
+	}
+	return f.fd, true
+}
+
+func (f *file) InitFd(fd int) {
+	f.fd = fd
+}
+
 // whiteout is a whiteout abstraction compliant to overlayfs.
 type whiteout struct {
 	fusefs.Inode
@@ -579,7 +644,7 @@ func (sf *statFile) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Err
 // The entries naming is kept to be consistend with the field naming in statJSON.
 func (sf *statFile) logContents() {
 	ctx := context.Background()
-	log.G(ctx).WithFields(logrus.Fields{
+	log.G(ctx).WithFields(log.Fields{
 		"digest": sf.statJSON.Digest, "size": sf.statJSON.Size,
 		"fetchedSize": sf.statJSON.FetchedSize, "fetchedPercent": sf.statJSON.FetchedPercent,
 	}).WithError(errors.New(sf.statJSON.Error)).Error("statFile error")
@@ -619,11 +684,11 @@ func (sf *statFile) updateStatUnlocked() ([]byte, error) {
 func entryToAttr(ino uint64, e metadata.Attr, out *fuse.Attr) fusefs.StableAttr {
 	out.Ino = ino
 	out.Size = uint64(e.Size)
-	out.Blksize = blockSize
-	out.Blocks = out.Size / uint64(out.Blksize)
-	if out.Size%uint64(out.Blksize) > 0 {
-		out.Blocks++
+	if e.Mode&os.ModeSymlink != 0 {
+		out.Size = uint64(len(e.LinkName))
 	}
+	out.Blksize = blockSize
+	out.Blocks = (out.Size + uint64(out.Blksize) - 1) / uint64(out.Blksize) * physicalBlockRatio
 	mtime := e.ModTime
 	out.SetTimes(nil, &mtime, nil)
 	out.Mode = fileModeToSystemMode(e.Mode)
@@ -700,7 +765,7 @@ func (fs *fs) statFileToAttr(size uint64, out *fuse.Attr) fusefs.StableAttr {
 	out.Ino = fs.inodeOfStatFile()
 	out.Size = size
 	out.Blksize = blockSize
-	out.Blocks = out.Size / uint64(out.Blksize)
+	out.Blocks = (out.Size + uint64(out.Blksize) - 1) / uint64(out.Blksize) * physicalBlockRatio
 	out.Nlink = 1
 
 	// Root can read it ("-r-------- root root").
@@ -767,7 +832,7 @@ func defaultStatfs(stat *fuse.StatfsOut) {
 	stat.Files = 0 // dummy
 	stat.Ffree = 0
 	stat.Bsize = blockSize
-	stat.NameLen = 1<<32 - 1
+	stat.NameLen = 255 // Standard max filename length for most filesystems (ext4, etc.) for compatibility
 	stat.Frsize = blockSize
 	stat.Padding = 0
 	stat.Spare = [6]uint32{}

fs/layer/node_test.go

@@ -1,721 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-/*
-   Copyright 2019 The Go Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style
-   license that can be found in the NOTICE.md file.
-*/
-
-package layer
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"strings"
-	"syscall"
-	"testing"
-	"time"
-
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/stargz-snapshotter/estargz"
-	"github.com/containerd/stargz-snapshotter/fs/reader"
-	"github.com/containerd/stargz-snapshotter/fs/remote"
-	"github.com/containerd/stargz-snapshotter/fs/source"
-	"github.com/containerd/stargz-snapshotter/metadata"
-	dbmetadata "github.com/containerd/stargz-snapshotter/metadata/db"
-	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
-	"github.com/containerd/stargz-snapshotter/util/testutil"
-	fusefs "github.com/hanwen/go-fuse/v2/fs"
-	"github.com/hanwen/go-fuse/v2/fuse"
-	"github.com/hashicorp/go-multierror"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	bolt "go.etcd.io/bbolt"
-	"golang.org/x/sys/unix"
-)
-
-const (
-	sampleMiddleOffset = sampleChunkSize / 2
-	lastChunkOffset1   = sampleChunkSize * (int64(len(sampleData1)) / sampleChunkSize)
-)
-
-// Tests Read method of each file node.
-func TestNodeRead(t *testing.T) {
-	testNodeRead(t, "db", newDBReader)
-	testNodeRead(t, "mem", newMemoryReader)
-}
-
-func testNodeRead(t *testing.T, name string, factory readerFactory) {
-	sizeCond := map[string]int64{
-		"single_chunk": sampleChunkSize - sampleMiddleOffset,
-		"multi_chunks": sampleChunkSize + sampleMiddleOffset,
-	}
-	innerOffsetCond := map[string]int64{
-		"at_top":    0,
-		"at_middle": sampleMiddleOffset,
-	}
-	baseOffsetCond := map[string]int64{
-		"of_1st_chunk":  sampleChunkSize * 0,
-		"of_2nd_chunk":  sampleChunkSize * 1,
-		"of_last_chunk": lastChunkOffset1,
-	}
-	fileSizeCond := map[string]int64{
-		"in_1_chunk_file":  sampleChunkSize * 1,
-		"in_2_chunks_file": sampleChunkSize * 2,
-		"in_max_size_file": int64(len(sampleData1)),
-	}
-	for sn, size := range sizeCond {
-		for in, innero := range innerOffsetCond {
-			for bo, baseo := range baseOffsetCond {
-				for fn, filesize := range fileSizeCond {
-					t.Run(fmt.Sprintf("reading_%s_%s_%s_%s_%s", name, sn, in, bo, fn), func(t *testing.T) {
-						if filesize > int64(len(sampleData1)) {
-							t.Fatal("sample file size is larger than sample data")
-						}
-
-						wantN := size
-						offset := baseo + innero
-						if remain := filesize - offset; remain < wantN {
-							if wantN = remain; wantN < 0 {
-								wantN = 0
-							}
-						}
-
-						// use constant string value as a data source.
-						want := strings.NewReader(sampleData1)
-
-						// data we want to get.
-						wantData := make([]byte, wantN)
-						_, err := want.ReadAt(wantData, offset)
-						if err != nil && err != io.EOF {
-							t.Fatalf("want.ReadAt (offset=%d,size=%d): %v", offset, wantN, err)
-						}
-
-						// data we get from the file node.
-						f, closeFn := makeNodeReader(t, []byte(sampleData1)[:filesize], sampleChunkSize, factory)
-						defer closeFn()
-						tmpbuf := make([]byte, size) // fuse library can request bigger than remain
-						rr, errno := f.Read(context.Background(), tmpbuf, offset)
-						if errno != 0 {
-							t.Errorf("failed to read off=%d, size=%d, filesize=%d: %v", offset, size, filesize, err)
-							return
-						}
-						if rsize := rr.Size(); int64(rsize) != wantN {
-							t.Errorf("read size: %d; want: %d; passed %d", rsize, wantN, size)
-							return
-						}
-						tmpbuf = make([]byte, len(tmpbuf))
-						respData, fs := rr.Bytes(tmpbuf)
-						if fs != fuse.OK {
-							t.Errorf("failed to read result data for off=%d, size=%d, filesize=%d: %v", offset, size, filesize, err)
-						}
-
-						if !bytes.Equal(wantData, respData) {
-							t.Errorf("off=%d, filesize=%d; read data{size=%d,data=%q}; want (size=%d,data=%q)",
-								offset, filesize, len(respData), string(respData), wantN, string(wantData))
-							return
-						}
-					})
-				}
-			}
-		}
-	}
-}
-
-func makeNodeReader(t *testing.T, contents []byte, chunkSize int, factory readerFactory) (_ *file, closeFn func() error) {
-	testName := "test"
-	sr, _, err := testutil.BuildEStargz(
-		[]testutil.TarEntry{testutil.File(testName, string(contents))},
-		testutil.WithEStargzOptions(estargz.WithChunkSize(chunkSize)),
-	)
-	if err != nil {
-		t.Fatalf("failed to build sample eStargz: %v", err)
-	}
-	r, closeFn, err := factory(sr)
-	if err != nil {
-		t.Fatalf("failed to create reader: %v", err)
-	}
-	var closes closeFuncs
-	closes = append(closes, r.Close, closeFn)
-	rootNode := getRootNode(t, r)
-	var eo fuse.EntryOut
-	inode, errno := rootNode.Lookup(context.Background(), testName, &eo)
-	if errno != 0 {
-		closes.close()
-		t.Fatalf("failed to lookup test node; errno: %v", errno)
-	}
-	f, _, errno := inode.Operations().(fusefs.NodeOpener).Open(context.Background(), 0)
-	if errno != 0 {
-		closes.close()
-		t.Fatalf("failed to open test file; errno: %v", errno)
-	}
-	return f.(*file), closes.close
-}
-
-func TestExistence(t *testing.T) {
-	tests := []struct {
-		name string
-		in   []testutil.TarEntry
-		want []check
-	}{
-		{
-			name: "1_whiteout_with_sibling",
-			in: []testutil.TarEntry{
-				testutil.Dir("foo/"),
-				testutil.File("foo/bar.txt", ""),
-				testutil.File("foo/.wh.foo.txt", ""),
-			},
-			want: []check{
-				hasValidWhiteout("foo/foo.txt"),
-				fileNotExist("foo/.wh.foo.txt"),
-			},
-		},
-		{
-			name: "1_whiteout_with_duplicated_name",
-			in: []testutil.TarEntry{
-				testutil.Dir("foo/"),
-				testutil.File("foo/bar.txt", "test"),
-				testutil.File("foo/.wh.bar.txt", ""),
-			},
-			want: []check{
-				hasFileDigest("foo/bar.txt", digestFor("test")),
-				fileNotExist("foo/.wh.bar.txt"),
-			},
-		},
-		{
-			name: "1_opaque",
-			in: []testutil.TarEntry{
-				testutil.Dir("foo/"),
-				testutil.File("foo/.wh..wh..opq", ""),
-			},
-			want: []check{
-				hasNodeXattrs("foo/", opaqueXattrs[0], opaqueXattrValue),
-				hasNodeXattrs("foo/", opaqueXattrs[1], opaqueXattrValue),
-				fileNotExist("foo/.wh..wh..opq"),
-			},
-		},
-		{
-			name: "1_opaque_with_sibling",
-			in: []testutil.TarEntry{
-				testutil.Dir("foo/"),
-				testutil.File("foo/.wh..wh..opq", ""),
-				testutil.File("foo/bar.txt", "test"),
-			},
-			want: []check{
-				hasNodeXattrs("foo/", opaqueXattrs[0], opaqueXattrValue),
-				hasNodeXattrs("foo/", opaqueXattrs[1], opaqueXattrValue),
-				hasFileDigest("foo/bar.txt", digestFor("test")),
-				fileNotExist("foo/.wh..wh..opq"),
-			},
-		},
-		{
-			name: "1_opaque_with_xattr",
-			in: []testutil.TarEntry{
-				testutil.Dir("foo/", testutil.WithDirXattrs(map[string]string{"foo": "bar"})),
-				testutil.File("foo/.wh..wh..opq", ""),
-			},
-			want: []check{
-				hasNodeXattrs("foo/", opaqueXattrs[0], opaqueXattrValue),
-				hasNodeXattrs("foo/", opaqueXattrs[1], opaqueXattrValue),
-				hasNodeXattrs("foo/", "foo", "bar"),
-				fileNotExist("foo/.wh..wh..opq"),
-			},
-		},
-		{
-			name: "prefetch_landmark",
-			in: []testutil.TarEntry{
-				testutil.File(estargz.PrefetchLandmark, "test"),
-				testutil.Dir("foo/"),
-				testutil.File(fmt.Sprintf("foo/%s", estargz.PrefetchLandmark), "test"),
-			},
-			want: []check{
-				fileNotExist(estargz.PrefetchLandmark),
-				hasFileDigest(fmt.Sprintf("foo/%s", estargz.PrefetchLandmark), digestFor("test")),
-			},
-		},
-		{
-			name: "no_prefetch_landmark",
-			in: []testutil.TarEntry{
-				testutil.File(estargz.NoPrefetchLandmark, "test"),
-				testutil.Dir("foo/"),
-				testutil.File(fmt.Sprintf("foo/%s", estargz.NoPrefetchLandmark), "test"),
-			},
-			want: []check{
-				fileNotExist(estargz.NoPrefetchLandmark),
-				hasFileDigest(fmt.Sprintf("foo/%s", estargz.NoPrefetchLandmark), digestFor("test")),
-			},
-		},
-		{
-			name: "state_file",
-			in: []testutil.TarEntry{
-				testutil.File("test", "test"),
-			},
-			want: []check{
-				hasFileDigest("test", digestFor("test")),
-				hasStateFile(t, testStateLayerDigest.String()+".json"),
-			},
-		},
-		{
-			name: "file_suid",
-			in: []testutil.TarEntry{
-				testutil.File("test", "test", testutil.WithFileMode(0644|os.ModeSetuid)),
-			},
-			want: []check{
-				hasExtraMode("test", os.ModeSetuid),
-			},
-		},
-		{
-			name: "dir_sgid",
-			in: []testutil.TarEntry{
-				testutil.Dir("test/", testutil.WithDirMode(0755|os.ModeSetgid)),
-			},
-			want: []check{
-				hasExtraMode("test/", os.ModeSetgid),
-			},
-		},
-		{
-			name: "file_sticky",
-			in: []testutil.TarEntry{
-				testutil.File("test", "test", testutil.WithFileMode(0644|os.ModeSticky)),
-			},
-			want: []check{
-				hasExtraMode("test", os.ModeSticky),
-			},
-		},
-	}
-
-	for readerName, factory := range map[string]readerFactory{"db": newDBReader, "mem": newMemoryReader} {
-		for _, tt := range tests {
-			t.Run(tt.name+"-"+readerName, func(t *testing.T) {
-				sgz, _, err := testutil.BuildEStargz(tt.in)
-				if err != nil {
-					t.Fatalf("failed to build sample eStargz: %v", err)
-				}
-
-				r, closeFn, err := factory(sgz)
-				if err != nil {
-					t.Fatalf("failed to create reader: %v", err)
-				}
-				defer r.Close()
-				defer closeFn()
-				rootNode := getRootNode(t, r)
-				for _, want := range tt.want {
-					want(t, rootNode)
-				}
-			})
-		}
-	}
-}
-
-func getRootNode(t *testing.T, r metadata.Reader) *node {
-	rootNode, err := newNode(testStateLayerDigest, &testReader{r}, &testBlobState{10, 5}, 100)
-	if err != nil {
-		t.Fatalf("failed to get root node: %v", err)
-	}
-	fusefs.NewNodeFS(rootNode, &fusefs.Options{}) // initializes root node
-	return rootNode.(*node)
-}
-
-type testReader struct {
-	r metadata.Reader
-}
-
-func (tr *testReader) OpenFile(id uint32) (io.ReaderAt, error) { return tr.r.OpenFile(id) }
-func (tr *testReader) Metadata() metadata.Reader               { return tr.r }
-func (tr *testReader) Cache(opts ...reader.CacheOption) error  { return nil }
-func (tr *testReader) Close() error                            { return nil }
-func (tr *testReader) LastOnDemandReadTime() time.Time         { return time.Now() }
-
-type testBlobState struct {
-	size        int64
-	fetchedSize int64
-}
-
-func (tb *testBlobState) Check() error       { return nil }
-func (tb *testBlobState) Size() int64        { return tb.size }
-func (tb *testBlobState) FetchedSize() int64 { return tb.fetchedSize }
-func (tb *testBlobState) ReadAt(p []byte, offset int64, opts ...remote.Option) (int, error) {
-	return 0, nil
-}
-func (tb *testBlobState) Cache(offset int64, size int64, opts ...remote.Option) error { return nil }
-func (tb *testBlobState) Refresh(ctx context.Context, host source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) error {
-	return nil
-}
-func (tb *testBlobState) Close() error { return nil }
-
-type check func(*testing.T, *node)
-
-func fileNotExist(file string) check {
-	return func(t *testing.T, root *node) {
-		if _, _, err := getDirentAndNode(t, root, file); err == nil {
-			t.Errorf("Node %q exists", file)
-		}
-	}
-}
-
-func hasFileDigest(filename string, digest string) check {
-	return func(t *testing.T, root *node) {
-		_, n, err := getDirentAndNode(t, root, filename)
-		if err != nil {
-			t.Fatalf("failed to get node %q: %v", filename, err)
-		}
-		ni := n.Operations().(*node)
-		attr, err := ni.fs.r.Metadata().GetAttr(ni.id)
-		if err != nil {
-			t.Fatalf("failed to get attr %q(%d): %v", filename, ni.id, err)
-		}
-		fh, _, errno := ni.Open(context.Background(), 0)
-		if errno != 0 {
-			t.Fatalf("failed to open node %q: %v", filename, errno)
-		}
-		rr, errno := fh.(*file).Read(context.Background(), make([]byte, attr.Size), 0)
-		if errno != 0 {
-			t.Fatalf("failed to read node %q: %v", filename, errno)
-		}
-		res, status := rr.Bytes(make([]byte, attr.Size))
-		if status != fuse.OK {
-			t.Fatalf("failed to get read result of node %q: %v", filename, status)
-		}
-		if ndgst := digestFor(string(res)); ndgst != digest {
-			t.Fatalf("Digest(%q) = %q, want %q", filename, ndgst, digest)
-		}
-	}
-}
-
-func hasExtraMode(name string, mode os.FileMode) check {
-	return func(t *testing.T, root *node) {
-		_, n, err := getDirentAndNode(t, root, name)
-		if err != nil {
-			t.Fatalf("failed to get node %q: %v", name, err)
-		}
-		var ao fuse.AttrOut
-		if errno := n.Operations().(fusefs.NodeGetattrer).Getattr(context.Background(), nil, &ao); errno != 0 {
-			t.Fatalf("failed to get attributes of node %q: %v", name, errno)
-		}
-		a := ao.Attr
-		gotMode := a.Mode & (syscall.S_ISUID | syscall.S_ISGID | syscall.S_ISVTX)
-		wantMode := extraModeToTarMode(mode)
-		if gotMode != uint32(wantMode) {
-			t.Fatalf("got mode = %b, want %b", gotMode, wantMode)
-		}
-	}
-}
-
-func hasValidWhiteout(name string) check {
-	return func(t *testing.T, root *node) {
-		ent, n, err := getDirentAndNode(t, root, name)
-		if err != nil {
-			t.Fatalf("failed to get node %q: %v", name, err)
-		}
-		var ao fuse.AttrOut
-		if errno := n.Operations().(fusefs.NodeGetattrer).Getattr(context.Background(), nil, &ao); errno != 0 {
-			t.Fatalf("failed to get attributes of file %q: %v", name, errno)
-		}
-		a := ao.Attr
-		if a.Ino != ent.Ino {
-			t.Errorf("inconsistent inodes %d(Node) != %d(Dirent)", a.Ino, ent.Ino)
-			return
-		}
-
-		// validate the direntry
-		if ent.Mode != syscall.S_IFCHR {
-			t.Errorf("whiteout entry %q isn't a char device", name)
-			return
-		}
-
-		// validate the node
-		if a.Mode != syscall.S_IFCHR {
-			t.Errorf("whiteout %q has an invalid mode %o; want %o",
-				name, a.Mode, syscall.S_IFCHR)
-			return
-		}
-		if a.Rdev != uint32(unix.Mkdev(0, 0)) {
-			t.Errorf("whiteout %q has invalid device numbers (%d, %d); want (0, 0)",
-				name, unix.Major(uint64(a.Rdev)), unix.Minor(uint64(a.Rdev)))
-			return
-		}
-	}
-}
-
-func hasNodeXattrs(entry, name, value string) check {
-	return func(t *testing.T, root *node) {
-		_, n, err := getDirentAndNode(t, root, entry)
-		if err != nil {
-			t.Fatalf("failed to get node %q: %v", entry, err)
-		}
-
-		// check xattr exists in the xattrs list.
-		buf := make([]byte, 1000)
-		nb, errno := n.Operations().(fusefs.NodeListxattrer).Listxattr(context.Background(), buf)
-		if errno != 0 {
-			t.Fatalf("failed to get xattrs list of node %q: %v", entry, err)
-		}
-		attrs := strings.Split(string(buf[:nb]), "\x00")
-		var found bool
-		for _, x := range attrs {
-			if x == name {
-				found = true
-			}
-		}
-		if !found {
-			t.Errorf("node %q doesn't have an opaque xattr %q", entry, value)
-			return
-		}
-
-		// check the xattr has valid value.
-		v := make([]byte, len(value))
-		nv, errno := n.Operations().(fusefs.NodeGetxattrer).Getxattr(context.Background(), name, v)
-		if errno != 0 {
-			t.Fatalf("failed to get xattr %q of node %q: %v", name, entry, err)
-		}
-		if int(nv) != len(value) {
-			t.Fatalf("invalid xattr size for file %q, value %q got %d; want %d",
-				name, value, nv, len(value))
-		}
-		if string(v) != value {
-			t.Errorf("node %q has an invalid xattr %q; want %q", entry, v, value)
-			return
-		}
-	}
-}
-
-func hasEntry(t *testing.T, name string, ents fusefs.DirStream) (fuse.DirEntry, bool) {
-	for ents.HasNext() {
-		de, errno := ents.Next()
-		if errno != 0 {
-			t.Fatalf("faield to read entries for %q", name)
-		}
-		if de.Name == name {
-			return de, true
-		}
-	}
-	return fuse.DirEntry{}, false
-}
-
-func hasStateFile(t *testing.T, id string) check {
-	return func(t *testing.T, root *node) {
-
-		// Check the state dir is hidden on OpenDir for "/"
-		ents, errno := root.Readdir(context.Background())
-		if errno != 0 {
-			t.Errorf("failed to open root directory: %v", errno)
-			return
-		}
-		if _, ok := hasEntry(t, stateDirName, ents); ok {
-			t.Errorf("state direntry %q should not be listed", stateDirName)
-			return
-		}
-
-		// Check existence of state dir
-		var eo fuse.EntryOut
-		sti, errno := root.Lookup(context.Background(), stateDirName, &eo)
-		if errno != 0 {
-			t.Errorf("failed to lookup directory %q: %v", stateDirName, errno)
-			return
-		}
-		st, ok := sti.Operations().(*state)
-		if !ok {
-			t.Errorf("directory %q isn't a state node", stateDirName)
-			return
-		}
-
-		// Check existence of state file
-		ents, errno = st.Readdir(context.Background())
-		if errno != 0 {
-			t.Errorf("failed to open directory %q: %v", stateDirName, errno)
-			return
-		}
-		if _, ok := hasEntry(t, id, ents); !ok {
-			t.Errorf("direntry %q not found in %q", id, stateDirName)
-			return
-		}
-		inode, errno := st.Lookup(context.Background(), id, &eo)
-		if errno != 0 {
-			t.Errorf("failed to lookup node %q in %q: %v", id, stateDirName, errno)
-			return
-		}
-		n, ok := inode.Operations().(*statFile)
-		if !ok {
-			t.Errorf("entry %q isn't a normal node", id)
-			return
-		}
-
-		// wanted data
-		rand.Seed(time.Now().UnixNano())
-		wantErr := fmt.Errorf("test-%d", rand.Int63())
-
-		// report the data
-		root.fs.s.report(wantErr)
-
-		// obtain file size (check later)
-		var ao fuse.AttrOut
-		errno = n.Operations().(fusefs.NodeGetattrer).Getattr(context.Background(), nil, &ao)
-		if errno != 0 {
-			t.Errorf("failed to get attr of state file: %v", errno)
-			return
-		}
-		attr := ao.Attr
-
-		// get data via state file
-		tmp := make([]byte, 4096)
-		res, errno := n.Read(context.Background(), nil, tmp, 0)
-		if errno != 0 {
-			t.Errorf("failed to read state file: %v", errno)
-			return
-		}
-		gotState, status := res.Bytes(nil)
-		if status != fuse.OK {
-			t.Errorf("failed to get result bytes of state file: %v", errno)
-			return
-		}
-		if attr.Size != uint64(len(string(gotState))) {
-			t.Errorf("size %d; want %d", attr.Size, len(string(gotState)))
-			return
-		}
-
-		var j statJSON
-		if err := json.Unmarshal(gotState, &j); err != nil {
-			t.Errorf("failed to unmarshal %q: %v", string(gotState), err)
-			return
-		}
-		if wantErr.Error() != j.Error {
-			t.Errorf("expected error %q, got %q", wantErr.Error(), j.Error)
-			return
-		}
-	}
-}
-
-// getDirentAndNode gets dirent and node at the specified path at once and makes
-// sure that the both of them exist.
-func getDirentAndNode(t *testing.T, root *node, path string) (ent fuse.DirEntry, n *fusefs.Inode, err error) {
-	dir, base := filepath.Split(filepath.Clean(path))
-
-	// get the target's parent directory.
-	var eo fuse.EntryOut
-	d := root
-	for _, name := range strings.Split(dir, "/") {
-		if len(name) == 0 {
-			continue
-		}
-		di, errno := d.Lookup(context.Background(), name, &eo)
-		if errno != 0 {
-			err = fmt.Errorf("failed to lookup directory %q: %v", name, errno)
-			return
-		}
-		var ok bool
-		if d, ok = di.Operations().(*node); !ok {
-			err = fmt.Errorf("directory %q isn't a normal node", name)
-			return
-		}
-
-	}
-
-	// get the target's direntry.
-	ents, errno := d.Readdir(context.Background())
-	if errno != 0 {
-		err = fmt.Errorf("failed to open directory %q: %v", path, errno)
-	}
-	ent, ok := hasEntry(t, base, ents)
-	if !ok {
-		err = fmt.Errorf("direntry %q not found in the parent directory of %q", base, path)
-	}
-
-	// get the target's node.
-	n, errno = d.Lookup(context.Background(), base, &eo)
-	if errno != 0 {
-		err = fmt.Errorf("failed to lookup node %q: %v", path, errno)
-	}
-
-	return
-}
-
-func digestFor(content string) string {
-	sum := sha256.Sum256([]byte(content))
-	return fmt.Sprintf("sha256:%x", sum)
-}
-
-// suid, guid, sticky bits for archive/tar
-// https://github.com/golang/go/blob/release-branch.go1.13/src/archive/tar/common.go#L607-L609
-const (
-	cISUID = 04000 // Set uid
-	cISGID = 02000 // Set gid
-	cISVTX = 01000 // Save text (sticky bit)
-)
-
-func extraModeToTarMode(fm os.FileMode) (tm int64) {
-	if fm&os.ModeSetuid != 0 {
-		tm |= cISUID
-	}
-	if fm&os.ModeSetgid != 0 {
-		tm |= cISGID
-	}
-	if fm&os.ModeSticky != 0 {
-		tm |= cISVTX
-	}
-	return
-}
-
-type readerFactory func(sr *io.SectionReader) (mr metadata.Reader, close func() error, err error)
-
-func newDBReader(sr *io.SectionReader) (metadata.Reader, func() error, error) {
-	var closes closeFuncs
-	f, err := ioutil.TempFile("", "readertest")
-	if err != nil {
-		return nil, nil, err
-	}
-	closes = append(closes, func() error { return os.Remove(f.Name()) })
-	db, err := bolt.Open(f.Name(), 0666, nil)
-	if err != nil {
-		closes.close()
-		return nil, nil, err
-	}
-	closes = append(closes, db.Close)
-	mr, err := dbmetadata.NewReader(db, sr)
-	if err != nil {
-		closes.close()
-		return nil, nil, err
-	}
-	closes = append(closes, mr.Close)
-	return mr, closes.close, nil
-}
-
-func newMemoryReader(sr *io.SectionReader) (metadata.Reader, func() error, error) {
-	mr, err := memorymetadata.NewReader(sr)
-	if err != nil {
-		return nil, nil, err
-	}
-	return mr, func() error { return nil }, err
-}
-
-type closeFuncs []func() error
-
-func (fs closeFuncs) close() error {
-	var allErr error
-	for _, f := range fs {
-		if err := f(); err != nil {
-			allErr = multierror.Append(allErr, err)
-		}
-	}
-	return allErr
-}

fs/metrics/common/metrics.go

@@ -21,7 +21,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -129,6 +129,7 @@ var (
 )
 
 var register sync.Once
+var logLevel = log.DebugLevel
 
 // sinceInMilliseconds gets the time since the specified start in milliseconds.
 // The division by 1e6 is made to have the milliseconds value as floating point number, since the native method
@@ -145,8 +146,9 @@ func sinceInMicroseconds(start time.Time) float64 {
 }
 
 // Register registers metrics. This is always called only once.
-func Register() {
+func Register(l log.Level) {
 	register.Do(func() {
+		logLevel = l
 		prometheus.MustRegister(operationLatencyMilliseconds)
 		prometheus.MustRegister(operationLatencyMicroseconds)
 		prometheus.MustRegister(operationCount)
@@ -185,14 +187,14 @@ func AddBytesCount(operation string, layer digest.Digest, bytes int64) {
 // WriteLatencyLogValue wraps writing the log info record for latency in milliseconds. The log record breaks down by operation and layer digest.
 func WriteLatencyLogValue(ctx context.Context, layer digest.Digest, operation string, start time.Time) {
 	ctx = log.WithLogger(ctx, log.G(ctx).WithField("metrics", "latency").WithField("operation", operation).WithField("layer_sha", layer.String()))
-	log.G(ctx).Infof("value=%v milliseconds", sinceInMilliseconds(start))
+	log.G(ctx).Logf(logLevel, "value=%v milliseconds", sinceInMilliseconds(start))
 }
 
 // WriteLatencyWithBytesLogValue wraps writing the log info record for latency in milliseconds with adding the size in bytes.
 // The log record breaks down by operation, layer digest and byte value.
 func WriteLatencyWithBytesLogValue(ctx context.Context, layer digest.Digest, latencyOperation string, start time.Time, bytesMetricName string, bytesMetricValue int64) {
 	ctx = log.WithLogger(ctx, log.G(ctx).WithField("metrics", "latency").WithField("operation", latencyOperation).WithField("layer_sha", layer.String()))
-	log.G(ctx).Infof("value=%v milliseconds; %v=%v bytes", sinceInMilliseconds(start), bytesMetricName, bytesMetricValue)
+	log.G(ctx).Logf(logLevel, "value=%v milliseconds; %v=%v bytes", sinceInMilliseconds(start), bytesMetricName, bytesMetricValue)
 }
 
 // LogLatencyForLastOnDemandFetch implements a special case for measuring the latency of last on demand fetch, which must be invoked at the end of
@@ -209,6 +211,6 @@ func LogLatencyForLastOnDemandFetch(ctx context.Context, layer digest.Digest, st
 	// this can happen if there were no on-demand fetch for the particular layer
 	if diffInMilliseconds > 0 {
 		ctx = log.WithLogger(ctx, log.G(ctx).WithField("metrics", "latency").WithField("operation", MountLayerToLastOnDemandFetch).WithField("layer_sha", layer.String()))
-		log.G(ctx).Infof("value=%v milliseconds", diffInMilliseconds)
+		log.G(ctx).Logf(logLevel, "value=%v milliseconds", diffInMilliseconds)
 	}
 }

fs/reader/reader.go

@@ -27,22 +27,20 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"runtime"
+	"sort"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/containerd/stargz-snapshotter/cache"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	commonmetrics "github.com/containerd/stargz-snapshotter/fs/metrics/common"
 	"github.com/containerd/stargz-snapshotter/metadata"
-	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/semaphore"
 )
@@ -56,11 +54,16 @@ type Reader interface {
 	LastOnDemandReadTime() time.Time
 }
 
+type PassthroughFdGetter interface {
+	GetPassthroughFd(mergeBufferSize int64, mergeWorkerCount int) (uintptr, error)
+}
+
 // VerifiableReader produces a Reader with a given verifier.
 type VerifiableReader struct {
 	r *reader
 
-	lastVerifyErr           atomic.Value
+	lastVerifyErr           error
+	lastVerifyErrMu         sync.Mutex
 	prohibitVerifyFailure   bool
 	prohibitVerifyFailureMu sync.RWMutex
 
@@ -70,6 +73,19 @@ type VerifiableReader struct {
 	verifier func(uint32, string) (digest.Verifier, error)
 }
 
+func (vr *VerifiableReader) storeLastVerifyErr(err error) {
+	vr.lastVerifyErrMu.Lock()
+	vr.lastVerifyErr = err
+	vr.lastVerifyErrMu.Unlock()
+}
+
+func (vr *VerifiableReader) loadLastVerifyErr() error {
+	vr.lastVerifyErrMu.Lock()
+	err := vr.lastVerifyErr
+	vr.lastVerifyErrMu.Unlock()
+	return err
+}
+
 func (vr *VerifiableReader) SkipVerify() Reader {
 	return vr.r
 }
@@ -80,10 +96,10 @@ func (vr *VerifiableReader) VerifyTOC(tocDigest digest.Digest) (Reader, error) {
 	}
 	vr.prohibitVerifyFailureMu.Lock()
 	vr.prohibitVerifyFailure = true
-	lastVerifyErr := vr.lastVerifyErr.Load()
+	lastVerifyErr := vr.loadLastVerifyErr()
 	vr.prohibitVerifyFailureMu.Unlock()
 	if err := lastVerifyErr; err != nil {
-		return nil, errors.Wrapf(err.(error), "content error occures during caching contents")
+		return nil, fmt.Errorf("content error occurs during caching contents: %w", err)
 	}
 	if actual := vr.r.r.TOCDigest(); actual != tocDigest {
 		return nil, fmt.Errorf("invalid TOC JSON %q; want %q", actual, tocDigest)
@@ -137,7 +153,6 @@ func (vr *VerifiableReader) cacheWithReader(ctx context.Context, currentDepth in
 	if currentDepth > maxWalkDepth {
 		return fmt.Errorf("tree is too deep (depth:%d)", currentDepth)
 	}
-	gr := vr.r
 	rootID := r.RootID()
 	r.ForeachChild(dirID, func(name string, id uint32, mode os.FileMode) bool {
 		e, err := r.GetAttr(id)
@@ -177,7 +192,9 @@ func (vr *VerifiableReader) cacheWithReader(ctx context.Context, currentDepth in
 			return true
 		}
 
-		fr, err := r.OpenFile(id)
+		fr, err := r.OpenFileWithPreReader(id, func(nid uint32, chunkOffset, chunkSize int64, chunkDigest string, r io.Reader) (retErr error) {
+			return vr.readAndCache(nid, r, chunkOffset, chunkSize, chunkDigest, opts...)
+		})
 		if err != nil {
 			rErr = err
 			return false
@@ -196,63 +213,13 @@ func (vr *VerifiableReader) cacheWithReader(ctx context.Context, currentDepth in
 				return false
 			}
 
-			eg.Go(func() (retErr error) {
+			eg.Go(func() error {
 				defer sem.Release(1)
-				defer func() {
-					if retErr != nil {
-						vr.lastVerifyErr.Store(retErr)
-					}
-				}()
-
-				// Check if the target chunks exists in the cache
-				cacheID := genID(id, chunkOffset, chunkSize)
-				if r, err := gr.cache.Get(cacheID, opts...); err == nil {
-					return r.Close()
-				}
-
-				// missed cache, needs to fetch and add it to the cache
-				br := bufio.NewReaderSize(io.NewSectionReader(fr, chunkOffset, chunkSize), int(chunkSize))
-				if _, err := br.Peek(int(chunkSize)); err != nil {
-					return fmt.Errorf("cacheWithReader.peek: %v", err)
-				}
-				w, err := gr.cache.Add(cacheID, opts...)
+				err := vr.readAndCache(id, io.NewSectionReader(fr, chunkOffset, chunkSize), chunkOffset, chunkSize, chunkDigestStr, opts...)
 				if err != nil {
-					return err
+					return fmt.Errorf("failed to read %q (off:%d,size:%d): %w", name, chunkOffset, chunkSize, err)
 				}
-				defer w.Close()
-				v, err := vr.verifier(id, chunkDigestStr)
-				if err != nil {
-					vr.prohibitVerifyFailureMu.RLock()
-					if vr.prohibitVerifyFailure {
-						vr.prohibitVerifyFailureMu.RUnlock()
-						return errors.Wrapf(err, "verifier not found %q(off:%d,size:%d)", name, chunkOffset, chunkSize)
-					}
-					vr.lastVerifyErr.Store(err)
-					vr.prohibitVerifyFailureMu.RUnlock()
-				}
-				tee := ioutil.Discard
-				if v != nil {
-					tee = io.Writer(v) // verification is required
-				}
-				if _, err := io.CopyN(w, io.TeeReader(br, tee), chunkSize); err != nil {
-					w.Abort()
-					return errors.Wrapf(err,
-						"failed to cache file payload of %q (offset:%d,size:%d)",
-						name, chunkOffset, chunkSize)
-				}
-				if v != nil && !v.Verified() {
-					err := fmt.Errorf("invalid chunk %q (offset:%d,size:%d)", name, chunkOffset, chunkSize)
-					vr.prohibitVerifyFailureMu.RLock()
-					if vr.prohibitVerifyFailure {
-						vr.prohibitVerifyFailureMu.RUnlock()
-						w.Abort()
-						return err
-					}
-					vr.lastVerifyErr.Store(err)
-					vr.prohibitVerifyFailureMu.RUnlock()
-				}
-
-				return w.Commit()
+				return nil
 			})
 		}
 
@@ -262,6 +229,63 @@ func (vr *VerifiableReader) cacheWithReader(ctx context.Context, currentDepth in
 	return
 }
 
+func (vr *VerifiableReader) readAndCache(id uint32, fr io.Reader, chunkOffset, chunkSize int64, chunkDigest string, opts ...cache.Option) (retErr error) {
+	gr := vr.r
+
+	if retErr != nil {
+		vr.storeLastVerifyErr(retErr)
+	}
+
+	// Check if it already exists in the cache
+	cacheID := genID(id, chunkOffset, chunkSize)
+	if r, err := gr.cache.Get(cacheID); err == nil {
+		r.Close()
+		return nil
+	}
+
+	// missed cache, needs to fetch and add it to the cache
+	br := bufio.NewReaderSize(fr, int(chunkSize))
+	if _, err := br.Peek(int(chunkSize)); err != nil {
+		return fmt.Errorf("cacheWithReader.peek: %v", err)
+	}
+	w, err := gr.cache.Add(cacheID, opts...)
+	if err != nil {
+		return err
+	}
+	defer w.Close()
+	v, err := vr.verifier(id, chunkDigest)
+	if err != nil {
+		vr.prohibitVerifyFailureMu.RLock()
+		if vr.prohibitVerifyFailure {
+			vr.prohibitVerifyFailureMu.RUnlock()
+			return fmt.Errorf("verifier not found: %w", err)
+		}
+		vr.storeLastVerifyErr(err)
+		vr.prohibitVerifyFailureMu.RUnlock()
+	}
+	tee := io.Discard
+	if v != nil {
+		tee = io.Writer(v) // verification is required
+	}
+	if _, err := io.CopyN(w, io.TeeReader(br, tee), chunkSize); err != nil {
+		w.Abort()
+		return fmt.Errorf("failed to cache file payload: %w", err)
+	}
+	if v != nil && !v.Verified() {
+		err := fmt.Errorf("invalid chunk")
+		vr.prohibitVerifyFailureMu.RLock()
+		if vr.prohibitVerifyFailure {
+			vr.prohibitVerifyFailureMu.RUnlock()
+			w.Abort()
+			return err
+		}
+		vr.storeLastVerifyErr(err)
+		vr.prohibitVerifyFailureMu.RUnlock()
+	}
+
+	return w.Commit()
+}
+
 func (vr *VerifiableReader) Close() error {
 	vr.closedMu.Lock()
 	defer vr.closedMu.Unlock()
@@ -336,9 +360,29 @@ func (gr *reader) OpenFile(id uint32) (io.ReaderAt, error) {
 		return nil, fmt.Errorf("reader is already closed")
 	}
 	var fr metadata.File
-	fr, err := gr.r.OpenFile(id)
+	fr, err := gr.r.OpenFileWithPreReader(id, func(nid uint32, chunkOffset, chunkSize int64, chunkDigest string, r io.Reader) error {
+		// Check if it already exists in the cache
+		cacheID := genID(nid, chunkOffset, chunkSize)
+		if r, err := gr.cache.Get(cacheID); err == nil {
+			r.Close()
+			return nil
+		}
+
+		// Read and cache
+		b := gr.bufPool.Get().(*bytes.Buffer)
+		b.Reset()
+		b.Grow(int(chunkSize))
+		ip := b.Bytes()[:chunkSize]
+		if _, err := io.ReadFull(r, ip); err != nil {
+			gr.putBuffer(b)
+			return err
+		}
+		err := gr.verifyAndCache(nid, ip, chunkDigest, cacheID)
+		gr.putBuffer(b)
+		return err
+	})
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to open file %d", id)
+		return nil, fmt.Errorf("failed to open file %d: %w", id, err)
 	}
 	return &file{
 		id: id,
@@ -347,20 +391,21 @@ func (gr *reader) OpenFile(id uint32) (io.ReaderAt, error) {
 	}, nil
 }
 
-func (gr *reader) Close() (retErr error) {
+func (gr *reader) Close() error {
 	gr.closedMu.Lock()
 	defer gr.closedMu.Unlock()
 	if gr.closed {
 		return nil
 	}
 	gr.closed = true
+	var errs []error
 	if err := gr.cache.Close(); err != nil {
-		retErr = multierror.Append(retErr, err)
+		errs = append(errs, err)
 	}
 	if err := gr.r.Close(); err != nil {
-		retErr = multierror.Append(retErr, err)
+		errs = append(errs, err)
 	}
-	return
+	return errors.Join(errs...)
 }
 
 func (gr *reader) isClosed() bool {
@@ -416,26 +461,10 @@ func (sf *file) ReadAt(p []byte, offset int64) (int, error) {
 			ip := p[nr : int64(nr)+chunkSize]
 			n, err := sf.fr.ReadAt(ip, chunkOffset)
 			if err != nil && err != io.EOF {
-				return 0, errors.Wrap(err, "failed to read data")
+				return 0, fmt.Errorf("failed to read data: %w", err)
 			}
-
-			commonmetrics.IncOperationCount(commonmetrics.OnDemandRemoteRegistryFetchCount, sf.gr.layerSha) // increment the number of on demand file fetches from remote registry
-			commonmetrics.AddBytesCount(commonmetrics.OnDemandBytesFetched, sf.gr.layerSha, int64(n))       // record total bytes fetched
-			sf.gr.setLastReadTime(time.Now())
-
-			// Verify this chunk
-			if err := sf.verify(sf.id, ip, chunkDigestStr); err != nil {
-				return 0, errors.Wrap(err, "invalid chunk")
-			}
-
-			// Cache this chunk
-			if w, err := sf.gr.cache.Add(id); err == nil {
-				if cn, err := w.Write(ip); err != nil || cn != len(ip) {
-					w.Abort()
-				} else {
-					w.Commit()
-				}
-				w.Close()
+			if err := sf.gr.verifyAndCache(sf.id, ip, chunkDigestStr, id); err != nil {
+				return 0, err
 			}
 			nr += n
 			continue
@@ -448,28 +477,11 @@ func (sf *file) ReadAt(p []byte, offset int64) (int, error) {
 		ip := b.Bytes()[:chunkSize]
 		if _, err := sf.fr.ReadAt(ip, chunkOffset); err != nil && err != io.EOF {
 			sf.gr.putBuffer(b)
-			return 0, errors.Wrap(err, "failed to read data")
+			return 0, fmt.Errorf("failed to read data: %w", err)
 		}
-
-		// We can end up doing on demand registry fetch when aligning the chunk
-		commonmetrics.IncOperationCount(commonmetrics.OnDemandRemoteRegistryFetchCount, sf.gr.layerSha) // increment the number of on demand file fetches from remote registry
-		commonmetrics.AddBytesCount(commonmetrics.OnDemandBytesFetched, sf.gr.layerSha, int64(len(ip))) // record total bytes fetched
-		sf.gr.setLastReadTime(time.Now())
-
-		// Verify this chunk
-		if err := sf.verify(sf.id, ip, chunkDigestStr); err != nil {
+		if err := sf.gr.verifyAndCache(sf.id, ip, chunkDigestStr, id); err != nil {
 			sf.gr.putBuffer(b)
-			return 0, errors.Wrap(err, "invalid chunk")
-		}
-
-		// Cache this chunk
-		if w, err := sf.gr.cache.Add(id); err == nil {
-			if cn, err := w.Write(ip); err != nil || cn != len(ip) {
-				w.Abort()
-			} else {
-				w.Commit()
-			}
-			w.Close()
+			return 0, err
 		}
 		n := copy(p[nr:], ip[lowerDiscard:chunkSize-upperDiscard])
 		sf.gr.putBuffer(b)
@@ -484,16 +496,344 @@ func (sf *file) ReadAt(p []byte, offset int64) (int, error) {
 	return nr, nil
 }
 
-func (sf *file) verify(id uint32, p []byte, chunkDigestStr string) error {
-	if !sf.gr.verify {
+type chunkData struct {
+	offset    int64
+	size      int64
+	digestStr string
+	bufferPos int64
+}
+
+func (sf *file) GetPassthroughFd(mergeBufferSize int64, mergeWorkerCount int) (uintptr, error) {
+	var (
+		offset           int64
+		firstChunkOffset int64
+		totalSize        int64
+		hasLargeChunk    bool
+	)
+
+	var chunks []chunkData
+	for {
+		chunkOffset, chunkSize, digestStr, ok := sf.fr.ChunkEntryForOffset(offset)
+		if !ok {
+			break
+		}
+		// Check if any chunk size exceeds merge buffer size to avoid bounds out of range
+		if chunkSize > mergeBufferSize {
+			hasLargeChunk = true
+		}
+		chunks = append(chunks, chunkData{
+			offset:    chunkOffset,
+			size:      chunkSize,
+			digestStr: digestStr,
+		})
+		totalSize += chunkSize
+		offset = chunkOffset + chunkSize
+	}
+
+	id := genID(sf.id, firstChunkOffset, totalSize)
+
+	// cache.PassThrough() is necessary to take over files
+	r, err := sf.gr.cache.Get(id, cache.PassThrough())
+	if err != nil {
+		if hasLargeChunk {
+			if err := sf.prefetchEntireFileSequential(id); err != nil {
+				return 0, err
+			}
+		} else {
+			if err := sf.prefetchEntireFile(id, chunks, totalSize, mergeBufferSize, mergeWorkerCount); err != nil {
+				return 0, err
+			}
+		}
+
+		// just retry once to avoid exception stuck
+		r, err = sf.gr.cache.Get(id, cache.PassThrough())
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	readerAt := r.GetReaderAt()
+	file, ok := readerAt.(*os.File)
+	if !ok {
+		r.Close()
+		return 0, fmt.Errorf("the cached ReaderAt is not of type *os.File, fd obtain failed")
+	}
+
+	fd := file.Fd()
+	r.Close()
+	return fd, nil
+}
+
+// prefetchEntireFileSequential uses the legacy sequential approach for processing chunks
+// when chunk size exceeds merge buffer size to avoid slice bounds out of range panic
+func (sf *file) prefetchEntireFileSequential(entireCacheID string) error {
+	w, err := sf.gr.cache.Add(entireCacheID)
+	if err != nil {
+		return fmt.Errorf("failed to create cache writer: %w", err)
+	}
+	defer w.Close()
+
+	var offset int64
+
+	for {
+		chunkOffset, chunkSize, chunkDigestStr, ok := sf.fr.ChunkEntryForOffset(offset)
+		if !ok {
+			break
+		}
+
+		id := genID(sf.id, chunkOffset, chunkSize)
+		b := sf.gr.bufPool.Get().(*bytes.Buffer)
+		b.Reset()
+		b.Grow(int(chunkSize))
+		ip := b.Bytes()[:chunkSize]
+
+		if r, err := sf.gr.cache.Get(id); err == nil {
+			n, err := r.ReadAt(ip, 0)
+			if (err == nil || err == io.EOF) && int64(n) == chunkSize {
+				if _, err := w.Write(ip[:n]); err != nil {
+					r.Close()
+					sf.gr.putBuffer(b)
+					w.Abort()
+					return fmt.Errorf("failed to write cached data: %w", err)
+				}
+				offset = chunkOffset + int64(n)
+				r.Close()
+				sf.gr.putBuffer(b)
+				continue
+			}
+			r.Close()
+		}
+
+		if _, err := sf.fr.ReadAt(ip, chunkOffset); err != nil && err != io.EOF {
+			sf.gr.putBuffer(b)
+			w.Abort()
+			return fmt.Errorf("failed to read data: %w", err)
+		}
+		if err := sf.gr.verifyOneChunk(sf.id, ip, chunkDigestStr); err != nil {
+			sf.gr.putBuffer(b)
+			w.Abort()
+			return err
+		}
+		if _, err := w.Write(ip); err != nil {
+			sf.gr.putBuffer(b)
+			w.Abort()
+			return fmt.Errorf("failed to write fetched data: %w", err)
+		}
+		offset = chunkOffset + chunkSize
+		sf.gr.putBuffer(b)
+	}
+
+	return w.Commit()
+}
+
+type batchWorkerArgs struct {
+	workerID    int
+	chunks      []chunkData
+	buffer      []byte
+	workerCount int
+	readInfos   []chunkReadInfo
+}
+
+func (sf *file) prefetchEntireFile(entireCacheID string, chunks []chunkData, totalSize int64, bufferSize int64, workerCount int) error {
+
+	w, err := sf.gr.cache.Add(entireCacheID)
+	if err != nil {
+		return fmt.Errorf("failed to create cache writer: %w", err)
+	}
+	defer w.Close()
+
+	batchCount := (totalSize + bufferSize - 1) / bufferSize
+
+	for batchIdx := int64(0); batchIdx < batchCount; batchIdx++ {
+		batchStart := batchIdx * bufferSize
+		batchEnd := (batchIdx + 1) * bufferSize
+		if batchEnd > totalSize {
+			batchEnd = totalSize
+		}
+
+		var batchChunks []chunkData
+		var batchOffset int64
+		for i := range chunks {
+			chunkStart := chunks[i].offset
+			chunkEnd := chunkStart + chunks[i].size
+
+			if chunkEnd <= batchStart {
+				continue
+			}
+			if chunkStart >= batchEnd {
+				break
+			}
+
+			chunks[i].bufferPos = batchOffset
+			batchOffset += chunks[i].size
+			batchChunks = append(batchChunks, chunks[i])
+		}
+
+		batchSize := batchEnd - batchStart
+		buffer := make([]byte, batchSize)
+
+		eg := errgroup.Group{}
+		allReadInfos := make([][]chunkReadInfo, workerCount)
+
+		for i := 0; i < workerCount && i < len(batchChunks); i++ {
+			workerID := i
+			args := &batchWorkerArgs{
+				workerID:    workerID,
+				chunks:      batchChunks,
+				buffer:      buffer,
+				workerCount: workerCount,
+			}
+			eg.Go(func() error {
+				err := sf.processBatchChunks(args)
+				if err == nil && len(args.readInfos) > 0 {
+					allReadInfos[args.workerID] = args.readInfos
+				}
+				return err
+			})
+		}
+
+		if err := eg.Wait(); err != nil {
+			w.Abort()
+			return err
+		}
+
+		var mergedReadInfos []chunkReadInfo
+		for _, infos := range allReadInfos {
+			mergedReadInfos = append(mergedReadInfos, infos...)
+		}
+
+		if err := sf.checkHoles(mergedReadInfos, batchSize); err != nil {
+			w.Abort()
+			return fmt.Errorf("hole check failed: %w", err)
+		}
+
+		n, err := w.Write(buffer)
+		if err != nil {
+			w.Abort()
+			return fmt.Errorf("failed to write batch data: %w", err)
+		}
+
+		if int64(n) != batchSize {
+			w.Abort()
+			return fmt.Errorf("incomplete write: expected %d bytes, wrote %d bytes", batchSize, n)
+		}
+	}
+
+	return w.Commit()
+}
+
+type chunkReadInfo struct {
+	offset int64
+	size   int64
+}
+
+func (sf *file) checkHoles(readInfos []chunkReadInfo, totalSize int64) error {
+	if len(readInfos) == 0 {
+		return nil
+	}
+
+	sort.Slice(readInfos, func(i, j int) bool {
+		return readInfos[i].offset < readInfos[j].offset
+	})
+
+	end := readInfos[0].offset
+	for _, info := range readInfos {
+		if info.offset < end {
+			return fmt.Errorf("overlapping read detected: previous end %d, current start %d", end, info.offset)
+		} else if info.offset > end {
+			return fmt.Errorf("hole detected in read: previous end %d, current start %d", end, info.offset)
+		}
+		end = info.offset + info.size
+	}
+
+	if end != totalSize {
+		return fmt.Errorf("incomplete read: expected total size %d, actual end %d", totalSize, end)
+	}
+
+	return nil
+}
+
+func (sf *file) processBatchChunks(args *batchWorkerArgs) error {
+	var readInfos []chunkReadInfo
+
+	for chunkIdx := args.workerID; chunkIdx < len(args.chunks); chunkIdx += args.workerCount {
+		chunk := args.chunks[chunkIdx]
+		bufStart := args.buffer[chunk.bufferPos : chunk.bufferPos+chunk.size]
+
+		id := genID(sf.id, chunk.offset, chunk.size)
+		if r, err := sf.gr.cache.Get(id); err == nil {
+			n, err := r.ReadAt(bufStart, 0)
+			r.Close()
+			if err == nil || err == io.EOF {
+				if int64(n) == chunk.size {
+					readInfos = append(readInfos, chunkReadInfo{
+						offset: chunk.bufferPos,
+						size:   int64(n),
+					})
+					continue
+				}
+			}
+		}
+
+		n, err := sf.fr.ReadAt(bufStart, chunk.offset)
+		if err != nil && err != io.EOF {
+			return fmt.Errorf("failed to read data at offset %d: %w", chunk.offset, err)
+		}
+
+		readInfos = append(readInfos, chunkReadInfo{
+			offset: chunk.bufferPos,
+			size:   int64(n),
+		})
+
+		if err := sf.gr.verifyOneChunk(sf.id, bufStart, chunk.digestStr); err != nil {
+			return fmt.Errorf("chunk verification failed at offset %d: %w", chunk.offset, err)
+		}
+	}
+
+	args.readInfos = readInfos
+	return nil
+}
+
+func (gr *reader) verifyOneChunk(entryID uint32, ip []byte, chunkDigestStr string) error {
+	// We can end up doing on demand registry fetch when aligning the chunk
+	commonmetrics.IncOperationCount(commonmetrics.OnDemandRemoteRegistryFetchCount, gr.layerSha)
+	commonmetrics.AddBytesCount(commonmetrics.OnDemandBytesFetched, gr.layerSha, int64(len(ip)))
+	gr.setLastReadTime(time.Now())
+	if err := gr.verifyChunk(entryID, ip, chunkDigestStr); err != nil {
+		return fmt.Errorf("invalid chunk: %w", err)
+	}
+	return nil
+}
+
+func (gr *reader) cacheData(ip []byte, cacheID string) {
+	if w, err := gr.cache.Add(cacheID); err == nil {
+		if cn, err := w.Write(ip); err != nil || cn != len(ip) {
+			w.Abort()
+		} else {
+			w.Commit()
+		}
+		w.Close()
+	}
+}
+
+func (gr *reader) verifyAndCache(entryID uint32, ip []byte, chunkDigestStr string, cacheID string) error {
+	if err := gr.verifyOneChunk(entryID, ip, chunkDigestStr); err != nil {
+		return err
+	}
+	gr.cacheData(ip, cacheID)
+	return nil
+}
+
+func (gr *reader) verifyChunk(id uint32, p []byte, chunkDigestStr string) error {
+	if !gr.verify {
 		return nil // verification is not required
 	}
-	v, err := sf.gr.verifier(id, chunkDigestStr)
+	v, err := gr.verifier(id, chunkDigestStr)
 	if err != nil {
-		return errors.Wrapf(err, "invalid chunk")
+		return fmt.Errorf("invalid chunk: %w", err)
 	}
 	if _, err := v.Write(p); err != nil {
-		return errors.Wrap(err, "invalid chunk: failed to write to verifier")
+		return fmt.Errorf("invalid chunk: failed to write to verifier: %w", err)
 	}
 	if !v.Verified() {
 		return fmt.Errorf("invalid chunk: not verified")
@@ -543,7 +883,7 @@ func WithReader(sr *io.SectionReader) CacheOption {
 func digestVerifier(id uint32, chunkDigestStr string) (digest.Verifier, error) {
 	chunkDigest, err := digest.Parse(chunkDigestStr)
 	if err != nil {
-		return nil, errors.Wrap(err, "invalid chunk: no digset is recorded")
+		return nil, fmt.Errorf("invalid chunk: no digest is recorded(len=%d): %w", len(chunkDigestStr), err)
 	}
 	return chunkDigest.Verifier(), nil
 }

fs/reader/reader_test.go

@@ -23,587 +23,26 @@
 package reader
 
 import (
-	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"strings"
-	"sync"
 	"testing"
-	"time"
 
-	"github.com/containerd/stargz-snapshotter/cache"
-	"github.com/containerd/stargz-snapshotter/estargz"
-	"github.com/containerd/stargz-snapshotter/metadata"
-	dbmetadata "github.com/containerd/stargz-snapshotter/metadata/db"
 	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
-	"github.com/containerd/stargz-snapshotter/util/testutil"
-	"github.com/hashicorp/go-multierror"
-	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
-	bolt "go.etcd.io/bbolt"
-	"golang.org/x/sync/errgroup"
 )
 
-const (
-	sampleChunkSize    = 3
-	sampleMiddleOffset = sampleChunkSize / 2
-	sampleData1        = "0123456789"
-	lastChunkOffset1   = sampleChunkSize * (int64(len(sampleData1)) / sampleChunkSize)
-)
-
-// Tests Reader for failure cases.
-func TestFailReader(t *testing.T) {
-	testFileName := "test"
-	stargzFile, tocDigest, err := testutil.BuildEStargz([]testutil.TarEntry{
-		testutil.File(testFileName, sampleData1),
-	}, testutil.WithEStargzOptions(estargz.WithChunkSize(sampleChunkSize)))
-	if err != nil {
-		t.Fatalf("failed to build sample estargz")
-	}
-
-	for _, rs := range []bool{true, false} {
-		for _, vs := range []bool{true, false} {
-			br := &breakReaderAt{
-				ReaderAt: stargzFile,
-				success:  true,
-			}
-			bev := &testChunkVerifier{true}
-			mcache := cache.NewMemoryCache()
-			vr, closeFn, err := newMemoryReader(io.NewSectionReader(br, 0, stargzFile.Size()), mcache)
-			if err != nil {
-				t.Fatalf("failed to create reader: %v", err)
-			}
-			defer closeFn()
-			defer vr.Close()
-			vr.verifier = bev.verifier
-			vr.r.verifier = bev.verifier
-			gr, err := vr.VerifyTOC(tocDigest)
-			if err != nil {
-				t.Fatalf("failed to verify TOC: %v", err)
-			}
-
-			notexist := uint32(0)
-			found := false
-			for i := uint32(0); i < 1000000; i++ {
-				if _, err := gr.Metadata().GetAttr(i); err != nil {
-					notexist, found = i, true
-					break
-				}
-			}
-			if !found {
-				t.Fatalf("free ID not found")
-			}
-
-			// tests for opening non-existing file
-			_, err = gr.OpenFile(notexist)
-			if err == nil {
-				t.Errorf("succeeded to open file but wanted to fail")
+func TestReader(t *testing.T) {
+	testRunner := &TestRunner{
+		TestingT: t,
+		Runner: func(testingT TestingT, name string, run func(t TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
 				return
 			}
 
-			// tests failure behaviour of a file read
-			tid, _, err := gr.Metadata().GetChild(gr.Metadata().RootID(), testFileName)
-			if err != nil {
-				t.Errorf("failed to get %q: %v", testFileName, err)
-				return
-			}
-			fr, err := gr.OpenFile(tid)
-			if err != nil {
-				t.Errorf("failed to open file but wanted to succeed: %v", err)
-				return
-			}
-
-			mcache.(*cache.MemoryCache).Membuf = map[string]*bytes.Buffer{}
-			br.success = rs
-			bev.success = vs
-
-			// tests for reading file
-			p := make([]byte, len(sampleData1))
-			n, err := fr.ReadAt(p, 0)
-			if rs && vs {
-				if err != nil || n != len(sampleData1) || !bytes.Equal([]byte(sampleData1), p) {
-					t.Errorf("failed to read data but wanted to succeed: %v", err)
-					return
-				}
-			} else {
-				if err == nil {
-					t.Errorf("succeeded to read data but wanted to fail (reader:%v,verify:%v)", rs, vs)
-					return
-				}
-			}
-		}
-	}
-}
-
-type breakReaderAt struct {
-	io.ReaderAt
-	success bool
-}
-
-func (br *breakReaderAt) ReadAt(p []byte, off int64) (int, error) {
-	if br.success {
-		return br.ReaderAt.ReadAt(p, off)
-	}
-	return 0, fmt.Errorf("failed")
-}
-
-type testChunkVerifier struct {
-	success bool
-}
-
-func (bev *testChunkVerifier) verifier(id uint32, chunkDigest string) (digest.Verifier, error) {
-	return &testVerifier{bev.success}, nil
-}
-
-type testVerifier struct {
-	success bool
-}
-
-func (bv *testVerifier) Write(p []byte) (n int, err error) {
-	return len(p), nil
-}
-
-func (bv *testVerifier) Verified() bool {
-	return bv.success
-}
-
-type region struct{ b, e int64 }
-
-// Tests ReadAt method of each file.
-func TestFileReadAt(t *testing.T) {
-	testFileReadAt(t, "db", newDBReader)
-	testFileReadAt(t, "mem", newMemoryReader)
-}
-
-func testFileReadAt(t *testing.T, name string, factory readerFactory) {
-	sizeCond := map[string]int64{
-		"single_chunk": sampleChunkSize - sampleMiddleOffset,
-		"multi_chunks": sampleChunkSize + sampleMiddleOffset,
-	}
-	innerOffsetCond := map[string]int64{
-		"at_top":    0,
-		"at_middle": sampleMiddleOffset,
-	}
-	baseOffsetCond := map[string]int64{
-		"of_1st_chunk":  sampleChunkSize * 0,
-		"of_2nd_chunk":  sampleChunkSize * 1,
-		"of_last_chunk": lastChunkOffset1,
-	}
-	fileSizeCond := map[string]int64{
-		"in_1_chunk_file":  sampleChunkSize * 1,
-		"in_2_chunks_file": sampleChunkSize * 2,
-		"in_max_size_file": int64(len(sampleData1)),
-	}
-	cacheCond := map[string][]region{
-		"with_clean_cache": nil,
-		"with_edge_filled_cache": {
-			region{0, sampleChunkSize - 1},
-			region{lastChunkOffset1, int64(len(sampleData1)) - 1},
-		},
-		"with_sparse_cache": {
-			region{0, sampleChunkSize - 1},
-			region{2 * sampleChunkSize, 3*sampleChunkSize - 1},
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
 		},
 	}
-	for sn, size := range sizeCond {
-		for in, innero := range innerOffsetCond {
-			for bo, baseo := range baseOffsetCond {
-				for fn, filesize := range fileSizeCond {
-					for cc, cacheExcept := range cacheCond {
-						t.Run(fmt.Sprintf("reading_%s_%s_%s_%s_%s_%s", name, sn, in, bo, fn, cc), func(t *testing.T) {
-							if filesize > int64(len(sampleData1)) {
-								t.Fatal("sample file size is larger than sample data")
-							}
 
-							wantN := size
-							offset := baseo + innero
-							if remain := filesize - offset; remain < wantN {
-								if wantN = remain; wantN < 0 {
-									wantN = 0
-								}
-							}
-
-							// use constant string value as a data source.
-							want := strings.NewReader(sampleData1)
-
-							// data we want to get.
-							wantData := make([]byte, wantN)
-							_, err := want.ReadAt(wantData, offset)
-							if err != nil && err != io.EOF {
-								t.Fatalf("want.ReadAt (offset=%d,size=%d): %v", offset, wantN, err)
-							}
-
-							// data we get through a file.
-							f, closeFn := makeFile(t, []byte(sampleData1)[:filesize], sampleChunkSize, factory)
-							defer closeFn()
-							f.fr = newExceptFile(t, f.fr, cacheExcept...)
-							for _, reg := range cacheExcept {
-								id := genID(f.id, reg.b, reg.e-reg.b+1)
-								w, err := f.gr.cache.Add(id)
-								if err != nil {
-									w.Close()
-									t.Fatalf("failed to add cache %v: %v", id, err)
-								}
-								if _, err := w.Write([]byte(sampleData1[reg.b : reg.e+1])); err != nil {
-									w.Close()
-									t.Fatalf("failed to write cache %v: %v", id, err)
-								}
-								if err := w.Commit(); err != nil {
-									w.Close()
-									t.Fatalf("failed to commit cache %v: %v", id, err)
-								}
-								w.Close()
-							}
-							respData := make([]byte, size)
-							n, err := f.ReadAt(respData, offset)
-							if err != nil {
-								t.Errorf("failed to read off=%d, size=%d, filesize=%d: %v", offset, size, filesize, err)
-								return
-							}
-							respData = respData[:n]
-
-							if !bytes.Equal(wantData, respData) {
-								t.Errorf("off=%d, filesize=%d; read data{size=%d,data=%q}; want (size=%d,data=%q)",
-									offset, filesize, len(respData), string(respData), wantN, string(wantData))
-								return
-							}
-
-							// check cache has valid contents.
-							cn := 0
-							nr := 0
-							for int64(nr) < wantN {
-								chunkOffset, chunkSize, _, ok := f.fr.ChunkEntryForOffset(offset + int64(nr))
-								if !ok {
-									break
-								}
-								data := make([]byte, chunkSize)
-								id := genID(f.id, chunkOffset, chunkSize)
-								r, err := f.gr.cache.Get(id)
-								if err != nil {
-									t.Errorf("missed cache of offset=%d, size=%d: %v(got size=%d)", chunkOffset, chunkSize, err, n)
-									return
-								}
-								defer r.Close()
-								if n, err := r.ReadAt(data, 0); (err != nil && err != io.EOF) || n != int(chunkSize) {
-									t.Errorf("failed to read cache of offset=%d, size=%d: %v(got size=%d)", chunkOffset, chunkSize, err, n)
-									return
-								}
-								nr += n
-								cn++
-							}
-						})
-					}
-				}
-			}
-		}
-	}
-}
-
-func newExceptFile(t *testing.T, fr metadata.File, except ...region) metadata.File {
-	er := exceptFile{fr: fr, t: t}
-	er.except = map[region]bool{}
-	for _, reg := range except {
-		er.except[reg] = true
-	}
-	return &er
-}
-
-type exceptFile struct {
-	fr     metadata.File
-	except map[region]bool
-	t      *testing.T
-}
-
-func (er *exceptFile) ReadAt(p []byte, offset int64) (int, error) {
-	if er.except[region{offset, offset + int64(len(p)) - 1}] {
-		er.t.Fatalf("Requested prohibited region of chunk: (%d, %d)", offset, offset+int64(len(p))-1)
-	}
-	return er.fr.ReadAt(p, offset)
-}
-
-func (er *exceptFile) ChunkEntryForOffset(offset int64) (off int64, size int64, dgst string, ok bool) {
-	return er.fr.ChunkEntryForOffset(offset)
-}
-
-func makeFile(t *testing.T, contents []byte, chunkSize int, factory readerFactory) (*file, func() error) {
-	testName := "test"
-	sr, dgst, err := testutil.BuildEStargz([]testutil.TarEntry{
-		testutil.File(testName, string(contents)),
-	}, testutil.WithEStargzOptions(estargz.WithChunkSize(chunkSize)))
-	if err != nil {
-		t.Fatalf("failed to build sample estargz")
-	}
-	vr, closeFn, err := factory(sr, cache.NewMemoryCache())
-	if err != nil {
-		t.Fatalf("failed to create reader: %v", err)
-	}
-	var closes closeFuncs
-	closes = append(closes, vr.Close, closeFn)
-	r, err := vr.VerifyTOC(dgst)
-	if err != nil {
-		closes.close()
-		t.Fatalf("failed to verify TOC: %v", err)
-	}
-	tid, _, err := r.Metadata().GetChild(r.Metadata().RootID(), testName)
-	if err != nil {
-		closes.close()
-		t.Fatalf("failed to get %q: %v", testName, err)
-	}
-	ra, err := r.OpenFile(tid)
-	if err != nil {
-		closes.close()
-		t.Fatalf("Failed to open testing file: %v", err)
-	}
-	f, ok := ra.(*file)
-	if !ok {
-		closes.close()
-		t.Fatalf("invalid type of file %q", tid)
-	}
-	return f, closes.close
-}
-
-func TestCacheVerify(t *testing.T) {
-	sr, tocDgst, err := testutil.BuildEStargz([]testutil.TarEntry{
-		testutil.File("a", sampleData1+"a"),
-		testutil.File("b", sampleData1+"b"),
-	}, testutil.WithEStargzOptions(estargz.WithChunkSize(sampleChunkSize)))
-	if err != nil {
-		t.Fatalf("failed to build sample estargz")
-	}
-	for readerName, factory := range map[string]readerFactory{"mem": newMemoryReader, "db": newDBReader} {
-		readerName, factory := readerName, factory
-		for _, skipVerify := range [2]bool{true, false} {
-			for _, invalidChunkBeforeVerify := range [2]bool{true, false} {
-				for _, invalidChunkAfterVerify := range [2]bool{true, false} {
-					name := fmt.Sprintf("test_cache_verify_%v_%v_%v_%v",
-						readerName, skipVerify, invalidChunkBeforeVerify, invalidChunkAfterVerify)
-					t.Run(name, func(t *testing.T) {
-
-						// Determine the expected behaviour
-						var wantVerifyFail, wantCacheFail, wantCacheFail2 bool
-						if skipVerify {
-							// always no error if verification is disabled
-							wantVerifyFail, wantCacheFail, wantCacheFail2 = false, false, false
-						} else if invalidChunkBeforeVerify {
-							// errors occurred before verifying TOC must be reported via VerifyTOC()
-							wantVerifyFail = true
-						} else if invalidChunkAfterVerify {
-							// errors occurred after verifying TOC must be reported via Cache()
-							wantVerifyFail, wantCacheFail, wantCacheFail2 = false, true, true
-						} else {
-							// otherwise no verification error
-							wantVerifyFail, wantCacheFail, wantCacheFail2 = false, false, false
-						}
-
-						// Prepare reader
-						verifier := &failIDVerifier{}
-						vr, closeFn, err := factory(sr, cache.NewMemoryCache())
-						if err != nil {
-							t.Fatalf("failed to prepare reader %v", err)
-						}
-						defer closeFn()
-						defer vr.Close()
-						if verifier != nil {
-							vr.verifier = verifier.verifier
-							vr.r.verifier = verifier.verifier
-						}
-
-						off2id, id2path, err := prepareMap(vr.Metadata(), vr.Metadata().RootID(), "")
-						if err != nil || off2id == nil || id2path == nil {
-							t.Fatalf("failed to prepare offset map %v, off2id = %+v, id2path = %+v", err, off2id, id2path)
-						}
-
-						// Perform Cache() before verification
-						// 1. Either of "a" or "b" is read and verified
-						// 2. VerifyTOC/SkipVerify is called
-						// 3. Another entry ("a" or "b") is called
-						verifyDone := make(chan struct{})
-						var firstEntryCalled bool
-						var eg errgroup.Group
-						eg.Go(func() error {
-							return vr.Cache(WithFilter(func(off int64) bool {
-								id, ok := off2id[off]
-								if !ok {
-									t.Fatalf("no ID is assigned to offset %d", off)
-								}
-								name, ok := id2path[id]
-								if !ok {
-									t.Fatalf("no name is assigned to id %d", id)
-								}
-								if name == "a" || name == "b" {
-									if !firstEntryCalled {
-										firstEntryCalled = true
-										if invalidChunkBeforeVerify {
-											verifier.registerFails([]uint32{id})
-										}
-										return true
-									}
-									<-verifyDone
-									if invalidChunkAfterVerify {
-										verifier.registerFails([]uint32{id})
-									}
-									return true
-								}
-								return false
-							}))
-						})
-						time.Sleep(10 * time.Millisecond)
-
-						// Perform verification
-						if skipVerify {
-							vr.SkipVerify()
-						} else {
-							_, err = vr.VerifyTOC(tocDgst)
-						}
-						if checkErr := checkError(wantVerifyFail, err); checkErr != nil {
-							t.Errorf("verify: %v", checkErr)
-							return
-						}
-						if err != nil {
-							return
-						}
-						close(verifyDone)
-
-						// Check the result of Cache()
-						if checkErr := checkError(wantCacheFail, eg.Wait()); checkErr != nil {
-							t.Errorf("cache: %v", checkErr)
-							return
-						}
-
-						// Call Cache() again and check the result
-						if checkErr := checkError(wantCacheFail2, vr.Cache()); checkErr != nil {
-							t.Errorf("cache(2): %v", checkErr)
-							return
-						}
-					})
-				}
-			}
-		}
-	}
-}
-
-type failIDVerifier struct {
-	fails   []uint32
-	failsMu sync.Mutex
-}
-
-func (f *failIDVerifier) registerFails(fails []uint32) {
-	f.failsMu.Lock()
-	defer f.failsMu.Unlock()
-	f.fails = fails
-
-}
-
-func (f *failIDVerifier) verifier(id uint32, chunkDigest string) (digest.Verifier, error) {
-	f.failsMu.Lock()
-	defer f.failsMu.Unlock()
-	success := true
-	for _, n := range f.fails {
-		if n == id {
-			success = false
-			break
-		}
-	}
-	return &testVerifier{success}, nil
-}
-
-func checkError(wantFail bool, err error) error {
-	if wantFail && err == nil {
-		return fmt.Errorf("wanted to fail but succeeded")
-	} else if !wantFail && err != nil {
-		return errors.Wrapf(err, "wanted to succeed verification but failed")
-	}
-	return nil
-}
-
-func prepareMap(mr metadata.Reader, id uint32, p string) (off2id map[int64]uint32, id2path map[uint32]string, _ error) {
-	attr, err := mr.GetAttr(id)
-	if err != nil {
-		return nil, nil, err
-	}
-	id2path = map[uint32]string{id: p}
-	off2id = make(map[int64]uint32)
-	if attr.Mode.IsRegular() {
-		off, err := mr.GetOffset(id)
-		if err != nil {
-			return nil, nil, err
-		}
-		off2id[off] = id
-	}
-	var retErr error
-	mr.ForeachChild(id, func(name string, id uint32, mode os.FileMode) bool {
-		o2i, i2p, err := prepareMap(mr, id, path.Join(p, name))
-		if err != nil {
-			retErr = err
-			return false
-		}
-		for k, v := range o2i {
-			off2id[k] = v
-		}
-		for k, v := range i2p {
-			id2path[k] = v
-		}
-		return true
-	})
-	if retErr != nil {
-		return nil, nil, retErr
-	}
-	return off2id, id2path, nil
-}
-
-type readerFactory func(sr *io.SectionReader, cache cache.BlobCache) (r *VerifiableReader, close func() error, err error)
-
-func newDBReader(sr *io.SectionReader, cache cache.BlobCache) (*VerifiableReader, func() error, error) {
-	var closes closeFuncs
-	f, err := ioutil.TempFile("", "readertest")
-	if err != nil {
-		return nil, nil, err
-	}
-	closes = append(closes, func() error { return os.Remove(f.Name()) })
-	db, err := bolt.Open(f.Name(), 0666, nil)
-	if err != nil {
-		closes.close()
-		return nil, nil, err
-	}
-	closes = append(closes, db.Close)
-	mr, err := dbmetadata.NewReader(db, sr)
-	if err != nil {
-		closes.close()
-		return nil, nil, err
-	}
-	closes = append(closes, mr.Close)
-	r, err := NewReader(mr, cache, digest.FromString(""))
-	if err != nil {
-		closes.close()
-		return nil, nil, err
-	}
-	return r, closes.close, nil
-}
-
-func newMemoryReader(sr *io.SectionReader, cache cache.BlobCache) (*VerifiableReader, func() error, error) {
-	mr, err := memorymetadata.NewReader(sr)
-	if err != nil {
-		return nil, nil, err
-	}
-	r, err := NewReader(mr, cache, digest.FromString(""))
-	if err != nil {
-		mr.Close()
-		return nil, nil, err
-	}
-	return r, mr.Close, nil
-}
-
-type closeFuncs []func() error
-
-func (fs closeFuncs) close() error {
-	var allErr error
-	for _, f := range fs {
-		if err := f(); err != nil {
-			allErr = multierror.Append(allErr, err)
-		}
-	}
-	return allErr
+	TestSuiteReader(testRunner, memorymetadata.NewReader)
 }

fs/remote/blob.go

@@ -26,18 +26,16 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"regexp"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/v2/pkg/reference"
 	"github.com/containerd/stargz-snapshotter/cache"
 	"github.com/containerd/stargz-snapshotter/fs/source"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/singleflight"
 )
@@ -122,7 +120,7 @@ func (b *blob) Refresh(ctx context.Context, hosts source.RegistryHosts, refspec
 		return err
 	}
 	if newSize != b.size {
-		return fmt.Errorf("Invalid size of new blob %d; want %d", newSize, b.size)
+		return fmt.Errorf("invalid size of new blob %d; want %d", newSize, b.size)
 	}
 
 	// update the blob's fetcher with new one
@@ -194,7 +192,7 @@ func (b *blob) cacheAt(offset int64, size int64, fr fetcher, cacheOpts *options)
 		if r, err := b.cache.Get(fr.genID(reg), cacheOpts.cacheOpts...); err == nil {
 			return r.Close() // nop if the cache hits
 		}
-		discard[reg] = ioutil.Discard
+		discard[reg] = io.Discard
 		return nil
 	})
 	if err != nil {
@@ -261,13 +259,23 @@ func (b *blob) ReadAt(p []byte, offset int64, opts ...Option) (int, error) {
 		o(&readAtOpts)
 	}
 
-	// Fetcher can be suddenly updated so we take and use the snapshot of it for
-	// consistency.
-	b.fetcherMu.Lock()
-	fr := b.fetcher
-	b.fetcherMu.Unlock()
+	fr := b.getFetcher()
 
-	b.walkChunks(allRegion, func(chunk region) error {
+	if err := b.prepareChunksForRead(allRegion, offset, p, fr, allData, &readAtOpts); err != nil {
+		return 0, err
+	}
+
+	// Read required data
+	if err := b.fetchRange(allData, &readAtOpts); err != nil {
+		return 0, err
+	}
+
+	return b.adjustBufferSize(p, offset), nil
+}
+
+// prepareChunksForRead prepares chunks for reading by checking cache and setting up writers
+func (b *blob) prepareChunksForRead(allRegion region, offset int64, p []byte, fr fetcher, allData map[region]io.Writer, opts *options) error {
+	return b.walkChunks(allRegion, func(chunk region) error {
 		var (
 			base         = positive(chunk.b - offset)
 			lowerUnread  = positive(offset - chunk.b)
@@ -275,14 +283,9 @@ func (b *blob) ReadAt(p []byte, offset int64, opts ...Option) (int, error) {
 			expectedSize = chunk.size() - upperUnread - lowerUnread
 		)
 
-		// Check if the content exists in the cache
-		r, err := b.cache.Get(fr.genID(chunk), readAtOpts.cacheOpts...)
-		if err == nil {
-			defer r.Close()
-			n, err := r.ReadAt(p[base:base+expectedSize], lowerUnread)
-			if (err == nil || err == io.EOF) && int64(n) == expectedSize {
-				return nil
-			}
+		// Try to read from cache first
+		if err := b.readFromCache(chunk, p[base:base+expectedSize], lowerUnread, fr, opts); err == nil {
+			return nil
 		}
 
 		// We missed cache. Take it from remote registry.
@@ -291,21 +294,23 @@ func (b *blob) ReadAt(p []byte, offset int64, opts ...Option) (int, error) {
 		allData[chunk] = newBytesWriter(p[base:base+expectedSize], lowerUnread)
 		return nil
 	})
+}
 
-	// Read required data
-	if err := b.fetchRange(allData, &readAtOpts); err != nil {
-		return 0, err
+// readFromCache attempts to read chunk data from cache
+func (b *blob) readFromCache(chunk region, dest []byte, offset int64, fr fetcher, opts *options) error {
+	r, err := b.cache.Get(fr.genID(chunk), opts.cacheOpts...)
+	if err != nil {
+		return err
 	}
-
-	// Adjust the buffer size according to the blob size
-	if remain := b.size - offset; int64(len(p)) >= remain {
-		if remain < 0 {
-			remain = 0
-		}
-		p = p[:remain]
+	defer r.Close()
+	n, err := r.ReadAt(dest, offset)
+	if err != nil && err != io.EOF {
+		return err
 	}
-
-	return len(p), nil
+	if n != len(dest) {
+		return fmt.Errorf("incomplete read from cache: read %d bytes, expected %d bytes", n, len(dest))
+	}
+	return nil
 }
 
 // fetchRegions fetches all specified chunks from remote blob and puts it in the local cache.
@@ -315,11 +320,7 @@ func (b *blob) fetchRegions(allData map[region]io.Writer, fetched map[region]boo
 		return nil
 	}
 
-	// Fetcher can be suddenly updated so we take and use the snapshot of it for
-	// consistency.
-	b.fetcherMu.Lock()
-	fr := b.fetcher
-	b.fetcherMu.Unlock()
+	fr := b.getFetcher()
 
 	// request missed regions
 	var req []region
@@ -334,7 +335,6 @@ func (b *blob) fetchRegions(allData map[region]io.Writer, fetched map[region]boo
 		fetchCtx = opts.ctx
 	}
 	mr, err := fr.fetch(fetchCtx, req, true)
-
 	if err != nil {
 		return err
 	}
@@ -352,41 +352,15 @@ func (b *blob) fetchRegions(allData map[region]io.Writer, fetched map[region]boo
 		if err == io.EOF {
 			break
 		} else if err != nil {
-			return errors.Wrapf(err, "failed to read multipart resp")
+			return fmt.Errorf("failed to read multipart resp: %w", err)
 		}
 		if err := b.walkChunks(reg, func(chunk region) (retErr error) {
-			id := fr.genID(chunk)
-			cw, err := b.cache.Add(id, opts.cacheOpts...)
-			if err != nil {
+			if err := b.cacheChunkData(chunk, p, fr, allData, fetched, opts); err != nil {
 				return err
 			}
-			defer cw.Close()
-			w := io.Writer(cw)
-
-			// If this chunk is one of the targets, write the content to the
-			// passed reader too.
-			if _, ok := fetched[chunk]; ok {
-				w = io.MultiWriter(w, allData[chunk])
-			}
-
-			// Copy the target chunk
-			if _, err := io.CopyN(w, p, chunk.size()); err != nil {
-				cw.Abort()
-				return err
-			}
-
-			// Add the target chunk to the cache
-			if err := cw.Commit(); err != nil {
-				return err
-			}
-
-			b.fetchedRegionSetMu.Lock()
-			b.fetchedRegionSet.add(chunk)
-			b.fetchedRegionSetMu.Unlock()
-			fetched[chunk] = true
 			return nil
 		}); err != nil {
-			return errors.Wrapf(err, "failed to get chunks")
+			return fmt.Errorf("failed to get chunks: %w", err)
 		}
 	}
 
@@ -410,9 +384,6 @@ func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
 		return nil
 	}
 
-	// We build a key based on regions we need to fetch and pass it to singleflightGroup.Do(...)
-	// to block simultaneous same requests. Once the request is finished and the data is ready,
-	// all blocked callers will be unblocked and that same data will be returned by all blocked callers.
 	key := makeSyncKey(allData)
 	fetched := make(map[region]bool)
 	_, err, shared := b.fetchedRegionGroup.Do(key, func() (interface{}, error) {
@@ -422,46 +393,66 @@ func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
 	// When unblocked try to read from cache in case if there were no errors
 	// If we fail reading from cache, fetch from remote registry again
 	if err == nil && shared {
-		for reg := range allData {
-			if _, ok := fetched[reg]; ok {
-				continue
-			}
-			err = b.walkChunks(reg, func(chunk region) error {
-				b.fetcherMu.Lock()
-				fr := b.fetcher
-				b.fetcherMu.Unlock()
-
-				// Check if the content exists in the cache
-				// And if exists, read from cache
-				r, err := b.cache.Get(fr.genID(chunk), opts.cacheOpts...)
-				if err != nil {
-					return err
-				}
-				defer r.Close()
-				rr := io.NewSectionReader(r, 0, chunk.size())
-
-				// Copy the target chunk
-				b.fetchedRegionCopyMu.Lock()
-				defer b.fetchedRegionCopyMu.Unlock()
-				if _, err := io.CopyN(allData[chunk], rr, chunk.size()); err != nil {
-					return err
-				}
-				return nil
-			})
-			if err != nil {
-				break
-			}
-		}
-
-		// if we cannot read the data from cache, do fetch again
-		if err != nil {
-			return b.fetchRange(allData, opts)
+		if err := b.handleSharedFetch(allData, fetched, opts); err != nil {
+			return b.fetchRange(allData, opts) // retry on error
 		}
 	}
 
 	return err
 }
 
+// handleSharedFetch handles the case when multiple goroutines share the same fetch result
+func (b *blob) handleSharedFetch(allData map[region]io.Writer, fetched map[region]bool, opts *options) error {
+	for reg := range allData {
+		if _, ok := fetched[reg]; ok {
+			continue
+		}
+		if err := b.copyFetchedChunks(reg, allData, opts); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// copyFetchedChunks copies fetched chunks from cache to target writer
+func (b *blob) copyFetchedChunks(reg region, allData map[region]io.Writer, opts *options) error {
+	return b.walkChunks(reg, func(chunk region) error {
+		fr := b.getFetcher()
+		r, err := b.cache.Get(fr.genID(chunk), opts.cacheOpts...)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+
+		b.fetchedRegionCopyMu.Lock()
+		defer b.fetchedRegionCopyMu.Unlock()
+
+		if _, err := io.CopyN(allData[chunk], io.NewSectionReader(r, 0, chunk.size()), chunk.size()); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+// getFetcher safely gets the current fetcher
+// Fetcher can be suddenly updated so we take and use the snapshot of it for consistency.
+func (b *blob) getFetcher() fetcher {
+	b.fetcherMu.Lock()
+	defer b.fetcherMu.Unlock()
+	return b.fetcher
+}
+
+// adjustBufferSize adjusts buffer size according to the blob size
+func (b *blob) adjustBufferSize(p []byte, offset int64) int {
+	if remain := b.size - offset; int64(len(p)) >= remain {
+		if remain < 0 {
+			remain = 0
+		}
+		p = p[:remain]
+	}
+	return len(p)
+}
+
 type walkFunc func(reg region) error
 
 // walkChunks walks chunks from begin to end in order in the specified region.
@@ -535,3 +526,34 @@ func positive(n int64) int64 {
 	}
 	return n
 }
+
+// cacheChunkData handles caching of chunk data
+func (b *blob) cacheChunkData(chunk region, r io.Reader, fr fetcher, allData map[region]io.Writer, fetched map[region]bool, opts *options) error {
+	id := fr.genID(chunk)
+	cw, err := b.cache.Add(id, opts.cacheOpts...)
+	if err != nil {
+		return fmt.Errorf("failed to create cache writer: %w", err)
+	}
+	defer cw.Close()
+
+	w := io.Writer(cw)
+	if _, ok := fetched[chunk]; ok {
+		w = io.MultiWriter(w, allData[chunk])
+	}
+
+	if _, err := io.CopyN(w, r, chunk.size()); err != nil {
+		cw.Abort()
+		return fmt.Errorf("failed to write chunk data: %w", err)
+	}
+
+	if err := cw.Commit(); err != nil {
+		return fmt.Errorf("failed to commit chunk: %w", err)
+	}
+
+	b.fetchedRegionSetMu.Lock()
+	b.fetchedRegionSet.add(chunk)
+	b.fetchedRegionSetMu.Unlock()
+	fetched[chunk] = true
+
+	return nil
+}

fs/remote/blob_test.go

@@ -26,7 +26,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"mime/multipart"
 	"net/http"
@@ -610,7 +609,7 @@ func TestCheckInterval(t *testing.T) {
 		if !tr.called {
 			return b.lastCheck, false
 		}
-		if !(b.lastCheck.After(beforeUpdate) && b.lastCheck.Before(afterUpdate)) {
+		if !b.lastCheck.After(beforeUpdate) || !b.lastCheck.Before(afterUpdate) {
 			t.Errorf("%q: updated time must be after %q and before %q but %q", name, beforeUpdate, afterUpdate, b.lastCheck)
 		}
 
@@ -646,7 +645,7 @@ func (c *callsCountRoundTripper) RoundTrip(req *http.Request) (res *http.Respons
 	return &http.Response{
 		StatusCode: http.StatusOK,
 		Header:     header,
-		Body:       convertBody(ioutil.NopCloser(bytes.NewReader([]byte(c.content)))),
+		Body:       convertBody(io.NopCloser(bytes.NewReader([]byte(c.content)))),
 	}, nil
 }
 
@@ -659,7 +658,7 @@ func (c *calledRoundTripper) RoundTrip(req *http.Request) (res *http.Response, e
 	res = &http.Response{
 		StatusCode: http.StatusOK,
 		Header:     make(http.Header),
-		Body:       ioutil.NopCloser(bytes.NewReader([]byte("test"))),
+		Body:       io.NopCloser(bytes.NewReader([]byte("test"))),
 	}
 	return
 }
@@ -691,7 +690,7 @@ func multiRoundTripper(t *testing.T, contents []byte, opts ...interface{}) Round
 		return &http.Response{
 			StatusCode: statusCode,
 			Header:     make(http.Header),
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
 		}
 	}
 
@@ -740,7 +739,7 @@ func multiRoundTripper(t *testing.T, contents []byte, opts ...interface{}) Round
 				return &http.Response{
 					StatusCode: http.StatusOK,
 					Header:     header,
-					Body:       convertBody(ioutil.NopCloser(bytes.NewReader(contents))),
+					Body:       convertBody(io.NopCloser(bytes.NewReader(contents))),
 				}
 			}
 		}
@@ -768,7 +767,7 @@ func multiRoundTripper(t *testing.T, contents []byte, opts ...interface{}) Round
 			return &http.Response{
 				StatusCode: http.StatusPartialContent,
 				Header:     header,
-				Body:       convertBody(ioutil.NopCloser(bytes.NewReader(part))),
+				Body:       convertBody(io.NopCloser(bytes.NewReader(part))),
 			}
 		}
 
@@ -809,7 +808,7 @@ func multiRoundTripper(t *testing.T, contents []byte, opts ...interface{}) Round
 		return &http.Response{
 			StatusCode: http.StatusPartialContent,
 			Header:     header,
-			Body:       convertBody(ioutil.NopCloser(&buf)),
+			Body:       convertBody(io.NopCloser(&buf)),
 		}
 	}
 }
@@ -819,7 +818,7 @@ func failRoundTripper() RoundTripFunc {
 		return &http.Response{
 			StatusCode: http.StatusInternalServerError,
 			Header:     make(http.Header),
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
 		}
 	}
 }
@@ -827,11 +826,11 @@ func failRoundTripper() RoundTripFunc {
 func brokenBodyRoundTripper(t *testing.T, contents []byte, multiRange bool) RoundTripFunc {
 	breakReadCloser := func(r io.ReadCloser) io.ReadCloser {
 		defer r.Close()
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		if err != nil {
 			t.Fatalf("failed to break read closer faild to read original: %v", err)
 		}
-		return ioutil.NopCloser(bytes.NewReader(data[:len(data)/2]))
+		return io.NopCloser(bytes.NewReader(data[:len(data)/2]))
 	}
 	tr := multiRoundTripper(t, contents, allowMultiRange(multiRange), bodyConverter(breakReadCloser))
 	return func(req *http.Request) *http.Response {

fs/remote/resolver.go

@@ -24,35 +24,32 @@ package remote
 
 import (
 	"context"
+	"crypto/rand"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
-	"math/rand"
+	"math/big"
 	"mime"
 	"mime/multipart"
 	"net/http"
-	"net/url"
 	"path"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/errdefs"
+	"github.com/containerd/log"
 	"github.com/containerd/stargz-snapshotter/cache"
 	"github.com/containerd/stargz-snapshotter/fs/config"
 	commonmetrics "github.com/containerd/stargz-snapshotter/fs/metrics/common"
 	"github.com/containerd/stargz-snapshotter/fs/source"
-	"github.com/hashicorp/go-multierror"
 	rhttp "github.com/hashicorp/go-retryablehttp"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -125,27 +122,29 @@ func (r *Resolver) Resolve(ctx context.Context, hosts source.RegistryHosts, refs
 func (r *Resolver) resolveFetcher(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) (f fetcher, size int64, err error) {
 	blobConfig := &r.blobConfig
 	fc := &fetcherConfig{
-		hosts:       hosts,
-		refspec:     refspec,
-		desc:        desc,
-		maxRetries:  blobConfig.MaxRetries,
-		minWaitMSec: time.Duration(blobConfig.MinWaitMSec) * time.Millisecond,
-		maxWaitMSec: time.Duration(blobConfig.MaxWaitMSec) * time.Millisecond,
+		hosts:      hosts,
+		refspec:    refspec,
+		desc:       desc,
+		maxRetries: blobConfig.MaxRetries,
+		minWait:    time.Duration(blobConfig.MinWaitMSec) * time.Millisecond,
+		maxWait:    time.Duration(blobConfig.MaxWaitMSec) * time.Millisecond,
 	}
-	var handlersErr error
+	var errs []error
 	for name, p := range r.handlers {
 		// TODO: allow to configure the selection of readers based on the hostname in refspec
 		r, size, err := p.Handle(ctx, desc)
 		if err != nil {
-			handlersErr = multierror.Append(handlersErr, err)
+			errs = append(errs, err)
 			continue
 		}
-		log.G(ctx).WithField("handler name", name).WithField("ref", refspec.String).WithField("digest", desc.Digest).
+		log.G(ctx).WithField("handler name", name).WithField("ref", refspec.String()).WithField("digest", desc.Digest).
 			Debugf("contents is provided by a handler")
 		return &remoteFetcher{r}, size, nil
 	}
 
-	log.G(ctx).WithError(handlersErr).WithField("ref", refspec.String).WithField("digest", desc.Digest).Debugf("using default handler")
+	handlersErr := errors.Join(errs...)
+
+	log.G(ctx).WithError(handlersErr).WithField("ref", refspec.String()).WithField("digest", desc.Digest).Debugf("using default handler")
 	hf, size, err := newHTTPFetcher(ctx, fc)
 	if err != nil {
 		return nil, 0, err
@@ -157,16 +156,23 @@ func (r *Resolver) resolveFetcher(ctx context.Context, hosts source.RegistryHost
 }
 
 type fetcherConfig struct {
-	hosts       source.RegistryHosts
-	refspec     reference.Spec
-	desc        ocispec.Descriptor
-	maxRetries  int
-	minWaitMSec time.Duration
-	maxWaitMSec time.Duration
+	hosts      source.RegistryHosts
+	refspec    reference.Spec
+	desc       ocispec.Descriptor
+	maxRetries int
+	minWait    time.Duration
+	maxWait    time.Duration
 }
 
 func jitter(duration time.Duration) time.Duration {
-	return time.Duration(rand.Int63n(int64(duration)) + int64(duration))
+	if duration <= 0 {
+		return duration
+	}
+	b, err := rand.Int(rand.Reader, big.NewInt(int64(duration)))
+	if err != nil {
+		panic(err)
+	}
+	return time.Duration(b.Int64() + int64(duration))
 }
 
 // backoffStrategy extends retryablehttp's DefaultBackoff to add a random jitter to avoid overwhelming the repository
@@ -178,16 +184,13 @@ func backoffStrategy(min, max time.Duration, attemptNum int, resp *http.Response
 	return jitter(delayTime)
 }
 
-// retryStrategy extends retryablehttp's DefaultRetryPolicy to log the error and response when retrying
+// retryStrategy extends retryablehttp's DefaultRetryPolicy to debug log the error when retrying
 // DefaultRetryPolicy retries whenever err is non-nil (except for some url errors) or if returned
 // status code is 429 or 5xx (except 501)
 func retryStrategy(ctx context.Context, resp *http.Response, err error) (bool, error) {
 	retry, err2 := rhttp.DefaultRetryPolicy(ctx, resp, err)
 	if retry {
-		log.G(ctx).WithFields(logrus.Fields{
-			"error":    err,
-			"response": resp,
-		}).Infof("Retrying request")
+		log.G(ctx).WithError(err).Debugf("Retrying request")
 	}
 	return retry, err2
 }
@@ -199,10 +202,10 @@ func newHTTPFetcher(ctx context.Context, fc *fetcherConfig) (*httpFetcher, int64
 	}
 	desc := fc.desc
 	if desc.Digest.String() == "" {
-		return nil, 0, fmt.Errorf("Digest is mandatory in layer descriptor")
+		return nil, 0, fmt.Errorf("digest is mandatory in layer descriptor")
 	}
 	digest := desc.Digest
-	pullScope, err := repositoryScope(fc.refspec, false)
+	pullScope, err := docker.RepositoryScope(fc.refspec, false)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -211,8 +214,7 @@ func newHTTPFetcher(ctx context.Context, fc *fetcherConfig) (*httpFetcher, int64
 	rErr := fmt.Errorf("failed to resolve")
 	for _, host := range reghosts {
 		if host.Host == "" || strings.Contains(host.Host, "/") {
-			rErr = errors.Wrapf(rErr, "invalid destination (host %q, ref:%q, digest:%q)",
-				host.Host, fc.refspec, digest)
+			rErr = fmt.Errorf("invalid destination (host %q, ref:%q, digest:%q): %w", host.Host, fc.refspec, digest, rErr)
 			continue // Try another
 
 		}
@@ -220,15 +222,16 @@ func newHTTPFetcher(ctx context.Context, fc *fetcherConfig) (*httpFetcher, int64
 		// Prepare transport with authorization functionality
 		tr := host.Client.Transport
 
+		timeout := host.Client.Timeout
 		if rt, ok := tr.(*rhttp.RoundTripper); ok {
 			rt.Client.RetryMax = fc.maxRetries
-			rt.Client.RetryWaitMin = fc.minWaitMSec
-			rt.Client.RetryWaitMax = fc.maxWaitMSec
+			rt.Client.RetryWaitMin = fc.minWait
+			rt.Client.RetryWaitMax = fc.maxWait
 			rt.Client.Backoff = backoffStrategy
 			rt.Client.CheckRetry = retryStrategy
+			timeout = rt.Client.HTTPClient.Timeout
 		}
 
-		timeout := host.Client.Timeout
 		if host.Authorizer != nil {
 			tr = &transport{
 				inner: tr,
@@ -243,35 +246,35 @@ func newHTTPFetcher(ctx context.Context, fc *fetcherConfig) (*httpFetcher, int64
 			path.Join(host.Host, host.Path),
 			strings.TrimPrefix(fc.refspec.Locator, fc.refspec.Hostname()+"/"),
 			digest)
-		url, err := redirect(ctx, blobURL, tr, timeout)
+		url, header, err := redirect(ctx, blobURL, tr, timeout, host.Header)
 		if err != nil {
-			rErr = errors.Wrapf(rErr, "failed to redirect (host %q, ref:%q, digest:%q): %v",
-				host.Host, fc.refspec, digest, err)
+			rErr = fmt.Errorf("failed to redirect (host %q, ref:%q, digest:%q): %v: %w", host.Host, fc.refspec, digest, err, rErr)
 			continue // Try another
 		}
 
 		// Get size information
 		// TODO: we should try to use the Size field in the descriptor here.
 		start := time.Now() // start time before getting layer header
-		size, err := getSize(ctx, url, tr, timeout)
+		size, err := getSize(ctx, url, tr, timeout, header)
 		commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.StargzHeaderGet, digest, start) // time to get layer header
 		if err != nil {
-			rErr = errors.Wrapf(rErr, "failed to get size (host %q, ref:%q, digest:%q): %v",
-				host.Host, fc.refspec, digest, err)
+			rErr = fmt.Errorf("failed to get size (host %q, ref:%q, digest:%q): %v: %w", host.Host, fc.refspec, digest, err, rErr)
 			continue // Try another
 		}
 
 		// Hit one destination
 		return &httpFetcher{
-			url:     url,
-			tr:      tr,
-			blobURL: blobURL,
-			digest:  digest,
-			timeout: timeout,
+			url:       url,
+			tr:        tr,
+			blobURL:   blobURL,
+			digest:    digest,
+			timeout:   timeout,
+			header:    header,
+			orgHeader: host.Header,
 		}, size, nil
 	}
 
-	return nil, 0, errors.Wrapf(rErr, "cannot resolve layer")
+	return nil, 0, fmt.Errorf("cannot resolve layer: %w", rErr)
 }
 
 type transport struct {
@@ -316,7 +319,7 @@ func (tr *transport) RoundTrip(req *http.Request) (*http.Response, error) {
 	return resp, nil
 }
 
-func redirect(ctx context.Context, blobURL string, tr http.RoundTripper, timeout time.Duration) (url string, err error) {
+func redirect(ctx context.Context, blobURL string, tr http.RoundTripper, timeout time.Duration, header http.Header) (url string, withHeader http.Header, err error) {
 	if timeout > 0 {
 		var cancel context.CancelFunc
 		ctx, cancel = context.WithTimeout(ctx, timeout)
@@ -327,32 +330,38 @@ func redirect(ctx context.Context, blobURL string, tr http.RoundTripper, timeout
 	// ghcr.io returns 200 on HEAD without Location header (2020).
 	req, err := http.NewRequestWithContext(ctx, "GET", blobURL, nil)
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to make request to the registry")
+		return "", nil, fmt.Errorf("failed to make request to the registry: %w", err)
+	}
+	req.Header = http.Header{}
+	for k, v := range header {
+		req.Header[k] = v
 	}
 	req.Close = false
 	req.Header.Set("Range", "bytes=0-1")
 	res, err := tr.RoundTrip(req)
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to request")
+		return "", nil, fmt.Errorf("failed to request: %w", err)
 	}
 	defer func() {
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 		res.Body.Close()
 	}()
 
 	if res.StatusCode/100 == 2 {
 		url = blobURL
+		withHeader = header
 	} else if redir := res.Header.Get("Location"); redir != "" && res.StatusCode/100 == 3 {
 		// TODO: Support nested redirection
 		url = redir
+		// Do not pass headers to the redirected location.
 	} else {
-		return "", fmt.Errorf("failed to access to the registry with code %v", res.StatusCode)
+		return "", nil, fmt.Errorf("failed to access to the registry with code %v", res.StatusCode)
 	}
 
 	return
 }
 
-func getSize(ctx context.Context, url string, tr http.RoundTripper, timeout time.Duration) (int64, error) {
+func getSize(ctx context.Context, url string, tr http.RoundTripper, timeout time.Duration, header http.Header) (int64, error) {
 	if timeout > 0 {
 		var cancel context.CancelFunc
 		ctx, cancel = context.WithTimeout(ctx, timeout)
@@ -362,6 +371,10 @@ func getSize(ctx context.Context, url string, tr http.RoundTripper, timeout time
 	if err != nil {
 		return 0, err
 	}
+	req.Header = http.Header{}
+	for k, v := range header {
+		req.Header[k] = v
+	}
 	req.Close = false
 	res, err := tr.RoundTrip(req)
 	if err != nil {
@@ -378,22 +391,27 @@ func getSize(ctx context.Context, url string, tr http.RoundTripper, timeout time
 	// HEAD request (2020).
 	req, err = http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
-		return 0, errors.Wrapf(err, "failed to make request to the registry")
+		return 0, fmt.Errorf("failed to make request to the registry: %w", err)
+	}
+	req.Header = http.Header{}
+	for k, v := range header {
+		req.Header[k] = v
 	}
 	req.Close = false
 	req.Header.Set("Range", "bytes=0-1")
 	res, err = tr.RoundTrip(req)
 	if err != nil {
-		return 0, errors.Wrapf(err, "failed to request")
+		return 0, fmt.Errorf("failed to request: %w", err)
 	}
 	defer func() {
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 		res.Body.Close()
 	}()
 
-	if res.StatusCode == http.StatusOK {
+	switch res.StatusCode {
+	case http.StatusOK:
 		return strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
-	} else if res.StatusCode == http.StatusPartialContent {
+	case http.StatusPartialContent:
 		_, size, err := parseRange(res.Header.Get("Content-Range"))
 		return size, err
 	}
@@ -411,6 +429,8 @@ type httpFetcher struct {
 	singleRange   bool
 	singleRangeMu sync.Mutex
 	timeout       time.Duration
+	header        http.Header
+	orgHeader     http.Header
 }
 
 type multipartReadCloser interface {
@@ -450,6 +470,10 @@ func (f *httpFetcher) fetch(ctx context.Context, rs []region, retry bool) (multi
 	if err != nil {
 		return nil, err
 	}
+	req.Header = http.Header{}
+	for k, v := range f.header {
+		req.Header[k] = v
+	}
 	var ranges string
 	for _, reg := range requests {
 		ranges += fmt.Sprintf("%d-%d,", reg.b, reg.e)
@@ -469,13 +493,13 @@ func (f *httpFetcher) fetch(ctx context.Context, rs []region, retry bool) (multi
 		// We are getting the whole blob in one part (= status 200)
 		size, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse Content-Length")
+			return nil, fmt.Errorf("failed to parse Content-Length: %w", err)
 		}
 		return newSinglePartReader(region{0, size - 1}, res.Body), nil
 	} else if res.StatusCode == http.StatusPartialContent {
 		mediaType, params, err := mime.ParseMediaType(res.Header.Get("Content-Type"))
 		if err != nil {
-			return nil, errors.Wrapf(err, "invalid media type %q", mediaType)
+			return nil, fmt.Errorf("invalid media type %q: %w", mediaType, err)
 		}
 		if strings.HasPrefix(mediaType, "multipart/") {
 			// We are getting a set of chunks as a multipart body.
@@ -485,7 +509,7 @@ func (f *httpFetcher) fetch(ctx context.Context, rs []region, retry bool) (multi
 		// We are getting single range
 		reg, _, err := parseRange(res.Header.Get("Content-Range"))
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse Content-Range")
+			return nil, fmt.Errorf("failed to parse Content-Range: %w", err)
 		}
 		return newSinglePartReader(reg, res.Body), nil
 	} else if retry && res.StatusCode == http.StatusForbidden {
@@ -493,7 +517,7 @@ func (f *httpFetcher) fetch(ctx context.Context, rs []region, retry bool) (multi
 
 		// re-redirect and retry this once.
 		if err := f.refreshURL(ctx); err != nil {
-			return nil, errors.Wrapf(err, "failed to refresh URL on %v", res.Status)
+			return nil, fmt.Errorf("failed to refresh URL on %v: %w", res.Status, err)
 		}
 		return f.fetch(ctx, rs, false)
 	} else if retry && res.StatusCode == http.StatusBadRequest && !singleRangeMode {
@@ -519,21 +543,26 @@ func (f *httpFetcher) check() error {
 	f.urlMu.Unlock()
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
-		return errors.Wrapf(err, "check failed: failed to make request")
+		return fmt.Errorf("check failed: failed to make request: %w", err)
+	}
+	req.Header = http.Header{}
+	for k, v := range f.header {
+		req.Header[k] = v
 	}
 	req.Close = false
 	req.Header.Set("Range", "bytes=0-1")
 	res, err := f.tr.RoundTrip(req)
 	if err != nil {
-		return errors.Wrapf(err, "check failed: failed to request to registry")
+		return fmt.Errorf("check failed: failed to request to registry: %w", err)
 	}
 	defer func() {
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 		res.Body.Close()
 	}()
-	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusPartialContent {
+	switch res.StatusCode {
+	case http.StatusOK, http.StatusPartialContent:
 		return nil
-	} else if res.StatusCode == http.StatusForbidden {
+	case http.StatusForbidden:
 		// Try to re-redirect this blob
 		rCtx := context.Background()
 		if f.timeout > 0 {
@@ -551,12 +580,13 @@ func (f *httpFetcher) check() error {
 }
 
 func (f *httpFetcher) refreshURL(ctx context.Context) error {
-	newURL, err := redirect(ctx, f.blobURL, f.tr, f.timeout)
+	newURL, headers, err := redirect(ctx, f.blobURL, f.tr, f.timeout, f.orgHeader)
 	if err != nil {
 		return err
 	}
 	f.urlMu.Lock()
 	f.url = newURL
+	f.header = headers
 	f.urlMu.Unlock()
 	return nil
 }
@@ -621,7 +651,7 @@ func (sr *multipartReader) Next() (region, io.Reader, error) {
 	}
 	reg, _, err := parseRange(p.Header.Get("Content-Range"))
 	if err != nil {
-		return region{}, nil, errors.Wrapf(err, "failed to parse Content-Range")
+		return region{}, nil, fmt.Errorf("failed to parse Content-Range: %w", err)
 	}
 	return reg, p, nil
 }
@@ -633,15 +663,15 @@ func parseRange(header string) (region, int64, error) {
 	}
 	begin, err := strconv.ParseInt(submatches[1], 10, 64)
 	if err != nil {
-		return region{}, 0, errors.Wrapf(err, "failed to parse beginning offset %q", submatches[1])
+		return region{}, 0, fmt.Errorf("failed to parse beginning offset %q: %w", submatches[1], err)
 	}
 	end, err := strconv.ParseInt(submatches[2], 10, 64)
 	if err != nil {
-		return region{}, 0, errors.Wrapf(err, "failed to parse end offset %q", submatches[2])
+		return region{}, 0, fmt.Errorf("failed to parse end offset %q: %w", submatches[2], err)
 	}
 	blobSize, err := strconv.ParseInt(submatches[3], 10, 64)
 	if err != nil {
-		return region{}, 0, errors.Wrapf(err, "failed to parse blob size %q", submatches[3])
+		return region{}, 0, fmt.Errorf("failed to parse blob size %q: %w", submatches[3], err)
 	}
 
 	return region{begin, end}, blobSize, nil
@@ -666,24 +696,6 @@ func WithCacheOpts(cacheOpts ...cache.Option) Option {
 	}
 }
 
-// NOTE: ported from https://github.com/containerd/containerd/blob/v1.5.2/remotes/docker/scope.go#L29-L42
-// TODO: import this from containerd package once we drop support to continerd v1.4.x
-//
-// repositoryScope returns a repository scope string such as "repository:foo/bar:pull"
-// for "host/foo/bar:baz".
-// When push is true, both pull and push are added to the scope.
-func repositoryScope(refspec reference.Spec, push bool) (string, error) {
-	u, err := url.Parse("dummy://" + refspec.Locator)
-	if err != nil {
-		return "", err
-	}
-	s := "repository:" + strings.TrimPrefix(u.Path, "/") + ":pull"
-	if push {
-		s += ",push"
-	}
-	return s, nil
-}
-
 type remoteFetcher struct {
 	r Fetcher
 }

fs/remote/resolver_test.go

@@ -26,15 +26,16 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
 	"testing"
 
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/stargz-snapshotter/fs/source"
 	rhttp "github.com/hashicorp/go-retryablehttp"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -55,117 +56,219 @@ func TestMirror(t *testing.T) {
 
 	tests := []struct {
 		name     string
-		tr       http.RoundTripper
-		mirrors  []string
+		hosts    func(t *testing.T) source.RegistryHosts
 		wantHost string
 		error    bool
 	}{
 		{
-			name:     "no-mirror",
-			tr:       &sampleRoundTripper{okURLs: []string{refHost}},
-			mirrors:  nil,
+			name: "no-mirror",
+			hosts: hostsConfig(
+				&sampleRoundTripper{okURLs: []string{refHost}},
+			),
 			wantHost: refHost,
 		},
 		{
-			name:     "valid-mirror",
-			tr:       &sampleRoundTripper{okURLs: []string{"mirrorexample.com"}},
-			mirrors:  []string{"mirrorexample.com"},
+			name: "valid-mirror",
+			hosts: hostsConfig(
+				&sampleRoundTripper{okURLs: []string{"mirrorexample.com"}},
+				hostSimple("mirrorexample.com"),
+			),
 			wantHost: "mirrorexample.com",
 		},
 		{
 			name: "invalid-mirror",
-			tr: &sampleRoundTripper{
-				withCode: map[string]int{
-					"mirrorexample1.com": http.StatusInternalServerError,
-					"mirrorexample2.com": http.StatusUnauthorized,
-					"mirrorexample3.com": http.StatusNotFound,
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					withCode: map[string]int{
+						"mirrorexample1.com": http.StatusInternalServerError,
+						"mirrorexample2.com": http.StatusUnauthorized,
+						"mirrorexample3.com": http.StatusNotFound,
+					},
+					okURLs: []string{"mirrorexample4.com", refHost},
 				},
-				okURLs: []string{"mirrorexample4.com", refHost},
-			},
-			mirrors: []string{
-				"mirrorexample1.com",
-				"mirrorexample2.com",
-				"mirrorexample3.com",
-				"mirrorexample4.com",
-			},
+				hostSimple("mirrorexample1.com"),
+				hostSimple("mirrorexample2.com"),
+				hostSimple("mirrorexample3.com"),
+				hostSimple("mirrorexample4.com"),
+			),
 			wantHost: "mirrorexample4.com",
 		},
 		{
 			name: "invalid-all-mirror",
-			tr: &sampleRoundTripper{
-				withCode: map[string]int{
-					"mirrorexample1.com": http.StatusInternalServerError,
-					"mirrorexample2.com": http.StatusUnauthorized,
-					"mirrorexample3.com": http.StatusNotFound,
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					withCode: map[string]int{
+						"mirrorexample1.com": http.StatusInternalServerError,
+						"mirrorexample2.com": http.StatusUnauthorized,
+						"mirrorexample3.com": http.StatusNotFound,
+					},
+					okURLs: []string{refHost},
 				},
-				okURLs: []string{refHost},
-			},
-			mirrors: []string{
-				"mirrorexample1.com",
-				"mirrorexample2.com",
-				"mirrorexample3.com",
-			},
+				hostSimple("mirrorexample1.com"),
+				hostSimple("mirrorexample2.com"),
+				hostSimple("mirrorexample3.com"),
+			),
 			wantHost: refHost,
 		},
 		{
 			name: "invalid-hostname-of-mirror",
-			tr: &sampleRoundTripper{
-				okURLs: []string{`.*`},
-			},
-			mirrors:  []string{"mirrorexample.com/somepath/"},
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					okURLs: []string{`.*`},
+				},
+				hostSimple("mirrorexample.com/somepath/"),
+			),
 			wantHost: refHost,
 		},
 		{
 			name: "redirected-mirror",
-			tr: &sampleRoundTripper{
-				redirectURL: map[string]string{
-					regexp.QuoteMeta(fmt.Sprintf("mirrorexample.com%s", blobPath)): "https://backendexample.com/blobs/" + blobDigest.String(),
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					redirectURL: map[string]string{
+						regexp.QuoteMeta(fmt.Sprintf("mirrorexample.com%s", blobPath)): "https://backendexample.com/blobs/" + blobDigest.String(),
+					},
+					okURLs: []string{`.*`},
 				},
-				okURLs: []string{`.*`},
-			},
-			mirrors:  []string{"mirrorexample.com"},
+				hostSimple("mirrorexample.com"),
+			),
 			wantHost: "backendexample.com",
 		},
 		{
 			name: "invalid-redirected-mirror",
-			tr: &sampleRoundTripper{
-				withCode: map[string]int{
-					"backendexample.com": http.StatusInternalServerError,
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					withCode: map[string]int{
+						"backendexample.com": http.StatusInternalServerError,
+					},
+					redirectURL: map[string]string{
+						regexp.QuoteMeta(fmt.Sprintf("mirrorexample.com%s", blobPath)): "https://backendexample.com/blobs/" + blobDigest.String(),
+					},
+					okURLs: []string{`.*`},
 				},
-				redirectURL: map[string]string{
-					regexp.QuoteMeta(fmt.Sprintf("mirrorexample.com%s", blobPath)): "https://backendexample.com/blobs/" + blobDigest.String(),
-				},
-				okURLs: []string{`.*`},
-			},
-			mirrors:  []string{"mirrorexample.com"},
+				hostSimple("mirrorexample.com"),
+			),
 			wantHost: refHost,
 		},
 		{
-			name:     "fail-all",
-			tr:       &sampleRoundTripper{},
-			mirrors:  []string{"mirrorexample.com"},
+			name: "fail-all",
+			hosts: hostsConfig(
+				&sampleRoundTripper{},
+				hostSimple("mirrorexample.com"),
+			),
 			wantHost: "",
 			error:    true,
 		},
+		{
+			name: "headers",
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					okURLs: []string{`.*`},
+					wantHeaders: map[string]http.Header{
+						"mirrorexample.com": http.Header(map[string][]string{
+							"test-a-key": {"a-value-1", "a-value-2"},
+							"test-b-key": {"b-value-1"},
+						}),
+					},
+				},
+				hostWithHeaders("mirrorexample.com", map[string][]string{
+					"test-a-key": {"a-value-1", "a-value-2"},
+					"test-b-key": {"b-value-1"},
+				}),
+			),
+			wantHost: "mirrorexample.com",
+		},
+		{
+			name: "headers-with-mirrors",
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					withCode: map[string]int{
+						"mirrorexample1.com": http.StatusInternalServerError,
+						"mirrorexample2.com": http.StatusInternalServerError,
+					},
+					okURLs: []string{"mirrorexample3.com", refHost},
+					wantHeaders: map[string]http.Header{
+						"mirrorexample1.com": http.Header(map[string][]string{
+							"test-a-key": {"a-value"},
+						}),
+						"mirrorexample2.com": http.Header(map[string][]string{
+							"test-b-key":   {"b-value"},
+							"test-b-key-2": {"b-value-2", "b-value-3"},
+						}),
+						"mirrorexample3.com": http.Header(map[string][]string{
+							"test-c-key": {"c-value"},
+						}),
+					},
+				},
+				hostWithHeaders("mirrorexample1.com", map[string][]string{
+					"test-a-key": {"a-value"},
+				}),
+				hostWithHeaders("mirrorexample2.com", map[string][]string{
+					"test-b-key":   {"b-value"},
+					"test-b-key-2": {"b-value-2", "b-value-3"},
+				}),
+				hostWithHeaders("mirrorexample3.com", map[string][]string{
+					"test-c-key": {"c-value"},
+				}),
+			),
+			wantHost: "mirrorexample3.com",
+		},
+		{
+			name: "headers-with-mirrors-invalid-all",
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					withCode: map[string]int{
+						"mirrorexample1.com": http.StatusInternalServerError,
+						"mirrorexample2.com": http.StatusInternalServerError,
+					},
+					okURLs: []string{"mirrorexample3.com", refHost},
+					wantHeaders: map[string]http.Header{
+						"mirrorexample1.com": http.Header(map[string][]string{
+							"test-a-key": {"a-value"},
+						}),
+						"mirrorexample2.com": http.Header(map[string][]string{
+							"test-b-key":   {"b-value"},
+							"test-b-key-2": {"b-value-2", "b-value-3"},
+						}),
+					},
+				},
+				hostWithHeaders("mirrorexample1.com", map[string][]string{
+					"test-a-key": {"a-value"},
+				}),
+				hostWithHeaders("mirrorexample2.com", map[string][]string{
+					"test-b-key":   {"b-value"},
+					"test-b-key-2": {"b-value-2", "b-value-3"},
+				}),
+			),
+			wantHost: refHost,
+		},
+		{
+			name: "headers-with-redirected-mirror",
+			hosts: hostsConfig(
+				&sampleRoundTripper{
+					redirectURL: map[string]string{
+						regexp.QuoteMeta(fmt.Sprintf("mirrorexample.com%s", blobPath)): "https://backendexample.com/blobs/" + blobDigest.String(),
+					},
+					okURLs: []string{`.*`},
+					wantHeaders: map[string]http.Header{
+						"mirrorexample.com": http.Header(map[string][]string{
+							"test-a-key":   {"a-value"},
+							"test-b-key-2": {"b-value-2", "b-value-3"},
+						}),
+					},
+				},
+				hostWithHeaders("mirrorexample.com", map[string][]string{
+					"test-a-key":   {"a-value"},
+					"test-b-key-2": {"b-value-2", "b-value-3"},
+				}),
+			),
+			wantHost: "backendexample.com",
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			hosts := func(refspec reference.Spec) (reghosts []docker.RegistryHost, _ error) {
-				host := refspec.Hostname()
-				for _, m := range append(tt.mirrors, host) {
-					reghosts = append(reghosts, docker.RegistryHost{
-						Client:       &http.Client{Transport: tt.tr},
-						Host:         m,
-						Scheme:       "https",
-						Path:         "/v2",
-						Capabilities: docker.HostCapabilityPull,
-					})
-				}
-				return
-			}
 			fetcher, _, err := newHTTPFetcher(context.Background(), &fetcherConfig{
-				hosts:   hosts,
+				hosts:   tt.hosts(t),
 				refspec: refspec,
 				desc:    ocispec.Descriptor{Digest: blobDigest},
 			})
@@ -175,31 +278,90 @@ func TestMirror(t *testing.T) {
 				}
 				t.Fatalf("failed to resolve reference: %v", err)
 			}
-			nurl, err := url.Parse(fetcher.url)
-			if err != nil {
-				t.Fatalf("failed to parse url %q: %v", fetcher.url, err)
+			checkFetcherURL(t, fetcher, tt.wantHost)
+
+			// Test check()
+			if err := fetcher.check(); err != nil {
+				t.Fatalf("failed to check fetcher: %v", err)
 			}
-			if nurl.Hostname() != tt.wantHost {
-				t.Errorf("invalid hostname %q(%q); want %q",
-					nurl.Hostname(), nurl.String(), tt.wantHost)
+
+			// Test refreshURL()
+			if err := fetcher.refreshURL(context.TODO()); err != nil {
+				t.Fatalf("failed to refresh URL: %v", err)
 			}
+			checkFetcherURL(t, fetcher, tt.wantHost)
 		})
 	}
 }
 
+func checkFetcherURL(t *testing.T, f *httpFetcher, wantHost string) {
+	nurl, err := url.Parse(f.url)
+	if err != nil {
+		t.Fatalf("failed to parse url %q: %v", f.url, err)
+	}
+	if nurl.Hostname() != wantHost {
+		t.Errorf("invalid hostname %q(%q); want %q", nurl.Hostname(), nurl.String(), wantHost)
+	}
+}
+
 type sampleRoundTripper struct {
+	t           *testing.T
 	withCode    map[string]int
 	redirectURL map[string]string
 	okURLs      []string
+	wantHeaders map[string]http.Header
+}
+
+func getTestHeaders(headers map[string][]string) map[string][]string {
+	res := make(map[string][]string)
+	for k, v := range headers {
+		if strings.HasPrefix(k, "test-") {
+			res[k] = v
+		}
+	}
+	return res
 }
 
 func (tr *sampleRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	reqHeader := getTestHeaders(req.Header)
+	for host, wHeaders := range tr.wantHeaders {
+		wantHeader := getTestHeaders(wHeaders)
+		if ok, _ := regexp.Match(host, []byte(req.URL.String())); ok {
+			if len(wantHeader) != len(reqHeader) {
+				tr.t.Fatalf("unexpected num of headers; got %d, wanted %d", len(wantHeader), len(reqHeader))
+			}
+			for k, v := range wantHeader {
+				gotV, ok := reqHeader[k]
+				if !ok {
+					tr.t.Fatalf("required header %q not found; got %+v", k, reqHeader)
+				}
+				wantVM := make(map[string]struct{})
+				for _, e := range v {
+					wantVM[e] = struct{}{}
+				}
+				if len(gotV) != len(v) {
+					tr.t.Fatalf("unexpected num of header values of %q; got %d, wanted %d", k, len(gotV), len(v))
+				}
+				for _, gotE := range gotV {
+					delete(wantVM, gotE)
+				}
+				if len(wantVM) != 0 {
+					tr.t.Fatalf("header %q must have elements %+v", k, wantVM)
+				}
+				delete(reqHeader, k)
+			}
+		}
+	}
+	if len(reqHeader) != 0 {
+		tr.t.Fatalf("unexpected headers %+v", reqHeader)
+	}
+
 	for host, code := range tr.withCode {
 		if ok, _ := regexp.Match(host, []byte(req.URL.String())); ok {
 			return &http.Response{
 				StatusCode: code,
 				Header:     make(http.Header),
-				Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+				Body:       io.NopCloser(bytes.NewReader([]byte{})),
 				Request:    req,
 			}, nil
 		}
@@ -211,7 +373,7 @@ func (tr *sampleRoundTripper) RoundTrip(req *http.Request) (*http.Response, erro
 			return &http.Response{
 				StatusCode: http.StatusMovedPermanently,
 				Header:     header,
-				Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+				Body:       io.NopCloser(bytes.NewReader([]byte{})),
 				Request:    req,
 			}, nil
 		}
@@ -223,7 +385,7 @@ func (tr *sampleRoundTripper) RoundTrip(req *http.Request) (*http.Response, erro
 			return &http.Response{
 				StatusCode: http.StatusOK,
 				Header:     header,
-				Body:       ioutil.NopCloser(bytes.NewReader([]byte{0})),
+				Body:       io.NopCloser(bytes.NewReader([]byte{0})),
 				Request:    req,
 			}, nil
 		}
@@ -231,7 +393,7 @@ func (tr *sampleRoundTripper) RoundTrip(req *http.Request) (*http.Response, erro
 	return &http.Response{
 		StatusCode: http.StatusNotFound,
 		Header:     make(http.Header),
-		Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+		Body:       io.NopCloser(bytes.NewReader([]byte{})),
 		Request:    req,
 	}, nil
 }
@@ -262,13 +424,13 @@ func (b *breakRoundTripper) RoundTrip(req *http.Request) (res *http.Response, er
 		res = &http.Response{
 			StatusCode: http.StatusPartialContent,
 			Header:     make(http.Header),
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte("test"))),
+			Body:       io.NopCloser(bytes.NewReader([]byte("test"))),
 		}
 	} else {
 		res = &http.Response{
 			StatusCode: http.StatusInternalServerError,
 			Header:     make(http.Header),
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
 		}
 	}
 	return
@@ -313,13 +475,13 @@ func (r *retryRoundTripper) RoundTrip(req *http.Request) (res *http.Response, er
 		res = &http.Response{
 			StatusCode: http.StatusTooManyRequests,
 			Header:     make(http.Header),
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
 		}
 	case 2:
 		res = &http.Response{
 			StatusCode: http.StatusServiceUnavailable,
 			Header:     make(http.Header),
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
 		}
 	default:
 		header := make(http.Header)
@@ -327,8 +489,49 @@ func (r *retryRoundTripper) RoundTrip(req *http.Request) (res *http.Response, er
 		res = &http.Response{
 			StatusCode: http.StatusOK,
 			Header:     header,
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte("test"))),
+			Body:       io.NopCloser(bytes.NewReader([]byte("test"))),
 		}
 	}
 	return
 }
+
+type hostFactory func(tr http.RoundTripper) docker.RegistryHost
+
+func hostSimple(host string) hostFactory {
+	return func(tr http.RoundTripper) docker.RegistryHost {
+		return docker.RegistryHost{
+			Client:       &http.Client{Transport: tr},
+			Host:         host,
+			Scheme:       "https",
+			Path:         "/v2",
+			Capabilities: docker.HostCapabilityPull,
+		}
+	}
+}
+
+func hostWithHeaders(host string, headers http.Header) hostFactory {
+	return func(tr http.RoundTripper) docker.RegistryHost {
+		return docker.RegistryHost{
+			Client:       &http.Client{Transport: tr},
+			Host:         host,
+			Scheme:       "https",
+			Path:         "/v2",
+			Capabilities: docker.HostCapabilityPull,
+			Header:       headers,
+		}
+	}
+}
+
+func hostsConfig(tr *sampleRoundTripper, mirrors ...hostFactory) func(t *testing.T) source.RegistryHosts {
+	return func(t *testing.T) source.RegistryHosts {
+		tr.t = t
+		return func(refspec reference.Spec) (reghosts []docker.RegistryHost, _ error) {
+			host := refspec.Hostname()
+			for _, m := range mirrors {
+				reghosts = append(reghosts, m(tr))
+			}
+			reghosts = append(reghosts, hostSimple(host)(tr))
+			return
+		}
+	}
+}

fs/source/source.go

@@ -21,10 +21,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/labels"
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/labels"
+	"github.com/containerd/containerd/v2/pkg/reference"
 	"github.com/containerd/stargz-snapshotter/fs/config"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -200,3 +200,72 @@ func appendWithValidation(key string, values []string) string {
 	}
 	return strings.TrimSuffix(v, ",")
 }
+
+// TODO: switch to "github.com/containerd/containerd/pkg/snapshotters" once all tools using
+//
+//	stargz-snapshotter (e.g. k3s) move to containerd version where that pkg is available.
+const (
+	// targetImageLayersLabel is a label which contains layer digests contained in
+	// the target image and will be passed to snapshotters for preparing layers in
+	// parallel. Skipping some layers is allowed and only affects performance.
+	targetImageLayersLabelContainerd = "containerd.io/snapshot/cri.image-layers"
+)
+
+// AppendExtraLabelsHandler adds optional labels that aren't provided by
+// "github.com/containerd/containerd/pkg/snapshotters" but can be used for stargz snapshotter's extra functionalities.
+func AppendExtraLabelsHandler(prefetchSize int64, wrapper func(images.Handler) images.Handler) func(images.Handler) images.Handler {
+	return func(f images.Handler) images.Handler {
+		return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+			children, err := wrapper(f).Handle(ctx, desc)
+			if err != nil {
+				return nil, err
+			}
+			switch desc.MediaType {
+			case ocispec.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest:
+				for i := range children {
+					c := &children[i]
+					if !images.IsLayerType(c.MediaType) {
+						continue
+					}
+					if _, ok := c.Annotations[targetURLsLabel]; !ok { // nop if this key is already set
+						c.Annotations[targetURLsLabel] = appendWithValidation(targetURLsLabel, c.URLs)
+					}
+
+					if _, ok := c.Annotations[config.TargetPrefetchSizeLabel]; !ok { // nop if this key is already set
+						c.Annotations[config.TargetPrefetchSizeLabel] = fmt.Sprintf("%d", prefetchSize)
+					}
+
+					// Store URLs of the neighbouring layer as well.
+					nlayers, ok := c.Annotations[targetImageLayersLabelContainerd]
+					if !ok {
+						continue
+					}
+					for j, dstr := range strings.Split(nlayers, ",") {
+						d, err := digest.Parse(dstr)
+						if err != nil {
+							return nil, err
+						}
+						l, ok := layerFromDigest(children, d)
+						if !ok {
+							continue
+						}
+						urlsKey := targetImageURLsLabelPrefix + fmt.Sprintf("%d", j)
+						if _, ok := c.Annotations[urlsKey]; !ok { // nop if this key is already set
+							c.Annotations[urlsKey] = appendWithValidation(urlsKey, l.URLs)
+						}
+					}
+				}
+			}
+			return children, nil
+		})
+	}
+}
+
+func layerFromDigest(layers []ocispec.Descriptor, target digest.Digest) (ocispec.Descriptor, bool) {
+	for _, l := range layers {
+		if l.Digest == target {
+			return l, images.IsLayerType(l.MediaType)
+		}
+	}
+	return ocispec.Descriptor{}, false
+}

fusemanager/api/api.pb.go

@@ -0,0 +1,566 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: api.proto
+
+package api
+
+import (
+	context "context"
+	fmt "fmt"
+	proto "github.com/gogo/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type StatusRequest struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *StatusRequest) Reset()         { *m = StatusRequest{} }
+func (m *StatusRequest) String() string { return proto.CompactTextString(m) }
+func (*StatusRequest) ProtoMessage()    {}
+func (*StatusRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{0}
+}
+func (m *StatusRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_StatusRequest.Unmarshal(m, b)
+}
+func (m *StatusRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_StatusRequest.Marshal(b, m, deterministic)
+}
+func (m *StatusRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_StatusRequest.Merge(m, src)
+}
+func (m *StatusRequest) XXX_Size() int {
+	return xxx_messageInfo_StatusRequest.Size(m)
+}
+func (m *StatusRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_StatusRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_StatusRequest proto.InternalMessageInfo
+
+type InitRequest struct {
+	Root                 string   `protobuf:"bytes,1,opt,name=root,proto3" json:"root,omitempty"`
+	Config               []byte   `protobuf:"bytes,2,opt,name=config,proto3" json:"config,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *InitRequest) Reset()         { *m = InitRequest{} }
+func (m *InitRequest) String() string { return proto.CompactTextString(m) }
+func (*InitRequest) ProtoMessage()    {}
+func (*InitRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{1}
+}
+func (m *InitRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_InitRequest.Unmarshal(m, b)
+}
+func (m *InitRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_InitRequest.Marshal(b, m, deterministic)
+}
+func (m *InitRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_InitRequest.Merge(m, src)
+}
+func (m *InitRequest) XXX_Size() int {
+	return xxx_messageInfo_InitRequest.Size(m)
+}
+func (m *InitRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_InitRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_InitRequest proto.InternalMessageInfo
+
+func (m *InitRequest) GetRoot() string {
+	if m != nil {
+		return m.Root
+	}
+	return ""
+}
+
+func (m *InitRequest) GetConfig() []byte {
+	if m != nil {
+		return m.Config
+	}
+	return nil
+}
+
+type MountRequest struct {
+	Mountpoint           string            `protobuf:"bytes,1,opt,name=mountpoint,proto3" json:"mountpoint,omitempty"`
+	Labels               map[string]string `protobuf:"bytes,2,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
+}
+
+func (m *MountRequest) Reset()         { *m = MountRequest{} }
+func (m *MountRequest) String() string { return proto.CompactTextString(m) }
+func (*MountRequest) ProtoMessage()    {}
+func (*MountRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{2}
+}
+func (m *MountRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_MountRequest.Unmarshal(m, b)
+}
+func (m *MountRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_MountRequest.Marshal(b, m, deterministic)
+}
+func (m *MountRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_MountRequest.Merge(m, src)
+}
+func (m *MountRequest) XXX_Size() int {
+	return xxx_messageInfo_MountRequest.Size(m)
+}
+func (m *MountRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_MountRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_MountRequest proto.InternalMessageInfo
+
+func (m *MountRequest) GetMountpoint() string {
+	if m != nil {
+		return m.Mountpoint
+	}
+	return ""
+}
+
+func (m *MountRequest) GetLabels() map[string]string {
+	if m != nil {
+		return m.Labels
+	}
+	return nil
+}
+
+type CheckRequest struct {
+	Mountpoint           string            `protobuf:"bytes,1,opt,name=mountpoint,proto3" json:"mountpoint,omitempty"`
+	Labels               map[string]string `protobuf:"bytes,2,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
+}
+
+func (m *CheckRequest) Reset()         { *m = CheckRequest{} }
+func (m *CheckRequest) String() string { return proto.CompactTextString(m) }
+func (*CheckRequest) ProtoMessage()    {}
+func (*CheckRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{3}
+}
+func (m *CheckRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_CheckRequest.Unmarshal(m, b)
+}
+func (m *CheckRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_CheckRequest.Marshal(b, m, deterministic)
+}
+func (m *CheckRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_CheckRequest.Merge(m, src)
+}
+func (m *CheckRequest) XXX_Size() int {
+	return xxx_messageInfo_CheckRequest.Size(m)
+}
+func (m *CheckRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_CheckRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CheckRequest proto.InternalMessageInfo
+
+func (m *CheckRequest) GetMountpoint() string {
+	if m != nil {
+		return m.Mountpoint
+	}
+	return ""
+}
+
+func (m *CheckRequest) GetLabels() map[string]string {
+	if m != nil {
+		return m.Labels
+	}
+	return nil
+}
+
+type UnmountRequest struct {
+	Mountpoint           string   `protobuf:"bytes,1,opt,name=mountpoint,proto3" json:"mountpoint,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *UnmountRequest) Reset()         { *m = UnmountRequest{} }
+func (m *UnmountRequest) String() string { return proto.CompactTextString(m) }
+func (*UnmountRequest) ProtoMessage()    {}
+func (*UnmountRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{4}
+}
+func (m *UnmountRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_UnmountRequest.Unmarshal(m, b)
+}
+func (m *UnmountRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_UnmountRequest.Marshal(b, m, deterministic)
+}
+func (m *UnmountRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_UnmountRequest.Merge(m, src)
+}
+func (m *UnmountRequest) XXX_Size() int {
+	return xxx_messageInfo_UnmountRequest.Size(m)
+}
+func (m *UnmountRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_UnmountRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UnmountRequest proto.InternalMessageInfo
+
+func (m *UnmountRequest) GetMountpoint() string {
+	if m != nil {
+		return m.Mountpoint
+	}
+	return ""
+}
+
+type StatusResponse struct {
+	Status               int32    `protobuf:"varint,1,opt,name=status,proto3" json:"status,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *StatusResponse) Reset()         { *m = StatusResponse{} }
+func (m *StatusResponse) String() string { return proto.CompactTextString(m) }
+func (*StatusResponse) ProtoMessage()    {}
+func (*StatusResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{5}
+}
+func (m *StatusResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_StatusResponse.Unmarshal(m, b)
+}
+func (m *StatusResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_StatusResponse.Marshal(b, m, deterministic)
+}
+func (m *StatusResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_StatusResponse.Merge(m, src)
+}
+func (m *StatusResponse) XXX_Size() int {
+	return xxx_messageInfo_StatusResponse.Size(m)
+}
+func (m *StatusResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_StatusResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_StatusResponse proto.InternalMessageInfo
+
+func (m *StatusResponse) GetStatus() int32 {
+	if m != nil {
+		return m.Status
+	}
+	return 0
+}
+
+type Response struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Response) Reset()         { *m = Response{} }
+func (m *Response) String() string { return proto.CompactTextString(m) }
+func (*Response) ProtoMessage()    {}
+func (*Response) Descriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{6}
+}
+func (m *Response) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Response.Unmarshal(m, b)
+}
+func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Response.Marshal(b, m, deterministic)
+}
+func (m *Response) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Response.Merge(m, src)
+}
+func (m *Response) XXX_Size() int {
+	return xxx_messageInfo_Response.Size(m)
+}
+func (m *Response) XXX_DiscardUnknown() {
+	xxx_messageInfo_Response.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Response proto.InternalMessageInfo
+
+func init() {
+	proto.RegisterType((*StatusRequest)(nil), "fusemanager.StatusRequest")
+	proto.RegisterType((*InitRequest)(nil), "fusemanager.InitRequest")
+	proto.RegisterType((*MountRequest)(nil), "fusemanager.MountRequest")
+	proto.RegisterMapType((map[string]string)(nil), "fusemanager.MountRequest.LabelsEntry")
+	proto.RegisterType((*CheckRequest)(nil), "fusemanager.CheckRequest")
+	proto.RegisterMapType((map[string]string)(nil), "fusemanager.CheckRequest.LabelsEntry")
+	proto.RegisterType((*UnmountRequest)(nil), "fusemanager.UnmountRequest")
+	proto.RegisterType((*StatusResponse)(nil), "fusemanager.StatusResponse")
+	proto.RegisterType((*Response)(nil), "fusemanager.Response")
+}
+
+func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) }
+
+var fileDescriptor_00212fb1f9d3bf1c = []byte{
+	// 386 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x53, 0x51, 0x4b, 0xf3, 0x30,
+	0x14, 0xa5, 0xdd, 0xd6, 0xef, 0xdb, 0xed, 0x9c, 0x12, 0x54, 0x6a, 0x05, 0x19, 0x05, 0xa1, 0x2f,
+	0x6b, 0x65, 0x3e, 0xe8, 0x84, 0x3d, 0xa8, 0x28, 0x08, 0xee, 0xa5, 0xc3, 0x17, 0xdf, 0xb2, 0x92,
+	0x75, 0x65, 0x6b, 0x52, 0x9b, 0x74, 0x30, 0x7f, 0x91, 0xff, 0xc5, 0x3f, 0x25, 0xcd, 0xba, 0x91,
+	0x8a, 0x13, 0x84, 0xbd, 0xe5, 0x24, 0xf7, 0xdc, 0x9e, 0x7b, 0xcf, 0x29, 0x34, 0x71, 0x1a, 0x7b,
+	0x69, 0xc6, 0x04, 0x43, 0xe6, 0x24, 0xe7, 0x24, 0xc1, 0x14, 0x47, 0x24, 0x73, 0xf6, 0x61, 0x6f,
+	0x24, 0xb0, 0xc8, 0x79, 0x40, 0xde, 0x72, 0xc2, 0x85, 0xd3, 0x07, 0xf3, 0x89, 0xc6, 0xa2, 0x84,
+	0x08, 0x41, 0x3d, 0x63, 0x4c, 0x58, 0x5a, 0x47, 0x73, 0x9b, 0x81, 0x3c, 0xa3, 0x63, 0x30, 0x42,
+	0x46, 0x27, 0x71, 0x64, 0xe9, 0x1d, 0xcd, 0x6d, 0x05, 0x25, 0x72, 0x3e, 0x34, 0x68, 0x0d, 0x59,
+	0x4e, 0x37, 0xe4, 0x33, 0x80, 0xa4, 0xc0, 0x29, 0x8b, 0xe9, 0xba, 0x85, 0x72, 0x83, 0x06, 0x60,
+	0xcc, 0xf1, 0x98, 0xcc, 0xb9, 0xa5, 0x77, 0x6a, 0xae, 0xd9, 0x3b, 0xf7, 0x14, 0x69, 0x9e, 0xda,
+	0xca, 0x7b, 0x96, 0x75, 0x0f, 0x54, 0x64, 0xcb, 0xa0, 0x24, 0xd9, 0x7d, 0x30, 0x95, 0x6b, 0x74,
+	0x00, 0xb5, 0x19, 0x59, 0x96, 0x9f, 0x29, 0x8e, 0xe8, 0x10, 0x1a, 0x0b, 0x3c, 0xcf, 0x89, 0xd4,
+	0xd9, 0x0c, 0x56, 0xe0, 0x46, 0xbf, 0xd6, 0xa4, 0xd4, 0xfb, 0x29, 0x09, 0x67, 0xbb, 0x91, 0xaa,
+	0xb6, 0xda, 0xb5, 0xd4, 0x0b, 0x68, 0xbf, 0xd0, 0xe4, 0x0f, 0x6b, 0x75, 0x5c, 0x68, 0xaf, 0x3d,
+	0xe5, 0x29, 0xa3, 0x9c, 0x14, 0x8e, 0x71, 0x79, 0x23, 0xab, 0x1b, 0x41, 0x89, 0x1c, 0x80, 0xff,
+	0xeb, 0x9a, 0xde, 0xa7, 0x0e, 0xd6, 0x48, 0xe0, 0x2c, 0x7a, 0x7f, 0xcc, 0x39, 0x19, 0xae, 0x26,
+	0x1b, 0x91, 0x6c, 0x11, 0x87, 0x04, 0xdd, 0x82, 0xb1, 0x6a, 0x89, 0xec, 0xca, 0xe0, 0x95, 0xec,
+	0xd8, 0xa7, 0x3f, 0xbe, 0x95, 0x1a, 0xae, 0xa0, 0x5e, 0x04, 0x0b, 0x59, 0x95, 0x22, 0x25, 0x6b,
+	0xf6, 0x51, 0xe5, 0x65, 0x43, 0xec, 0x43, 0x43, 0x46, 0x01, 0x9d, 0x6c, 0x8d, 0xc7, 0x2f, 0x54,
+	0x69, 0xcd, 0x37, 0xaa, 0x6a, 0xd7, 0x36, 0xea, 0x00, 0xfe, 0x95, 0x6b, 0x47, 0xd5, 0xb1, 0xaa,
+	0x66, 0x6c, 0xa1, 0xdf, 0xf9, 0xaf, 0xdd, 0x28, 0x16, 0xd3, 0x7c, 0xec, 0x85, 0x2c, 0xf1, 0xb9,
+	0xdc, 0x6b, 0x97, 0x53, 0x9c, 0xf2, 0x29, 0x13, 0x82, 0x64, 0xbe, 0xc2, 0xf2, 0x71, 0x1a, 0x8f,
+	0x0d, 0xf9, 0x73, 0x5e, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff, 0x9d, 0x24, 0xe1, 0x41, 0xa9, 0x03,
+	0x00, 0x00,
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// StargzFuseManagerServiceClient is the client API for StargzFuseManagerService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type StargzFuseManagerServiceClient interface {
+	Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error)
+	Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Response, error)
+	Mount(ctx context.Context, in *MountRequest, opts ...grpc.CallOption) (*Response, error)
+	Check(ctx context.Context, in *CheckRequest, opts ...grpc.CallOption) (*Response, error)
+	Unmount(ctx context.Context, in *UnmountRequest, opts ...grpc.CallOption) (*Response, error)
+}
+
+type stargzFuseManagerServiceClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewStargzFuseManagerServiceClient(cc *grpc.ClientConn) StargzFuseManagerServiceClient {
+	return &stargzFuseManagerServiceClient{cc}
+}
+
+func (c *stargzFuseManagerServiceClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) {
+	out := new(StatusResponse)
+	err := c.cc.Invoke(ctx, "/fusemanager.StargzFuseManagerService/Status", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *stargzFuseManagerServiceClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*Response, error) {
+	out := new(Response)
+	err := c.cc.Invoke(ctx, "/fusemanager.StargzFuseManagerService/Init", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *stargzFuseManagerServiceClient) Mount(ctx context.Context, in *MountRequest, opts ...grpc.CallOption) (*Response, error) {
+	out := new(Response)
+	err := c.cc.Invoke(ctx, "/fusemanager.StargzFuseManagerService/Mount", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *stargzFuseManagerServiceClient) Check(ctx context.Context, in *CheckRequest, opts ...grpc.CallOption) (*Response, error) {
+	out := new(Response)
+	err := c.cc.Invoke(ctx, "/fusemanager.StargzFuseManagerService/Check", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *stargzFuseManagerServiceClient) Unmount(ctx context.Context, in *UnmountRequest, opts ...grpc.CallOption) (*Response, error) {
+	out := new(Response)
+	err := c.cc.Invoke(ctx, "/fusemanager.StargzFuseManagerService/Unmount", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// StargzFuseManagerServiceServer is the server API for StargzFuseManagerService service.
+type StargzFuseManagerServiceServer interface {
+	Status(context.Context, *StatusRequest) (*StatusResponse, error)
+	Init(context.Context, *InitRequest) (*Response, error)
+	Mount(context.Context, *MountRequest) (*Response, error)
+	Check(context.Context, *CheckRequest) (*Response, error)
+	Unmount(context.Context, *UnmountRequest) (*Response, error)
+}
+
+// UnimplementedStargzFuseManagerServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedStargzFuseManagerServiceServer struct {
+}
+
+func (*UnimplementedStargzFuseManagerServiceServer) Status(ctx context.Context, req *StatusRequest) (*StatusResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Status not implemented")
+}
+func (*UnimplementedStargzFuseManagerServiceServer) Init(ctx context.Context, req *InitRequest) (*Response, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Init not implemented")
+}
+func (*UnimplementedStargzFuseManagerServiceServer) Mount(ctx context.Context, req *MountRequest) (*Response, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Mount not implemented")
+}
+func (*UnimplementedStargzFuseManagerServiceServer) Check(ctx context.Context, req *CheckRequest) (*Response, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Check not implemented")
+}
+func (*UnimplementedStargzFuseManagerServiceServer) Unmount(ctx context.Context, req *UnmountRequest) (*Response, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Unmount not implemented")
+}
+
+func RegisterStargzFuseManagerServiceServer(s *grpc.Server, srv StargzFuseManagerServiceServer) {
+	s.RegisterService(&_StargzFuseManagerService_serviceDesc, srv)
+}
+
+func _StargzFuseManagerService_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(StatusRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(StargzFuseManagerServiceServer).Status(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/fusemanager.StargzFuseManagerService/Status",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(StargzFuseManagerServiceServer).Status(ctx, req.(*StatusRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _StargzFuseManagerService_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(InitRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(StargzFuseManagerServiceServer).Init(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/fusemanager.StargzFuseManagerService/Init",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(StargzFuseManagerServiceServer).Init(ctx, req.(*InitRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _StargzFuseManagerService_Mount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(MountRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(StargzFuseManagerServiceServer).Mount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/fusemanager.StargzFuseManagerService/Mount",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(StargzFuseManagerServiceServer).Mount(ctx, req.(*MountRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _StargzFuseManagerService_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CheckRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(StargzFuseManagerServiceServer).Check(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/fusemanager.StargzFuseManagerService/Check",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(StargzFuseManagerServiceServer).Check(ctx, req.(*CheckRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _StargzFuseManagerService_Unmount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(UnmountRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(StargzFuseManagerServiceServer).Unmount(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/fusemanager.StargzFuseManagerService/Unmount",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(StargzFuseManagerServiceServer).Unmount(ctx, req.(*UnmountRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _StargzFuseManagerService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "fusemanager.StargzFuseManagerService",
+	HandlerType: (*StargzFuseManagerServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Status",
+			Handler:    _StargzFuseManagerService_Status_Handler,
+		},
+		{
+			MethodName: "Init",
+			Handler:    _StargzFuseManagerService_Init_Handler,
+		},
+		{
+			MethodName: "Mount",
+			Handler:    _StargzFuseManagerService_Mount_Handler,
+		},
+		{
+			MethodName: "Check",
+			Handler:    _StargzFuseManagerService_Check_Handler,
+		},
+		{
+			MethodName: "Unmount",
+			Handler:    _StargzFuseManagerService_Unmount_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "api.proto",
+}

fusemanager/api/api.proto

@@ -0,0 +1,58 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+syntax = "proto3";
+
+option go_package = "github.com/stargz-snapshotter/fusemanager/api";
+
+package fusemanager;
+
+service StargzFuseManagerService {
+    rpc Status (StatusRequest) returns (StatusResponse);
+    rpc Init (InitRequest) returns (Response);
+    rpc Mount (MountRequest) returns (Response);
+    rpc Check (CheckRequest) returns (Response);
+    rpc Unmount (UnmountRequest) returns (Response);
+}
+
+message StatusRequest {
+}
+
+message InitRequest {
+    string root = 1;
+    bytes config = 2;
+}
+
+message MountRequest {
+    string mountpoint = 1;
+    map<string, string> labels = 2;
+}
+
+message CheckRequest {
+    string mountpoint = 1;
+    map<string, string> labels = 2;
+}
+
+message UnmountRequest {
+    string mountpoint = 1;
+}
+
+message StatusResponse {
+    int32 status = 1;
+}
+
+message Response {
+}

fusemanager/api/generate.go

@@ -0,0 +1,19 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package api
+
+//go:generate protoc --gogo_out=paths=source_relative,plugins=grpc:. api.proto

fusemanager/client.go

@@ -0,0 +1,141 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fusemanager
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/containerd/containerd/v2/defaults"
+	"github.com/containerd/containerd/v2/pkg/dialer"
+	"github.com/containerd/log"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/credentials/insecure"
+
+	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
+	"github.com/containerd/stargz-snapshotter/snapshot"
+)
+
+type Client struct {
+	client pb.StargzFuseManagerServiceClient
+}
+
+func NewManagerClient(ctx context.Context, root, socket string, config *Config) (snapshot.FileSystem, error) {
+	grpcCli, err := newClient(socket)
+	if err != nil {
+		return nil, err
+	}
+
+	client := &Client{
+		client: grpcCli,
+	}
+
+	err = client.init(ctx, root, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}
+
+func newClient(socket string) (pb.StargzFuseManagerServiceClient, error) {
+	connParams := grpc.ConnectParams{
+		Backoff: backoff.DefaultConfig,
+	}
+	gopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithConnectParams(connParams),
+		grpc.WithContextDialer(dialer.ContextDialer),
+		grpc.WithDefaultCallOptions(
+			grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize),
+			grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize),
+		),
+	}
+
+	conn, err := grpc.NewClient(fmt.Sprintf("unix://%s", socket), gopts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return pb.NewStargzFuseManagerServiceClient(conn), nil
+}
+
+func (cli *Client) init(ctx context.Context, root string, config *Config) error {
+	configBytes, err := json.Marshal(config)
+	if err != nil {
+		return err
+	}
+
+	req := &pb.InitRequest{
+		Root:   root,
+		Config: configBytes,
+	}
+
+	_, err = cli.client.Init(ctx, req)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to call Init")
+		return err
+	}
+
+	return nil
+}
+
+func (cli *Client) Mount(ctx context.Context, mountpoint string, labels map[string]string) error {
+	req := &pb.MountRequest{
+		Mountpoint: mountpoint,
+		Labels:     labels,
+	}
+
+	_, err := cli.client.Mount(ctx, req)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to call Mount")
+		return err
+	}
+
+	return nil
+}
+
+func (cli *Client) Check(ctx context.Context, mountpoint string, labels map[string]string) error {
+	req := &pb.CheckRequest{
+		Mountpoint: mountpoint,
+		Labels:     labels,
+	}
+
+	_, err := cli.client.Check(ctx, req)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to call Check")
+		return err
+	}
+
+	return nil
+}
+
+func (cli *Client) Unmount(ctx context.Context, mountpoint string) error {
+	req := &pb.UnmountRequest{
+		Mountpoint: mountpoint,
+	}
+
+	_, err := cli.client.Unmount(ctx, req)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to call Unmount")
+		return err
+	}
+
+	return nil
+}

fusemanager/fusemanager.go

@@ -0,0 +1,259 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fusemanager
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	golog "log"
+	"net"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+
+	"github.com/containerd/log"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+	"google.golang.org/grpc"
+
+	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
+	"github.com/containerd/stargz-snapshotter/version"
+)
+
+var (
+	debugFlag     bool
+	versionFlag   bool
+	fuseStoreAddr string
+	address       string
+	logLevel      string
+	logPath       string
+	action        string
+)
+
+func parseFlags() {
+	flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs")
+	flag.BoolVar(&versionFlag, "v", false, "show the fusemanager version and exit")
+	flag.StringVar(&action, "action", "", "action of fusemanager")
+	flag.StringVar(&fuseStoreAddr, "fusestore-path", "/var/lib/containerd-stargz-grpc/fusestore.db", "address for the fusemanager's store")
+	flag.StringVar(&address, "address", "/run/containerd-stargz-grpc/fuse-manager.sock", "address for the fusemanager's gRPC socket")
+	flag.StringVar(&logLevel, "log-level", logrus.InfoLevel.String(), "set the logging level [trace, debug, info, warn, error, fatal, panic]")
+	flag.StringVar(&logPath, "log-path", "", "path to fusemanager's logs, no log recorded if empty")
+
+	flag.Parse()
+}
+
+func Run() {
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "failed to run fusemanager: %v", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	parseFlags()
+	if versionFlag {
+		fmt.Printf("%s:\n", os.Args[0])
+		fmt.Println("  Version: ", version.Version)
+		fmt.Println("  Revision:", version.Revision)
+		fmt.Println("")
+		return nil
+	}
+
+	if fuseStoreAddr == "" || address == "" {
+		return fmt.Errorf("fusemanager fusestore and socket path cannot be empty")
+	}
+
+	ctx := log.WithLogger(context.Background(), log.L)
+
+	switch action {
+	case "start":
+		return startNew(ctx, logPath, address, fuseStoreAddr, logLevel)
+	default:
+		return runFuseManager(ctx)
+	}
+}
+
+func startNew(ctx context.Context, logPath, address, fusestore, logLevel string) error {
+	self, err := os.Executable()
+	if err != nil {
+		return err
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-address", address,
+		"-fusestore-path", fusestore,
+		"-log-level", logLevel,
+	}
+
+	// we use shim-like approach to start new fusemanager process by self-invoking in the background
+	// and detach it from parent
+	cmd := exec.CommandContext(ctx, self, args...)
+	cmd.Dir = cwd
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+
+	if logPath != "" {
+		err := os.Remove(logPath)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		file, err := os.Create(logPath)
+		if err != nil {
+			return err
+		}
+		cmd.Stdout = file
+		cmd.Stderr = file
+	}
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	go cmd.Wait()
+
+	if ready, err := waitUntilReady(ctx); err != nil || !ready {
+		if err != nil {
+			return fmt.Errorf("failed to start new fusemanager: %w", err)
+		}
+		if !ready {
+			return fmt.Errorf("failed to start new fusemanager, fusemanager not ready")
+		}
+	}
+
+	return nil
+}
+
+// waitUntilReady waits until fusemanager is ready to accept requests
+func waitUntilReady(ctx context.Context) (bool, error) {
+	grpcCli, err := newClient(address)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := grpcCli.Status(ctx, &pb.StatusRequest{})
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to call Status")
+		return false, err
+	}
+
+	if resp.Status == FuseManagerNotReady {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func runFuseManager(ctx context.Context) error {
+	lvl, err := logrus.ParseLevel(logLevel)
+	if err != nil {
+		return fmt.Errorf("failed to prepare logger: %w", err)
+	}
+
+	logrus.SetLevel(lvl)
+	logrus.SetFormatter(&logrus.JSONFormatter{
+		TimestampFormat: log.RFC3339NanoFixed,
+	})
+
+	golog.SetOutput(log.G(ctx).WriterLevel(logrus.DebugLevel))
+
+	// Prepare the directory for the socket
+	if err := os.MkdirAll(filepath.Dir(address), 0700); err != nil {
+		return fmt.Errorf("failed to create directory %s: %w", filepath.Dir(address), err)
+	}
+
+	// Try to remove the socket file to avoid EADDRINUSE
+	if err := os.Remove(address); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to remove old socket file: %w", err)
+	}
+
+	l, err := net.Listen("unix", address)
+	if err != nil {
+		return fmt.Errorf("failed to listen socket: %w", err)
+	}
+
+	server := grpc.NewServer()
+	fm, err := NewFuseManager(ctx, l, server, fuseStoreAddr, address)
+	if err != nil {
+		return fmt.Errorf("failed to configure manager server: %w", err)
+	}
+
+	pb.RegisterStargzFuseManagerServiceServer(server, fm)
+
+	errCh := make(chan error, 1)
+	go func() {
+		if err := server.Serve(l); err != nil {
+			errCh <- fmt.Errorf("error on serving via socket %q: %w", address, err)
+		}
+	}()
+
+	var s os.Signal
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, unix.SIGINT, unix.SIGTERM)
+	select {
+	case s = <-sigCh:
+		log.G(ctx).Infof("Got %v", s)
+	case err := <-errCh:
+		log.G(ctx).WithError(err).Warnf("error during running the server")
+	}
+
+	server.Stop()
+	if err = fm.Close(ctx); err != nil {
+		return fmt.Errorf("failed to close fuse manager: %w", err)
+	}
+
+	return nil
+}
+
+func StartFuseManager(ctx context.Context, executable, address, fusestore, logLevel, logPath string) (newlyStarted bool, err error) {
+	// if socket exists, do not start it
+	if _, err := os.Stat(address); err == nil {
+		return false, nil
+	} else if !os.IsNotExist(err) {
+		return false, err
+	}
+
+	if _, err := os.Stat(executable); err != nil {
+		return false, fmt.Errorf("failed to stat fusemanager binary: %q", executable)
+	}
+
+	args := []string{
+		"-action", "start",
+		"-address", address,
+		"-fusestore-path", fusestore,
+		"-log-level", logLevel,
+		"-log-path", logPath,
+	}
+
+	cmd := exec.Command(executable, args...)
+	if err := cmd.Start(); err != nil {
+		return false, err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}

fusemanager/fusemanager_test.go

@@ -0,0 +1,235 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fusemanager
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+
+	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
+	"github.com/containerd/stargz-snapshotter/service"
+	"google.golang.org/grpc"
+)
+
+// mockFileSystem implements snapshot.FileSystem for testing
+type mockFileSystem struct {
+	t             *testing.T
+	mountErr      error
+	checkErr      error
+	unmountErr    error
+	mountPoints   map[string]bool
+	checkCalled   bool
+	mountCalled   bool
+	unmountCalled bool
+}
+
+func newMockFileSystem(t *testing.T) *mockFileSystem {
+	return &mockFileSystem{
+		t:           t,
+		mountPoints: make(map[string]bool),
+	}
+}
+
+func (fs *mockFileSystem) Mount(ctx context.Context, mountpoint string, labels map[string]string) error {
+	fs.mountCalled = true
+	if fs.mountErr != nil {
+		return fs.mountErr
+	}
+	fs.mountPoints[mountpoint] = true
+	return nil
+}
+
+func (fs *mockFileSystem) Check(ctx context.Context, mountpoint string, labels map[string]string) error {
+	fs.checkCalled = true
+	if fs.checkErr != nil {
+		return fs.checkErr
+	}
+	if _, ok := fs.mountPoints[mountpoint]; !ok {
+		return fmt.Errorf("mountpoint %s not found", mountpoint)
+	}
+	return nil
+}
+
+func (fs *mockFileSystem) Unmount(ctx context.Context, mountpoint string) error {
+	fs.unmountCalled = true
+	if fs.unmountErr != nil {
+		return fs.unmountErr
+	}
+	delete(fs.mountPoints, mountpoint)
+	return nil
+}
+
+// mockServer embeds Server struct and overrides Init method
+type mockServer struct {
+	*Server
+	initCalled bool
+	initErr    error
+}
+
+func newMockServer(ctx context.Context, listener net.Listener, server *grpc.Server, fuseStoreAddr, serverAddr string) (*mockServer, error) {
+	s, err := NewFuseManager(ctx, listener, server, fuseStoreAddr, serverAddr)
+	if err != nil {
+		return nil, err
+	}
+	return &mockServer{Server: s}, nil
+}
+
+// Init overrides Server.Init to avoid actual initialization
+func (s *mockServer) Init(ctx context.Context, req *pb.InitRequest) (*pb.Response, error) {
+	s.initCalled = true
+	if s.initErr != nil {
+		return nil, s.initErr
+	}
+
+	// Set only required fields
+	s.root = req.Root
+	config := &Config{}
+	if err := json.Unmarshal(req.Config, config); err != nil {
+		return nil, err
+	}
+	s.config = config
+	s.status = FuseManagerReady
+
+	return &pb.Response{}, nil
+}
+
+func TestFuseManager(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "fusemanager-test")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	socketPath := filepath.Join(tmpDir, "test.sock")
+	fuseStorePath := filepath.Join(tmpDir, "fusestore.db")
+	fuseManagerSocketPath := filepath.Join(tmpDir, "test-fusemanager.sock")
+
+	l, err := net.Listen("unix", socketPath)
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	defer l.Close()
+
+	// Create server with mock
+	grpcServer := grpc.NewServer()
+	mockFs := newMockFileSystem(t)
+	fm, err := newMockServer(context.Background(), l, grpcServer, fuseStorePath, fuseManagerSocketPath)
+	if err != nil {
+		t.Fatalf("failed to create fuse manager: %v", err)
+	}
+	defer fm.Close(context.Background())
+
+	pb.RegisterStargzFuseManagerServiceServer(grpcServer, fm)
+
+	// Set mock filesystem
+	fm.curFs = mockFs
+
+	go grpcServer.Serve(l)
+	defer grpcServer.Stop()
+
+	// Test cases to verify Init, Mount, Check and Unmount operations
+	testCases := []struct {
+		name       string
+		mountpoint string
+		labels     map[string]string
+		initErr    error
+		mountErr   error
+		checkErr   error
+		unmountErr error
+		wantErr    bool
+	}{
+		{
+			name:       "successful init and mount",
+			mountpoint: filepath.Join(tmpDir, "mount1"),
+			labels:     map[string]string{"key": "value"},
+		},
+		{
+			name:       "init error",
+			mountpoint: filepath.Join(tmpDir, "mount2"),
+			initErr:    fmt.Errorf("init error"),
+			wantErr:    true,
+		},
+		{
+			name:       "mount error",
+			mountpoint: filepath.Join(tmpDir, "mount3"),
+			mountErr:   fmt.Errorf("mount error"),
+			wantErr:    true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			mockFs.mountErr = tc.mountErr
+			mockFs.checkErr = tc.checkErr
+			mockFs.unmountErr = tc.unmountErr
+			mockFs.mountCalled = false
+			mockFs.checkCalled = false
+			mockFs.unmountCalled = false
+			fm.initErr = tc.initErr
+			fm.initCalled = false
+
+			config := &Config{
+				Config: service.Config{},
+			}
+			client, err := NewManagerClient(context.Background(), tmpDir, socketPath, config)
+			if err != nil {
+				if !tc.wantErr {
+					t.Fatalf("failed to create client: %v", err)
+				}
+				return
+			}
+
+			if !fm.initCalled {
+				t.Error("Init() was not called")
+			}
+
+			if !tc.wantErr {
+				// Test Mount
+				err = client.Mount(context.Background(), tc.mountpoint, tc.labels)
+				if err != nil {
+					t.Errorf("Mount() error = %v", err)
+				}
+				if !mockFs.mountCalled {
+					t.Error("Mount() was not called on filesystem")
+				}
+
+				// Test Check
+				err = client.Check(context.Background(), tc.mountpoint, tc.labels)
+				if err != nil {
+					t.Errorf("Check() error = %v", err)
+				}
+				if !mockFs.checkCalled {
+					t.Error("Check() was not called on filesystem")
+				}
+
+				// Test Unmount
+				err = client.Unmount(context.Background(), tc.mountpoint)
+				if err != nil {
+					t.Errorf("Unmount() error = %v", err)
+				}
+				if !mockFs.unmountCalled {
+					t.Error("Unmount() was not called on filesystem")
+				}
+			}
+		})
+	}
+}

fusemanager/fusestore.go

@@ -0,0 +1,99 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fusemanager
+
+import (
+	"context"
+	"encoding/json"
+
+	bolt "go.etcd.io/bbolt"
+
+	"github.com/containerd/stargz-snapshotter/service"
+)
+
+var (
+	fuseInfoBucket = []byte("fuse-info-bucket")
+)
+
+type fuseInfo struct {
+	Root       string
+	Mountpoint string
+	Labels     map[string]string
+	Config     service.Config
+}
+
+func (fm *Server) storeFuseInfo(fuseInfo *fuseInfo) error {
+	return fm.ms.Update(func(tx *bolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists(fuseInfoBucket)
+		if err != nil {
+			return err
+		}
+
+		key := []byte(fuseInfo.Mountpoint)
+
+		val, err := json.Marshal(fuseInfo)
+		if err != nil {
+			return err
+		}
+
+		err = bucket.Put(key, val)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func (fm *Server) removeFuseInfo(fuseInfo *fuseInfo) error {
+	return fm.ms.Update(func(tx *bolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists(fuseInfoBucket)
+		if err != nil {
+			return err
+		}
+
+		key := []byte(fuseInfo.Mountpoint)
+
+		err = bucket.Delete(key)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+// restoreFuseInfo restores fuseInfo when Init is called, it will skip mounted
+// layers whose mountpoint can be found in fsMap
+func (fm *Server) restoreFuseInfo(ctx context.Context) error {
+	return fm.ms.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(fuseInfoBucket)
+		if bucket == nil {
+			return nil
+		}
+
+		return bucket.ForEach(func(_, v []byte) error {
+			mi := &fuseInfo{}
+			err := json.Unmarshal(v, mi)
+			if err != nil {
+				return err
+			}
+
+			return fm.mount(ctx, mi.Mountpoint, mi.Labels)
+		})
+	})
+}

fusemanager/service.go

@@ -0,0 +1,358 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fusemanager
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/containerd/log"
+	"github.com/moby/sys/mountinfo"
+	bolt "go.etcd.io/bbolt"
+	"google.golang.org/grpc"
+
+	pb "github.com/containerd/stargz-snapshotter/fusemanager/api"
+	"github.com/containerd/stargz-snapshotter/service"
+	"github.com/containerd/stargz-snapshotter/snapshot"
+)
+
+const (
+	FuseManagerNotReady = iota
+	FuseManagerWaitInit
+	FuseManagerReady
+)
+
+type Config struct {
+	Config                     service.Config
+	IPFS                       bool   `toml:"ipfs" json:"ipfs"`
+	MetadataStore              string `toml:"metadata_store" default:"memory" json:"metadata_store"`
+	DefaultImageServiceAddress string `json:"default_image_service_address"`
+}
+
+type ConfigContext struct {
+	Ctx        context.Context
+	Config     *Config
+	RootDir    string
+	Server     *grpc.Server
+	OpenBoltDB func(string) (*bolt.DB, error)
+	Address    string
+	CRIServer  *grpc.Server
+}
+
+var (
+	configFuncs []ConfigFunc
+	configMu    sync.Mutex
+)
+
+type ConfigFunc func(cc *ConfigContext) ([]service.Option, error)
+
+func RegisterConfigFunc(f ConfigFunc) {
+	configMu.Lock()
+	defer configMu.Unlock()
+	configFuncs = append(configFuncs, f)
+}
+
+// Opens bolt DB with avoiding opening the same DB multiple times
+type dbOpener struct {
+	mu      sync.Mutex
+	handles map[string]*bolt.DB
+}
+
+func (o *dbOpener) openBoltDB(p string) (*bolt.DB, error) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+
+	if db, ok := o.handles[p]; ok && db != nil {
+		// we opened it before. avoid trying to open this again.
+		return db, nil
+	}
+
+	db, err := bolt.Open(p, 0600, &bolt.Options{
+		NoFreelistSync:  true,
+		InitialMmapSize: 64 * 1024 * 1024,
+		FreelistType:    bolt.FreelistMapType,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if o.handles == nil {
+		o.handles = make(map[string]*bolt.DB)
+	}
+	o.handles[p] = db
+	return db, nil
+}
+
+type Server struct {
+	pb.UnimplementedStargzFuseManagerServiceServer
+
+	lock   sync.RWMutex
+	status int32
+
+	listener net.Listener
+	server   *grpc.Server
+
+	// root is the latest root passed from containerd-stargz-grpc
+	root string
+	// config is the latest config passed from containerd-stargz-grpc
+	config *Config
+	// fsMap maps mountpoint to its filesystem instance to ensure Mount/Check/Unmount
+	// call the proper filesystem
+	fsMap sync.Map
+	// curFs is filesystem created by latest config
+	curFs snapshot.FileSystem
+	ms    *bolt.DB
+
+	fuseStoreAddr string
+
+	dbOpener *dbOpener
+
+	serverAddr string
+
+	curCRIServer *grpc.Server
+}
+
+func NewFuseManager(ctx context.Context, listener net.Listener, server *grpc.Server, fuseStoreAddr string, serverAddr string) (*Server, error) {
+	if err := os.MkdirAll(filepath.Dir(fuseStoreAddr), 0700); err != nil {
+		return nil, fmt.Errorf("failed to create directory %q: %w", filepath.Dir(fuseStoreAddr), err)
+	}
+
+	db, err := bolt.Open(fuseStoreAddr, 0666, &bolt.Options{Timeout: 10 * time.Second, ReadOnly: false})
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure fusestore: %w", err)
+	}
+
+	fm := &Server{
+		status:        FuseManagerWaitInit,
+		lock:          sync.RWMutex{},
+		fsMap:         sync.Map{},
+		ms:            db,
+		listener:      listener,
+		server:        server,
+		fuseStoreAddr: fuseStoreAddr,
+		dbOpener:      &dbOpener{},
+		serverAddr:    serverAddr,
+	}
+
+	return fm, nil
+}
+
+func (fm *Server) Status(ctx context.Context, _ *pb.StatusRequest) (*pb.StatusResponse, error) {
+	fm.lock.RLock()
+	defer fm.lock.RUnlock()
+
+	return &pb.StatusResponse{
+		Status: fm.status,
+	}, nil
+}
+
+func (fm *Server) Init(ctx context.Context, req *pb.InitRequest) (*pb.Response, error) {
+	fm.lock.Lock()
+	fm.status = FuseManagerWaitInit
+	defer func() {
+		fm.status = FuseManagerReady
+		fm.lock.Unlock()
+	}()
+
+	ctx = log.WithLogger(ctx, log.G(ctx))
+
+	config := &Config{}
+	err := json.Unmarshal(req.Config, config)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to get config")
+		return &pb.Response{}, err
+	}
+	fm.root = req.Root
+	fm.config = config
+
+	if fm.curCRIServer != nil {
+		fm.curCRIServer.Stop()
+		fm.curCRIServer = nil
+	}
+
+	cc := &ConfigContext{
+		Ctx:        ctx,
+		Config:     fm.config,
+		RootDir:    fm.root,
+		Server:     fm.server,
+		OpenBoltDB: fm.dbOpener.openBoltDB,
+		Address:    fm.serverAddr,
+	}
+
+	var opts []service.Option
+	for _, configFunc := range configFuncs {
+		funcOpts, err := configFunc(cc)
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("failed to apply config function")
+			return &pb.Response{}, err
+		}
+		opts = append(opts, funcOpts...)
+	}
+
+	fm.curCRIServer = cc.CRIServer
+
+	fs, err := service.NewFileSystem(ctx, fm.root, &fm.config.Config, opts...)
+	if err != nil {
+		return &pb.Response{}, err
+	}
+	fm.curFs = fs
+
+	err = fm.restoreFuseInfo(ctx)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to restore fuse info")
+		return &pb.Response{}, err
+	}
+
+	return &pb.Response{}, nil
+}
+
+func (fm *Server) Mount(ctx context.Context, req *pb.MountRequest) (*pb.Response, error) {
+	fm.lock.RLock()
+	defer fm.lock.RUnlock()
+	if fm.status != FuseManagerReady {
+		return &pb.Response{}, fmt.Errorf("fuse manager not ready")
+	}
+
+	ctx = log.WithLogger(ctx, log.G(ctx).WithField("mountpoint", req.Mountpoint))
+
+	err := fm.mount(ctx, req.Mountpoint, req.Labels)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to mount stargz")
+		return &pb.Response{}, err
+	}
+
+	fm.storeFuseInfo(&fuseInfo{
+		Root:       fm.root,
+		Mountpoint: req.Mountpoint,
+		Labels:     req.Labels,
+		Config:     fm.config.Config,
+	})
+
+	return &pb.Response{}, nil
+}
+
+func (fm *Server) Check(ctx context.Context, req *pb.CheckRequest) (*pb.Response, error) {
+	fm.lock.RLock()
+	defer fm.lock.RUnlock()
+	if fm.status != FuseManagerReady {
+		return &pb.Response{}, fmt.Errorf("fuse manager not ready")
+	}
+
+	ctx = log.WithLogger(ctx, log.G(ctx).WithField("mountpoint", req.Mountpoint))
+
+	obj, found := fm.fsMap.Load(req.Mountpoint)
+	if !found {
+		err := fmt.Errorf("failed to find filesystem of mountpoint %s", req.Mountpoint)
+		log.G(ctx).WithError(err).Errorf("failed to check filesystem")
+		return &pb.Response{}, err
+	}
+
+	fs := obj.(snapshot.FileSystem)
+	err := fs.Check(ctx, req.Mountpoint, req.Labels)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to check filesystem")
+		return &pb.Response{}, err
+	}
+
+	return &pb.Response{}, nil
+}
+
+func (fm *Server) Unmount(ctx context.Context, req *pb.UnmountRequest) (*pb.Response, error) {
+	fm.lock.RLock()
+	defer fm.lock.RUnlock()
+	if fm.status != FuseManagerReady {
+		return &pb.Response{}, fmt.Errorf("fuse manager not ready")
+	}
+
+	ctx = log.WithLogger(ctx, log.G(ctx).WithField("mountpoint", req.Mountpoint))
+
+	obj, found := fm.fsMap.Load(req.Mountpoint)
+	if !found {
+		// check whether already unmounted
+		mounts, err := mountinfo.GetMounts(func(info *mountinfo.Info) (skip, stop bool) {
+			if info.Mountpoint == req.Mountpoint {
+				return false, true
+			}
+			return true, false
+		})
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("failed to get mount info")
+			return &pb.Response{}, err
+		}
+
+		if len(mounts) <= 0 {
+			return &pb.Response{}, nil
+		}
+		err = fmt.Errorf("failed to find filesystem of mountpoint %s", req.Mountpoint)
+		log.G(ctx).WithError(err).Errorf("failed to unmount filesystem")
+		return &pb.Response{}, err
+	}
+
+	fs := obj.(snapshot.FileSystem)
+	err := fs.Unmount(ctx, req.Mountpoint)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to unmount filesystem")
+		return &pb.Response{}, err
+	}
+
+	fm.fsMap.Delete(req.Mountpoint)
+	fm.removeFuseInfo(&fuseInfo{
+		Mountpoint: req.Mountpoint,
+	})
+
+	return &pb.Response{}, nil
+}
+
+func (fm *Server) Close(ctx context.Context) error {
+	fm.lock.Lock()
+	defer fm.lock.Unlock()
+	fm.status = FuseManagerNotReady
+
+	err := fm.ms.Close()
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to close fusestore")
+		return err
+	}
+
+	if err := os.Remove(fm.fuseStoreAddr); err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to remove fusestore file %s", fm.fuseStoreAddr)
+		return err
+	}
+
+	return nil
+}
+
+func (fm *Server) mount(ctx context.Context, mountpoint string, labels map[string]string) error {
+	// mountpoint in fsMap means layer is already mounted, skip it
+	if _, found := fm.fsMap.Load(mountpoint); found {
+		return nil
+	}
+
+	err := fm.curFs.Mount(ctx, mountpoint, labels)
+	if err != nil {
+		log.G(ctx).WithError(err).Errorf("failed to mount stargz")
+		return err
+	}
+
+	fm.fsMap.Store(mountpoint, fm.curFs)
+	return nil
+}

go.mod

@@ -1,53 +1,138 @@
 module github.com/containerd/stargz-snapshotter
 
-go 1.16
+go 1.24.0
+
+toolchain go1.24.2
 
 require (
-	github.com/containerd/console v1.0.3
-	github.com/containerd/containerd v1.6.0-beta.1.0.20211101005050-f0d3ea96cf8c
-	github.com/containerd/continuity v0.2.1
-	github.com/containerd/stargz-snapshotter/estargz v0.10.0
-	github.com/docker/cli v20.10.10+incompatible
-	github.com/docker/docker v20.10.7+incompatible // indirect
-	github.com/docker/docker-credential-helpers v0.6.4 // indirect
+	github.com/containerd/console v1.0.5
+	github.com/containerd/containerd/v2 v2.1.4
+	github.com/containerd/continuity v0.4.5
+	github.com/containerd/errdefs v1.0.0
+	github.com/containerd/log v0.1.0
+	github.com/containerd/platforms v1.0.0-rc.1
+	github.com/containerd/plugin v1.0.0
+	github.com/containerd/stargz-snapshotter/estargz v0.17.0
+	github.com/distribution/reference v0.6.0
+	github.com/docker/cli v28.3.3+incompatible
 	github.com/docker/go-metrics v0.0.1
-	github.com/goccy/go-json v0.7.10
+	github.com/gogo/protobuf v1.3.2
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
-	github.com/hanwen/go-fuse/v2 v2.1.1-0.20210825171523-3ab5d95a30ae
-	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/go-retryablehttp v0.7.0
-	github.com/klauspost/compress v1.13.6
-	github.com/moby/sys/mountinfo v0.4.2-0.20211022201527-95edfa939201
+	github.com/hanwen/go-fuse/v2 v2.8.0
+	github.com/hashicorp/go-retryablehttp v0.7.8
+	github.com/klauspost/compress v1.18.0
+	github.com/moby/sys/mountinfo v0.7.2
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
-	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
-	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.11.0
-	github.com/rs/xid v1.3.0
-	github.com/sirupsen/logrus v1.8.1
-	go.etcd.io/bbolt v1.3.6
-	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
-	google.golang.org/grpc v1.42.0
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	k8s.io/api v0.22.3
-	k8s.io/apimachinery v0.22.3
-	k8s.io/client-go v0.22.3
-	k8s.io/cri-api v0.22.3
+	github.com/opencontainers/image-spec v1.1.1
+	github.com/opencontainers/runtime-spec v1.2.1
+	github.com/prometheus/client_golang v1.23.0
+	github.com/rs/xid v1.6.0
+	github.com/sirupsen/logrus v1.9.3
+	go.etcd.io/bbolt v1.4.2
+	golang.org/x/sync v0.16.0
+	golang.org/x/sys v0.34.0
+	google.golang.org/grpc v1.74.2
+	k8s.io/api v0.33.3
+	k8s.io/apimachinery v0.33.3
+	k8s.io/client-go v0.33.3
+	k8s.io/cri-api v0.33.3
 )
 
-replace (
-	// Import local package for estargz.
-	github.com/containerd/stargz-snapshotter/estargz => ./estargz
-
-	// Temporary fork for avoiding importing patent-protected code: https://github.com/hashicorp/golang-lru/issues/73
-	github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c
-
-	// NOTE1: github.com/containerd/containerd v1.4.0 depends on github.com/urfave/cli v1.22.1
-	//        because of https://github.com/urfave/cli/issues/1092
-	// NOTE2: Automatic upgrade of this is disabled in denendabot.yml. When we remove this replace
-	//        directive, we must remove the corresponding "ignore" configuration from dependabot.yml
-	github.com/urfave/cli => github.com/urfave/cli v1.22.1
+require (
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.13.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/cgroups/v3 v3.0.5 // indirect
+	github.com/containerd/containerd/api v1.9.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/go-cni v1.1.13 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
+	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/containernetworking/cni v1.3.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/selinux v1.12.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sasha-s/go-deadlock v0.3.5 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/stretchr/testify v1.10.0 // indirect
+	github.com/urfave/cli/v2 v2.27.7 // indirect
+	github.com/vbatts/tar-split v0.12.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools/v3 v3.5.0 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)
+
+// Import local package for estargz.
+replace github.com/containerd/stargz-snapshotter/estargz => ./estargz
+
+exclude (
+	// These dependencies were updated to "master" in some modules we depend on,
+	// but have no code-changes since their last release. Unfortunately, this also
+	// causes a ripple effect, forcing all users of the containerd module to also
+	// update these dependencies to an unrelease / un-tagged version.
+	//
+	// Both these dependencies will unlikely do a new release in the near future,
+	// so exclude these versions so that we can downgrade to the current release.
+	//
+	// For additional details, see this PR and links mentioned in that PR:
+	// https://github.com/kubernetes-sigs/kustomize/pull/5830#issuecomment-2569960859
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 )

ipfs/client/client.go

@@ -0,0 +1,228 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mitchellh/go-homedir"
+	ma "github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
+)
+
+// Client is an IPFS API client.
+type Client struct {
+	// Address is URL of IPFS API to connect to.
+	Address string
+
+	// Client is http client to use for connecting to IPFS API
+	Client *http.Client
+}
+
+// New creates a new IPFS API client of the specified address.
+func New(ipfsAPIAddress string) *Client {
+	return &Client{Address: ipfsAPIAddress, Client: http.DefaultClient}
+}
+
+// FileInfo represents the information provided by "/api/v0/files/stat" API of IPFS.
+// Please see details at: https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-files-stat
+type FileInfo struct {
+	Blocks         int    `json:"Blocks"`
+	CumulativeSize uint64 `json:"CumulativeSize"`
+	Hash           string `json:"Hash"`
+	Local          bool   `json:"Local"`
+	Size           uint64 `json:"Size"`
+	SizeLocal      uint64 `json:"SizeLocal"`
+	Type           string `json:"Type"`
+	WithLocality   bool   `json:"WithLocality"`
+}
+
+// StatCID gets and returns information of the file specified by the cid.
+func (c *Client) StatCID(cid string) (info *FileInfo, retErr error) {
+	if c.Address == "" {
+		return nil, fmt.Errorf("specify IPFS API address")
+	}
+	client := c.Client
+	if client == nil {
+		client = http.DefaultClient
+	}
+	ipfsAPIFilesStat := c.Address + "/api/v0/files/stat"
+	req, err := http.NewRequest("POST", ipfsAPIFilesStat, nil)
+	if err != nil {
+		return nil, err
+	}
+	q := req.URL.Query()
+	q.Add("arg", "/ipfs/"+cid)
+	req.URL.RawQuery = q.Encode()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if retErr != nil {
+			io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+		}
+	}()
+	if resp.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("failed to stat %v; status code: %v", cid, resp.StatusCode)
+	}
+	var rs FileInfo
+	if err := json.NewDecoder(resp.Body).Decode(&rs); err != nil {
+		return nil, err
+	}
+	return &rs, nil
+}
+
+// Get get the reader of the data specified by the IPFS path and optionally with
+// the offset and length.
+func (c *Client) Get(p string, offset *int, length *int) (_ io.ReadCloser, retErr error) {
+	if c.Address == "" {
+		return nil, fmt.Errorf("specify IPFS API address")
+	}
+	client := c.Client
+	if client == nil {
+		client = http.DefaultClient
+	}
+	ipfsAPICat := c.Address + "/api/v0/cat"
+	req, err := http.NewRequest("POST", ipfsAPICat, nil)
+	if err != nil {
+		return nil, err
+	}
+	q := req.URL.Query()
+	q.Add("arg", p)
+	if offset != nil {
+		q.Add("offset", fmt.Sprintf("%d", *offset))
+	}
+	if length != nil {
+		q.Add("length", fmt.Sprintf("%d", *length))
+	}
+	req.URL.RawQuery = q.Encode()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if retErr != nil {
+			io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+		}
+	}()
+	if resp.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("failed to cat %v; status code: %v", p, resp.StatusCode)
+	}
+	return resp.Body, nil
+}
+
+// Add adds the provided data to IPFS and returns its CID (v1).
+func (c *Client) Add(r io.Reader) (cidv1 string, retErr error) {
+	if c.Address == "" {
+		return "", fmt.Errorf("specify IPFS API address")
+	}
+	client := c.Client
+	if client == nil {
+		client = http.DefaultClient
+	}
+	ipfsAPIAdd := c.Address + "/api/v0/add"
+	pr, pw := io.Pipe()
+	mw := multipart.NewWriter(pw)
+	contentType := mw.FormDataContentType()
+	go func() {
+		fw, err := mw.CreateFormFile("file", "file")
+		if err != nil {
+			pw.CloseWithError(err)
+			return
+		}
+		if _, err := io.Copy(fw, r); err != nil {
+			pw.CloseWithError(err)
+			return
+		}
+		if err := mw.Close(); err != nil {
+			pw.CloseWithError(err)
+			return
+		}
+		pw.Close()
+	}()
+	req, err := http.NewRequest("POST", ipfsAPIAdd, pr)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Add("Content-Type", contentType)
+	q := req.URL.Query()
+	q.Add("cid-version", "1")
+	q.Add("pin", "true")
+	req.URL.RawQuery = q.Encode()
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		io.Copy(io.Discard, resp.Body)
+		resp.Body.Close()
+	}()
+	if resp.StatusCode/100 != 2 {
+		return "", fmt.Errorf("failed to add; status code: %v", resp.StatusCode)
+	}
+	var rs struct {
+		Hash string `json:"Hash"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&rs); err != nil {
+		return "", err
+	}
+	if rs.Hash == "" {
+		return "", fmt.Errorf("got empty hash")
+	}
+	return rs.Hash, nil
+}
+
+// GetIPFSAPIAddress get IPFS API URL from the specified IPFS repository.
+// If ipfsPath == "", then it's default is "~/.ipfs".
+// This is compatible to IPFS client behaviour: https://github.com/ipfs/go-ipfs-http-client/blob/171fcd55e3b743c38fb9d78a34a3a703ee0b5e89/api.go#L69-L81
+func GetIPFSAPIAddress(ipfsPath string, scheme string) (string, error) {
+	if ipfsPath == "" {
+		ipfsPath = "~/.ipfs"
+	}
+	baseDir, err := homedir.Expand(ipfsPath)
+	if err != nil {
+		return "", err
+	}
+	api, err := os.ReadFile(filepath.Join(baseDir, "api"))
+	if err != nil {
+		return "", err
+	}
+	a, err := ma.NewMultiaddr(strings.TrimSpace(string(api)))
+	if err != nil {
+		return "", err
+	}
+	_, iurl, err := manet.DialArgs(a)
+	if err != nil {
+		return "", err
+	}
+	iurl = scheme + "://" + iurl
+	if _, err := url.Parse(iurl); err != nil {
+		return "", err
+	}
+	return iurl, nil
+}

ipfs/client/client_test.go

@@ -0,0 +1,77 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// This test should run via "make test-ipfs".
+
+package client
+
+import (
+	"bytes"
+	"flag"
+	"io"
+	"testing"
+)
+
+var ipfsAPI string
+
+func init() {
+	flag.StringVar(&ipfsAPI, "ipfs-api", "", "Address of IPFS API")
+}
+
+func TestIPFSClient(t *testing.T) {
+	if ipfsAPI == "" {
+		t.Log("Specify IPFS API address for IPFS client tests")
+		t.Skip()
+		return
+	}
+	t.Logf("IPFS API address: %q", ipfsAPI)
+	c := New(ipfsAPI)
+	sampleString := "hello world 0123456789"
+	d := bytes.NewReader([]byte(sampleString))
+	cid, err := c.Add(d)
+	if err != nil {
+		t.Errorf("failed to add data to IPFS: %v", err)
+		return
+	}
+	checkData(t, c, cid, 0, len(sampleString), sampleString, len(sampleString))
+	checkData(t, c, cid, 10, 4, sampleString[10:14], len(sampleString))
+}
+
+func checkData(t *testing.T, c *Client, cid string, off, len int, wantData string, allSize int) {
+	st, err := c.StatCID(cid)
+	if err != nil {
+		t.Errorf("failed to stat data from IPFS: %v", err)
+		return
+	}
+	if st.Size != uint64(allSize) {
+		t.Errorf("unexpected size got from IPFS %v; wanted %v", st.Size, allSize)
+		return
+	}
+	dGotR, err := c.Get("/ipfs/"+cid, &off, &len)
+	if err != nil {
+		t.Errorf("failed to get data from IPFS: %v", err)
+		return
+	}
+	dGot, err := io.ReadAll(dGotR)
+	if err != nil {
+		t.Errorf("failed to read data from IPFS: %v", err)
+		return
+	}
+	if string(dGot) != wantData {
+		t.Errorf("unexpected data got from IPFS %q; wanted %q", string(dGot), wantData)
+		return
+	}
+}

ipfs/converter.go

@@ -17,48 +17,63 @@
 package ipfs
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"strings"
 
-	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/images/converter"
-	"github.com/containerd/containerd/platforms"
-	"github.com/ipfs/go-cid"
-	files "github.com/ipfs/go-ipfs-files"
-	iface "github.com/ipfs/interface-go-ipfs-core"
-	"github.com/ipfs/interface-go-ipfs-core/options"
-	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/containerd/platforms"
+	ipfsclient "github.com/containerd/stargz-snapshotter/ipfs/client"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // Push pushes the provided image ref to IPFS with converting it to IPFS-enabled format.
-func Push(ctx context.Context, client *containerd.Client, api iface.CoreAPI, ref string, layerConvert converter.ConvertFunc, platformMC platforms.MatchComparer) (ipath.Resolved, error) {
+func Push(ctx context.Context, client *containerd.Client, ref string, layerConvert converter.ConvertFunc, platformMC platforms.MatchComparer) (cidV1 string, _ error) {
+	return PushWithIPFSPath(ctx, client, ref, layerConvert, platformMC, nil)
+}
+
+func PushWithIPFSPath(ctx context.Context, client *containerd.Client, ref string, layerConvert converter.ConvertFunc, platformMC platforms.MatchComparer, ipfsPath *string) (cidV1 string, _ error) {
 	ctx, done, err := client.WithLease(ctx)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	defer done(ctx)
 	img, err := client.ImageService().Get(ctx, ref)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
+	var ipath string
+	if idir := os.Getenv("IPFS_PATH"); idir != "" {
+		ipath = idir
+	}
+	if ipfsPath != nil {
+		ipath = *ipfsPath
+	}
+	// HTTP is only supported as of now. We can add https support here if needed (e.g. for connecting to it via proxy, etc)
+	iurl, err := ipfsclient.GetIPFSAPIAddress(ipath, "http")
+	if err != nil {
+		return "", err
+	}
+	iclient := ipfsclient.New(iurl)
 	desc, err := converter.IndexConvertFuncWithHook(layerConvert, true, platformMC, converter.ConvertHooks{
-		PostConvertHook: pushBlobHook(api),
+		PostConvertHook: pushBlobHook(iclient),
 	})(ctx, client.ContentStore(), img.Target)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	root, err := json.Marshal(desc)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
-	return api.Unixfs().Add(ctx, files.NewBytesFile(root), options.Unixfs.Pin(true), options.Unixfs.CidVersion(1))
+	return iclient.Add(bytes.NewReader(root))
 }
 
-func pushBlobHook(api iface.CoreAPI) converter.ConvertHookFunc {
+func pushBlobHook(client *ipfsclient.Client) converter.ConvertHookFunc {
 	return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor, newDesc *ocispec.Descriptor) (*ocispec.Descriptor, error) {
 		resultDesc := newDesc
 		if resultDesc == nil {
@@ -69,29 +84,20 @@ func pushBlobHook(api iface.CoreAPI) converter.ConvertHookFunc {
 		if err != nil {
 			return nil, err
 		}
-		p, err := api.Unixfs().Add(ctx, files.NewReaderFile(content.NewReader(ra)), options.Unixfs.Pin(true), options.Unixfs.CidVersion(1))
+		cidv1, err := client.Add(content.NewReader(ra))
 		if err != nil {
 			return nil, err
 		}
-		// record IPFS URL using CIDv1 : https://docs.ipfs.io/how-to/address-ipfs-on-web/#native-urls
-		if p.Cid().Version() == 0 {
-			return nil, fmt.Errorf("CID verions 0 isn't supported")
-		}
-		resultDesc.URLs = []string{"ipfs://" + p.Cid().String()}
+		resultDesc.URLs = []string{"ipfs://" + cidv1}
 		return resultDesc, nil
 	}
 }
 
-func GetPath(desc ocispec.Descriptor) (ipath.Path, error) {
+func GetCID(desc ocispec.Descriptor) (string, error) {
 	for _, u := range desc.URLs {
 		if strings.HasPrefix(u, "ipfs://") {
-			// support only content addressable URL (ipfs://<CID>)
-			c, err := cid.Decode(u[7:])
-			if err != nil {
-				return nil, err
-			}
-			return ipath.IpfsPath(c), nil
+			return u[7:], nil
 		}
 	}
-	return nil, fmt.Errorf("no CID is recorded")
+	return "", fmt.Errorf("no CID is recorded")
 }

ipfs/go.mod

@@ -1,16 +1,73 @@
 module github.com/containerd/stargz-snapshotter/ipfs
 
-go 1.16
+go 1.23.0
+
+toolchain go1.24.1
 
 require (
-	github.com/containerd/containerd v1.6.0-beta.1.0.20211101005050-f0d3ea96cf8c
-	github.com/ipfs/go-cid v0.1.0
-	github.com/ipfs/go-ipfs-files v0.0.9
-	github.com/ipfs/interface-go-ipfs-core v0.5.2
-	github.com/libp2p/go-libp2p-record v0.1.1 // indirect
-	github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
-	github.com/pkg/errors v0.9.1
+	github.com/containerd/containerd/v2 v2.1.4
+	github.com/containerd/platforms v1.0.0-rc.1
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/multiformats/go-multiaddr v0.16.1
+	github.com/opencontainers/image-spec v1.1.1
 )
 
-// Temporary fork for avoiding importing patent-protected code: https://github.com/hashicorp/golang-lru/issues/73
-replace github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c
+require (
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.13.0 // indirect
+	github.com/containerd/cgroups/v3 v3.0.5 // indirect
+	github.com/containerd/containerd/api v1.9.0 // indirect
+	github.com/containerd/continuity v0.4.5 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/plugin v1.0.0 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
+	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/ipfs/go-cid v0.0.7 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/runtime-spec v1.2.1 // indirect
+	github.com/opencontainers/selinux v1.12.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
+	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
+	google.golang.org/grpc v1.74.2 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
+)

ipfs/resolver.go

@@ -21,57 +21,64 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
 	"path"
 
-	"github.com/containerd/containerd/remotes"
-	"github.com/ipfs/go-cid"
-	files "github.com/ipfs/go-ipfs-files"
-	iface "github.com/ipfs/interface-go-ipfs-core"
-	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/containerd/containerd/v2/core/remotes"
+	ipfsclient "github.com/containerd/stargz-snapshotter/ipfs/client"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 )
 
 type resolver struct {
-	api    iface.CoreAPI
 	scheme string
+	client *ipfsclient.Client
 }
 
 type ResolverOptions struct {
 	// Scheme is the scheme to fetch the specified IPFS content. "ipfs" or "ipns".
 	Scheme string
+
+	// IPFSPath is the path to the IPFS repository directory.
+	IPFSPath string
 }
 
-func NewResolver(client iface.CoreAPI, options ResolverOptions) (remotes.Resolver, error) {
+func NewResolver(options ResolverOptions) (remotes.Resolver, error) {
 	s := options.Scheme
 	if s != "ipfs" && s != "ipns" {
 		return nil, fmt.Errorf("unsupported scheme %q", s)
 	}
-	return &resolver{client, s}, nil
+	var ipath string
+	if idir := os.Getenv("IPFS_PATH"); idir != "" {
+		ipath = idir
+	}
+	if options.IPFSPath != "" {
+		ipath = options.IPFSPath
+	}
+	// HTTP is only supported as of now. We can add https support here if needed (e.g. for connecting to it via proxy, etc)
+	iurl, err := ipfsclient.GetIPFSAPIAddress(ipath, "http")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get IPFS URL from ipfs path")
+	}
+	return &resolver{
+		scheme: s,
+		client: ipfsclient.New(iurl),
+	}, nil
 }
 
 // Resolve resolves the provided ref for IPFS. ref must be a CID.
 // TODO: Allow specifying IPFS path or URL. This requires to modify `reference` pkg because
-//       it's incompatbile to the current reference specification.
+//
+//	it's incompatbile to the current reference specification.
 func (r *resolver) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) {
-	c, err := cid.Decode(ref)
+	rc, err := r.client.Get(path.Join("/", r.scheme, ref), nil, nil)
 	if err != nil {
 		return "", ocispec.Descriptor{}, err
 	}
-	p := ipath.New(path.Join("/", r.scheme, c.String()))
-	if err := p.IsValid(); err != nil {
-		return "", ocispec.Descriptor{}, err
-	}
-	n, err := r.api.Unixfs().Get(ctx, p)
-	if err != nil {
-		return "", ocispec.Descriptor{}, err
-	}
-	rc := files.ToFile(n)
 	defer rc.Close()
 	if err := json.NewDecoder(rc).Decode(&desc); err != nil {
 		return "", ocispec.Descriptor{}, err
 	}
-	if _, err := GetPath(desc); err != nil {
+	if _, err := GetCID(desc); err != nil {
 		return "", ocispec.Descriptor{}, err
 	}
 	return ref, desc, nil
@@ -90,13 +97,9 @@ type fetcher struct {
 }
 
 func (f *fetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
-	p, err := GetPath(desc)
+	cid, err := GetCID(desc)
 	if err != nil {
 		return nil, err
 	}
-	n, err := f.r.api.Unixfs().Get(ctx, p)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to get file %q", p.String())
-	}
-	return files.ToFile(n), nil
+	return f.r.client.Get(path.Join("/", f.r.scheme, cid), nil, nil)
 }