Merge pull request #2105 from containerd/dependabot/go_modules/containerd-2f3dabfc34

build(deps): bump the containerd group across 2 directories with 1 update

.dockerignore

@@ -1,2 +1 @@
-/.git
 /out

.github/dependabot.yml

@@ -0,0 +1,54 @@
+version: 2
+updates:
+
+  # Automatic upgrade for go modules.
+  - package-ecosystem: "gomod"
+    directories:
+      - "/estargz"
+      - "/ipfs"
+      - "/"
+      - "/cmd"
+    schedule:
+      interval: "daily"
+    ignore:
+      # We upgrade this manually on each release
+      - dependency-name: "github.com/containerd/stargz-snapshotter/estargz"
+    groups:
+      golang-x:
+        patterns:
+          - "golang.org/x/*"
+      google-golang:
+        patterns:
+          - "google.golang.org/*"
+      containerd:
+        patterns:
+          - "github.com/containerd/*"
+      opencontainers:
+        patterns:
+          - "github.com/opencontainers/*"
+      k8s:
+        patterns:
+          - "k8s.io/*"
+      gomod:
+        # this pattern covers all go dependencies that are not in
+        # the above groups. dependabot doesn't seem to update sub-modules if
+        # a dependency doesn't belong to a group, so we define this group
+        # explicitly.
+        exclude-patterns:
+          - "golang.org/x/*"
+          - "google.golang.org/*"
+          - "github.com/containerd/*"
+          - "github.com/opencontainers/*"
+          - "k8s.io/*"
+
+  # Automatic upgrade for base images used in the Dockerfile
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+  # Automatic upgrade for Github Actions
+  - package-ecosystem: "github-actions"
+    directory: "/" # means ".github/workflows"
+    schedule:
+      interval: "daily"

.github/workflows/benchmark.yml

@@ -1,28 +1,37 @@
 name: Benchmark
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 
 env:
   DOCKER_BUILDKIT: 1
 
 jobs:
   hello-bench:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     name: HelloBench
     env:
       BENCHMARK_LOG_DIR: ${{ github.workspace }}/log/
       BENCHMARK_RESULT_DIR: ${{ github.workspace }}/benchmark/
       BENCHMARK_REGISTRY: ghcr.io
       BENCHMARK_USER: stargz-containers
-      BENCHMARK_TARGETS: python:3.7 gcc:9.2.0 rethinkdb:2.3.6 glassfish:4.1-jdk8
+      BENCHMARK_TARGETS: python:3.10 gcc:11.2.0 postgres:14.2 tomcat:10.1.0-jdk17-openjdk-bullseye
       BENCHMARK_SAMPLES_NUM: 5
       BENCHMARK_PERCENTILE: 95
       BENCHMARK_PERCENTILES_GRANULARITY: 25
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        runtime: ["podman", "containerd"]
     steps:
     - name: Install tools
       run: |
-          sudo apt-get update && sudo apt-get --no-install-recommends install -y gnuplot
-          pip install numpy
-    - uses: actions/checkout@v2
+          sudo apt-get update && \
+          sudo apt-get install -y gnuplot python3-numpy
+    - uses: actions/checkout@v4
     - name: Prepare directories
       run: mkdir "${BENCHMARK_RESULT_DIR}" "${BENCHMARK_LOG_DIR}"
     - name: Get instance information
@@ -31,9 +40,11 @@ jobs:
           jq '{ location : .compute.location, vmSize : .compute.vmSize }' | \
           tee ${{ env.BENCHMARK_RESULT_DIR }}/instance.json
     - name: Run benchmark
+      env:
+        BENCHMARK_RUNTIME_MODE: ${{ matrix.runtime }}
       run: make benchmark
-    - uses: actions/upload-artifact@v1
+    - uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: benchmarking-result
+        name: benchmarking-result-${{ matrix.runtime }}
         path: ${{ env.BENCHMARK_RESULT_DIR }}

.github/workflows/kind-image.yml

@@ -0,0 +1,51 @@
+name: Kind image
+on:
+  push:
+    tags:
+      - 'v*'
+  pull_request:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  kind-image:
+    runs-on: ubuntu-24.04
+    name: Kind image
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}-kind
+
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v7.0.0-28
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        uses: docker/build-push-action@v6.18.0
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64

.github/workflows/nightly.yml

@@ -0,0 +1,107 @@
+name: Nightly
+on:
+  schedule:
+    - cron: '0 0 * * *' # Every day at midnight
+  pull_request:
+    paths:
+      - '.github/workflows/nightly.yml'
+
+# This nightly test helps us to track changes on containerd on daily basis
+# and enable us to quickly fix snapshotter when some of recent changes on
+# containerd cause incompatibility with this snapshotter.
+#
+# TODO1(ktock): Output binaries if needed.
+# TODO2(ktock): Ideally, this test should be invoked in containerd/containerd's CI on each PR.
+#               This will make sure that each commit merged into containerd/containerd safely
+#               works with stargz snapshotter.
+
+env:
+  DOCKER_BUILDKIT: 1
+  DOCKER_BUILD_ARGS: --build-arg=CONTAINERD_VERSION=main # do tests with the latest containerd
+
+jobs:
+  integration:
+    runs-on: ubuntu-24.04
+    name: Integration
+    steps:
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - uses: actions/checkout@v4
+    - name: Run integration test
+      run: make integration
+
+  test-optimize:
+    runs-on: ubuntu-24.04
+    name: Optimize
+    steps:
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - uses: actions/checkout@v4
+    - name: Run test for optimize subcommand of ctr-remote
+      run: make test-optimize
+
+  test-kind:
+    runs-on: ubuntu-24.04
+    name: Kind
+    steps:
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - uses: actions/checkout@v4
+    - name: Run test for pulling image from private registry on Kubernetes
+      run: make test-kind
+
+  test-criauth:
+    runs-on: ubuntu-24.04
+    name: CRIAuth
+    steps:
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - uses: actions/checkout@v4
+    - name: Run test for pulling image from private registry on Kubernetes
+      run: make test-criauth
+
+  test-cri-containerd:
+    runs-on: ubuntu-24.04
+    name: CRIValidationContainerd
+    steps:
+    - uses: actions/checkout@v4
+    - name: Varidate the runtime through CRI with containerd
+      run: make test-cri-containerd
+
+  test-cri-o:
+    runs-on: ubuntu-24.04
+    name: CRIValidationCRIO
+    steps:
+    - name: Install the latest docker
+      run: |
+        sudo apt-get remove moby-cli moby-engine
+        wget -O get-docker.sh https://get.docker.com
+        sh get-docker.sh
+    - uses: actions/checkout@v4
+    - name: Varidate the runtime through CRI with CRI-O
+      env:
+        DOCKER_BUILD_ARGS: "--build-arg=RUNC_VERSION=v1.0.3"
+      run: |
+        # needed to pass "runtime should output OOMKilled reason" test
+        sudo swapoff -a
+        make test-cri-o
+
+  test-k3s:
+    runs-on: ubuntu-24.04
+    name: K3S
+    steps:
+    - uses: actions/setup-go@v5
+      with:
+        go-version: '1.24.x'
+    - name: Install k3d
+      run: |
+        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.6.3/install.sh | bash
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - name: Install yq
+      run: |
+        sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64
+        sudo chmod +x /usr/local/bin/yq
+    - uses: actions/checkout@v4
+    - name: Run test with k3s
+      run: make test-k3s

.github/workflows/release.yml

@@ -9,32 +9,61 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
-    name: Release
+    runs-on: ubuntu-24.04
+    name: Build
+    strategy:
+      matrix:
+        arch: ["amd64", "arm-v7", "arm64", "ppc64le", "s390x"]
     env:
       OUTPUT_DIR: ${{ github.workspace }}/out
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build Binary
       env:
         DOCKER_BUILDKIT: 1
       run: |
         mkdir ${OUTPUT_DIR}
         RELEASE_TAG="${GITHUB_REF##*/}"
-        TAR_FILE_NAME="stargz-snapshotter-${RELEASE_TAG}-linux-amd64.tar.gz"
+        ARCH_ID="${{ matrix.arch }}"
+        BUILD_ARGS=--build-arg=TARGETARCH=${ARCH_ID}
+        if [ "${ARCH_ID}" == "arm-v7" ] ; then
+          BUILD_ARGS="--build-arg=TARGETARCH=arm --build-arg=GOARM=7"
+        fi
+        # make binaries static
+        BUILD_ARGS="$BUILD_ARGS --build-arg=CGO_ENABLED=0"
+        TAR_FILE_NAME="stargz-snapshotter-${RELEASE_TAG}-linux-${ARCH_ID}.tar.gz"
         SHA256SUM_FILE_NAME="${TAR_FILE_NAME}.sha256sum"
-        docker build --target release-binaries -o - . | gzip > "${OUTPUT_DIR}/${TAR_FILE_NAME}"
+        docker build ${BUILD_ARGS} --target release-binaries -o - . | gzip > "${OUTPUT_DIR}/${TAR_FILE_NAME}"
         ( cd ${OUTPUT_DIR}; sha256sum ${TAR_FILE_NAME} ) > "${OUTPUT_DIR}/${SHA256SUM_FILE_NAME}"
+    - name: Save Binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: builds-${{ matrix.arch }}
+        path: ${{ env.OUTPUT_DIR }}/*
+
+  release:
+    runs-on: ubuntu-24.04
+    name: Release
+    needs: [build]
+    env:
+      OUTPUT_DIR: ${{ github.workspace }}/builds
+    steps:
+    - uses: actions/checkout@v4
+    - name: Download Builds
+      uses: actions/download-artifact@v5
+      with:
+        path: ${{ env.OUTPUT_DIR }}
     - name: Create Release
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
         RELEASE_TAG="${GITHUB_REF##*/}"
         cat <<EOF > ${GITHUB_WORKSPACE}/release-note.txt
-        ${RELEASE_TAG}
-
         (TBD)
         EOF
-        ASSET_FLAGS=()
-        for F in ${OUTPUT_DIR}/*; do ASSET_FLAGS+=("-a" "$F"); done
-        hub release create "${ASSET_FLAGS[@]}" -F ${GITHUB_WORKSPACE}/release-note.txt --draft "${RELEASE_TAG}"
+        ASSET_ARGS=()
+        ls -al ${OUTPUT_DIR}/
+        for A in "amd64" "arm-v7" "arm64" "ppc64le" "s390x" ; do
+          ASSET_ARGS+=("${OUTPUT_DIR}/builds-${A}/*")
+        done
+        gh release create -F ${GITHUB_WORKSPACE}/release-note.txt --draft --title "${RELEASE_TAG}" "${RELEASE_TAG}" ${ASSET_ARGS[@]}

.github/workflows/tests.yml

@@ -1,67 +1,307 @@
 name: Tests
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 
 env:
   DOCKER_BUILDKIT: 1
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     name: Build
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build all
       run: ./script/util/make.sh build -j2
+
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     name: Test
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Test all
       run: ./script/util/make.sh test-all -j2
+
   linter:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     name: Linter
+    strategy:
+      fail-fast: false
+      matrix:
+        targetdir: [".", "./estargz", "./cmd", "./ipfs"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: '0'
-    - name: Run Linter
-      run: ./script/util/make.sh install-check-tools check
+    - uses: actions/setup-go@v5
+      with:
+        go-version: '1.24.x'
+    - name: golangci-lint
+      uses: golangci/golangci-lint-action@v8.0.0
+      with:
+        version: v2.1
+        args: --verbose --timeout=10m
+        working-directory: ${{ matrix.targetdir }}
+
   integration:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     name: Integration
+    strategy:
+      fail-fast: false
+      matrix:
+        buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version
+        builtin: ["true", "false"]
+        metadata-store: ["memory", "db"]
+        fuse-passthrough: ["true", "false"]
+        fuse-manager: ["true", "false"]
+        transfer-service: ["true", "false"]
+        exclude:
+        - buildargs: ""
+          builtin: "true"
+        - metadata-store: "db"
+          builtin: "true"
+        - metadata-store: "db"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - fuse-passthrough: "true"
+          builtin: "true"
+        - fuse-passthrough: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - fuse-passthrough: "true"
+          metadata-store: "db"
+        - fuse-manager: "true"
+          builtin: "true"
+        - fuse-manager: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - transfer-service: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - transfer-service: "true"
+          builtin: "true"
+        - transfer-service: "true"
+          metadata-store: "db"
+        - transfer-service: "true"
+          fuse-passthrough: "true"
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run integration test
+      env:
+        DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
+        BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
+        METADATA_STORE: ${{ matrix.metadata-store }}
+        FUSE_PASSTHROUGH: ${{ matrix.fuse-passthrough }}
+        FUSE_MANAGER: ${{ matrix.fuse-manager }}
+        TRANSFER_SERVICE: ${{ matrix.transfer-service }}
       run: make integration
+
   test-optimize:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     name: Optimize
+    strategy:
+      fail-fast: false
+      matrix:
+        buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test for optimize subcommand of ctr-remote
+      env:
+        DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
       run: make test-optimize
-  test-pullsecrets:
-    runs-on: ubuntu-18.04
-    name: PullSecrets
+
+  test-kind:
+    runs-on: ubuntu-24.04
+    name: Kind
+    strategy:
+      fail-fast: false
+      matrix:
+        buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version
+        builtin: ["true", "false"]
+        exclude:
+        - buildargs: ""
+          builtin: "true"
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run test for pulling image from private registry on Kubernetes
-      run: make test-pullsecrets
-  test-cri:
-    runs-on: ubuntu-18.04
-    name: CRIValidation
+      env:
+        DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
+        BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
+      run: make test-kind
+
+  test-criauth:
+    runs-on: ubuntu-24.04
+    name: CRIAuth
+    strategy:
+      fail-fast: false
+      matrix:
+        buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version
+        builtin: ["true", "false"]
+        exclude:
+        - buildargs: ""
+          builtin: "true"
     steps:
-    - uses: actions/checkout@v2
-    - name: Varidate the runtime through CRI
-      run: make test-cri
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - uses: actions/checkout@v4
+    - name: Run test for pulling image from private registry on Kubernetes with CRI keychain mode
+      env:
+        DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
+        BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
+      run: make test-criauth
+
+  test-cri-containerd:
+    runs-on: ubuntu-24.04
+    name: CRIValidationContainerd
+    strategy:
+      fail-fast: false
+      matrix:
+        buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version
+        builtin: ["true", "false"]
+        metadata-store: ["memory", "db"]
+        fuse-passthrough: ["true", "false"]
+        fuse-manager: ["true", "false"]
+        transfer-service: ["true", "false"]
+        exclude:
+        - buildargs: ""
+          builtin: "true"
+        - metadata-store: "db"
+          builtin: "true"
+        - metadata-store: "db"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - fuse-passthrough: "true"
+          builtin: "true"
+        - fuse-passthrough: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - fuse-passthrough: "true"
+          metadata-store: "db"
+        - fuse-manager: "true"
+          builtin: "true"
+        - fuse-manager: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - transfer-service: "true"
+          buildargs: "--build-arg=CONTAINERD_VERSION=main"
+        - transfer-service: "true"
+          builtin: "true"
+        - transfer-service: "true"
+          metadata-store: "db"
+        - transfer-service: "true"
+          fuse-passthrough: "true"
+    steps:
+    - uses: actions/checkout@v4
+    - name: Validate containerd through CRI
+      env:
+        DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
+        BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
+        METADATA_STORE: ${{ matrix.metadata-store }}
+        FUSE_PASSTHROUGH: ${{ matrix.fuse-passthrough }}
+        FUSE_MANAGER: ${{ matrix.fuse-manager }}
+        TRANSFER_SERVICE: ${{ matrix.transfer-service }}
+      run: make test-cri-containerd
+
+  test-cri-cri-o:
+    runs-on: ubuntu-24.04
+    name: CRIValidationCRIO
+    strategy:
+      fail-fast: false
+      matrix:
+        metadata-store: ["memory", "db"]
+    steps:
+    - name: Install the latest docker
+      run: |
+        sudo apt-get remove moby-cli moby-engine
+        wget -O get-docker.sh https://get.docker.com
+        sh get-docker.sh
+    - uses: actions/checkout@v4
+    - name: Validate CRI-O through CRI
+      env:
+        DOCKER_BUILD_ARGS: "--build-arg=RUNC_VERSION=v1.0.3"
+        METADATA_STORE: ${{ matrix.metadata-store }}
+      run: |
+        # needed to pass "runtime should output OOMKilled reason" test
+        sudo swapoff -a
+        make test-cri-o
+
+  test-podman:
+    runs-on: ubuntu-24.04
+    name: PodmanRootless
+    steps:
+    - uses: actions/checkout@v4
+    - name: Test Podman (rootless)
+      run: make test-podman
+
+  test-k3s:
+    runs-on: ubuntu-24.04
+    name: K3S
+    steps:
+    - uses: actions/setup-go@v5
+      with:
+        go-version: '1.24.x'
+    - name: Install k3d
+      run: |
+        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.6.3/install.sh | bash
+    - name: Install htpasswd for setting up private registry
+      run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
+    - name: Install yq
+      run: |
+        sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64
+        sudo chmod +x /usr/local/bin/yq
+    - uses: actions/checkout@v4
+    - name: Run test with k3s
+      run: make test-k3s
+
+  test-ipfs:
+    runs-on: ubuntu-24.04
+    name: IPFS
+    steps:
+    - uses: actions/checkout@v4
+    - name: Run test
+      run: make test-ipfs
+
+  test-k3s-argo-workflow:
+    runs-on: ubuntu-24.04
+    name: K3SArgoWorkflow
+    env:
+      RESULT_DIR: ${{ github.workspace }}/argo-workflow/
+    steps:
+    - uses: actions/setup-go@v5
+      with:
+        go-version: '1.24.x'
+    - name: Install k3d
+      run: |
+        wget -q -O - https://raw.githubusercontent.com/rancher/k3d/v5.6.3/install.sh | bash
+    - name: Install argo worklflow
+      run: |
+        wget -q https://github.com/argoproj/argo-workflows/releases/download/v3.0.10/argo-linux-amd64.gz
+        gunzip argo-linux-amd64.gz
+        sudo mv argo-linux-amd64 /usr/local/bin/argo
+        sudo chmod +x /usr/local/bin/argo
+    - name: Workaround for freeing up more disk space
+      # https://github.com/actions/runner-images/issues/2606
+      run: |
+        sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
+        sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
+    - uses: actions/checkout@v4
+    - name: Prepare directories
+      run: mkdir "${RESULT_DIR}"
+    - name: Get instance information
+      run: |
+          curl -H "Metadata:true" "http://169.254.169.254/metadata/instance?api-version=2019-11-01" | \
+          jq '{ location : .compute.location, vmSize : .compute.vmSize }' | \
+          tee ${{ env.RESULT_DIR }}/instance.json
+    - name: Run argo workflow
+      env:
+        RESULT: ${{ env.RESULT_DIR }}/result.json
+      run: make test-k3s-argo-workflow
+    - uses: actions/upload-artifact@v4
+      with:
+        name: k3s-argo-workflow
+        path: ${{ env.RESULT_DIR }}
 
   #
   # Project checks
@@ -71,13 +311,43 @@ jobs:
 
   project:
     name: Project Checks
-    runs-on: ubuntu-18.04
-    timeout-minutes: 5
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.x'
+      - uses: actions/checkout@v4
         with:
           path: src/github.com/containerd/stargz-snapshotter
           fetch-depth: 25
-      - uses: containerd/project-checks@v1
+      - uses: containerd/project-checks@v1.2.2
         with:
           working-directory: src/github.com/containerd/stargz-snapshotter
+          # go-licenses-ignore is set because go-licenses cannot correctly detect the license of the following packages:
+          # * estargz packages: Apache-2.0 and BSD-3-Clause dual license
+          #   (https://github.com/containerd/stargz-snapshotter/blob/main/NOTICE.md)
+          #
+          # The list of the CNCF-approved licenses can be found here:
+          # https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md
+          #
+          # hashicorp packages: MPL-2.0
+          # (https://github.com/hashicorp/go-cleanhttp/blob/master/LICENSE,
+          #  https://github.com/hashicorp/go-retryablehttp/blob/master/LICENSE)
+          # Note: MPL-2.0 is not in the CNCF-approved licenses list, but these packages are allowed as exceptions.
+          # See CNCF licensing exceptions:
+          # https://github.com/cncf/foundation/blob/main/license-exceptions/CNCF-licensing-exceptions.csv
+          go-licenses-ignore: |
+            github.com/containerd/stargz-snapshotter/estargz
+            github.com/containerd/stargz-snapshotter/estargz/errorutil
+            github.com/containerd/stargz-snapshotter/estargz/externaltoc
+            github.com/containerd/stargz-snapshotter/estargz/zstdchunked
+            github.com/hashicorp/go-cleanhttp
+            github.com/hashicorp/go-retryablehttp
+      - name: Check proto generated code
+        run: make validate-generated
+        working-directory: src/github.com/containerd/stargz-snapshotter
+      - run: ./script/util/verify-no-patent.sh
+        working-directory: src/github.com/containerd/stargz-snapshotter
+      - run: make validate-vendor
+        working-directory: src/github.com/containerd/stargz-snapshotter

.golangci.yml

@@ -1,23 +1,54 @@
+version: "2"
 linters:
   enable:
-    - structcheck
-    - varcheck
-    - staticcheck
-    - unconvert
-    - gofmt
-    - goimports
-    - golint
-    - ineffassign
-    - vet
-    - unused
+    - depguard
     - misspell
+    - revive
+    - unconvert
   disable:
     - errcheck
-
-run:
-  deadline: 4m
-  skip-dirs:
-    - images
-    - out
-    - script
-    - vendor
+  settings:
+    depguard:
+      rules:
+        main:
+          deny:
+            - pkg: github.com/containerd/containerd/errdefs
+              desc: The containerd errdefs package was migrated to a separate module. Use github.com/containerd/errdefs instead.
+            - pkg: github.com/containerd/containerd/log
+              desc: The containerd log package was migrated to a separate module. Use github.com/containerd/log instead.
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - linters:
+          - revive
+        text: unused-parameter
+      - linters:
+          - revive
+        text: redefines-builtin-id
+    paths:
+      - docs
+      - images
+      - out
+      - script
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - docs
+      - images
+      - out
+      - script
+      - third_party$
+      - builtin$
+      - examples$

Dockerfile

@@ -12,56 +12,144 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-ARG CONTAINERD_VERSION=v1.4.3
-ARG RUNC_VERSION=v1.0.0-rc92
-ARG CNI_PLUGINS_VERSION=v0.9.0
+ARG CONTAINERD_VERSION=v2.1.3
+ARG RUNC_VERSION=v1.3.0
+ARG CNI_PLUGINS_VERSION=v1.7.1
+ARG NERDCTL_VERSION=2.1.3
+
+ARG PODMAN_VERSION=v5.5.2
+ARG CRIO_VERSION=v1.33.2
+ARG CONMON_VERSION=v2.1.13
+ARG COMMON_VERSION=v0.63.0
+ARG CRIO_TEST_PAUSE_IMAGE_NAME=registry.k8s.io/pause:3.6
+ARG NETAVARK_VERSION=v1.15.2
+
+ARG CONTAINERIZED_SYSTEMD_VERSION=v0.1.1
+ARG SLIRP4NETNS_VERSION=v1.3.3
+ARG PAUSE_IMAGE_NAME_TEST=registry.k8s.io/pause:3.10.1
+
+# Used in CI
+ARG CRI_TOOLS_VERSION=v1.30.1
 
 # Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg.
 # If TARGETARCH isn't supported by the builder, the default value is "amd64".
 
-FROM golang:1.13-buster AS golang-base
+FROM golang:1.24-bullseye AS golang-base
 
 # Build containerd
-FROM golang-base AS containerd-dev
+FROM --platform=$BUILDPLATFORM golang:1.24-bullseye AS containerd-dev
 ARG CONTAINERD_VERSION
-RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \
-    git clone -b ${CONTAINERD_VERSION} --depth 1 \
+ARG TARGETARCH
+RUN git clone -b ${CONTAINERD_VERSION} --depth 1 \
               https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \
     cd $GOPATH/src/github.com/containerd/containerd && \
-    GO111MODULE=off make && DESTDIR=/out/ make install
+    GOARCH=$TARGETARCH make && DESTDIR=/out/ PREFIX= make install
+
+# Build containerd with builtin stargz snapshotter
+FROM --platform=$BUILDPLATFORM golang:1.24-bullseye AS containerd-snapshotter-dev
+ARG CONTAINERD_VERSION
+ARG TARGETARCH
+COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
+RUN git clone -b ${CONTAINERD_VERSION} --depth 1 \
+      https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \
+    cd $GOPATH/src/github.com/containerd/containerd && \
+    echo 'require github.com/containerd/stargz-snapshotter v0.0.0' >> go.mod && \
+    echo 'replace github.com/containerd/stargz-snapshotter => '$GOPATH'/src/github.com/containerd/stargz-snapshotter' >> go.mod && \
+    echo 'replace github.com/containerd/stargz-snapshotter/estargz => '$GOPATH'/src/github.com/containerd/stargz-snapshotter/estargz' >> go.mod && \
+    # recent containerd requires to update api/go.mod and integration/client/go.mod as well.
+    if [ -f api/go.mod ] ; then \
+      echo 'replace github.com/containerd/stargz-snapshotter => '$GOPATH'/src/github.com/containerd/stargz-snapshotter' >> api/go.mod && \
+      echo 'replace github.com/containerd/stargz-snapshotter/estargz => '$GOPATH'/src/github.com/containerd/stargz-snapshotter/estargz' >> api/go.mod ; \
+    fi && \
+    if [ -f integration/client/go.mod ] ; then \
+      echo 'replace github.com/containerd/stargz-snapshotter => '$GOPATH'/src/github.com/containerd/stargz-snapshotter' >> integration/client/go.mod && \
+      echo 'replace github.com/containerd/stargz-snapshotter/estargz => '$GOPATH'/src/github.com/containerd/stargz-snapshotter/estargz' >> integration/client/go.mod ; \
+    fi && \
+    echo 'package main \nimport _ "github.com/containerd/stargz-snapshotter/service/plugin"' > cmd/containerd/builtins_stargz_snapshotter.go && \
+    make vendor && GOARCH=$TARGETARCH make && DESTDIR=/out/ PREFIX= make install
 
 # Build runc
-FROM golang-base AS runc-dev
+FROM golang:1.24-bullseye AS runc-dev
 ARG RUNC_VERSION
 RUN apt-get update -y && apt-get install -y libseccomp-dev && \
     git clone -b ${RUNC_VERSION} --depth 1 \
               https://github.com/opencontainers/runc $GOPATH/src/github.com/opencontainers/runc && \
     cd $GOPATH/src/github.com/opencontainers/runc && \
-    GO111MODULE=off make && make install PREFIX=/out/
+    make && make install PREFIX=/out/
 
 # Build stargz snapshotter
-FROM golang-base AS snapshotter-dev
+FROM --platform=$BUILDPLATFORM golang:1.24-bullseye AS snapshotter-dev
+ARG TARGETARCH
+ARG GOARM
 ARG SNAPSHOTTER_BUILD_FLAGS
 ARG CTR_REMOTE_BUILD_FLAGS
 COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
+ARG CGO_ENABLED
 RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
-    PREFIX=/out/ GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make containerd-stargz-grpc && \
-    PREFIX=/out/ GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make ctr-remote
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make containerd-stargz-grpc && \
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make ctr-remote && \
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${CTR_REMOTE_BUILD_FLAGS} make stargz-fuse-manager
+
+# Build stargz store
+FROM golang-base AS stargz-store-dev
+ARG TARGETARCH
+ARG GOARM
+ARG SNAPSHOTTER_BUILD_FLAGS
+ARG CTR_REMOTE_BUILD_FLAGS
+COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
+ARG CGO_ENABLED
+RUN cd $GOPATH/src/github.com/containerd/stargz-snapshotter && \
+    PREFIX=/out/ GOARCH=${TARGETARCH:-amd64} GO_BUILD_FLAGS=${SNAPSHOTTER_BUILD_FLAGS} make stargz-store stargz-store-helper
+
+# Build podman
+FROM golang-base AS podman-dev
+ARG PODMAN_VERSION
+RUN apt-get update -y && apt-get install -y libseccomp-dev libgpgme-dev && \
+    git clone https://github.com/containers/podman $GOPATH/src/github.com/containers/podman && \
+    cd $GOPATH/src/github.com/containers/podman && \
+    git checkout ${PODMAN_VERSION} && \
+    make && make install PREFIX=/out/
+
+# Build CRI-O
+# FROM golang-base AS cri-o-dev
+FROM golang:1.24-bullseye AS cri-o-dev
+ARG CRIO_VERSION
+RUN apt-get update -y && apt-get install -y libseccomp-dev libgpgme-dev && \
+    git clone https://github.com/cri-o/cri-o $GOPATH/src/github.com/cri-o/cri-o && \
+    cd $GOPATH/src/github.com/cri-o/cri-o && \
+    git checkout ${CRIO_VERSION} && \
+    make && make install PREFIX=/out/ && \
+    curl -sSL --output /out/crio.service https://raw.githubusercontent.com/cri-o/cri-o/${CRIO_VERSION}/contrib/systemd/crio.service
+
+# Build conmon
+FROM golang-base AS conmon-dev
+ARG CONMON_VERSION
+RUN apt-get update -y && apt-get install -y gcc git libc6-dev libglib2.0-dev pkg-config make libseccomp-dev && \
+    git clone -b ${CONMON_VERSION} --depth 1 \
+              https://github.com/containers/conmon $GOPATH/src/github.com/containers/conmon && \
+    cd $GOPATH/src/github.com/containers/conmon && \
+    mkdir /out/ && make && make install PREFIX=/out/
+
+# Get seccomp.json for Podman/CRI-O
+FROM golang-base AS containers-common-dev
+ARG COMMON_VERSION
+RUN git clone https://github.com/containers/common $GOPATH/src/github.com/containers/common && \
+    cd $GOPATH/src/github.com/containers/common && \
+    git checkout ${COMMON_VERSION} && mkdir /out/ && cp pkg/seccomp/seccomp.json /out/
 
 # Binaries for release
 FROM scratch AS release-binaries
 COPY --from=snapshotter-dev /out/* /
+COPY --from=stargz-store-dev /out/* /
 
 # Base image which contains containerd with default snapshotter
-# `docker-ce-cli` is used only for users to `docker login` to registries (e.g. DockerHub)
-# with configuring ~/.docker/config.json
 FROM golang-base AS containerd-base
-RUN apt-get update -y && apt-get --no-install-recommends install -y fuse \
-                                 apt-transport-https gnupg2 software-properties-common && \
-    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - && \
-    add-apt-repository \
-      "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \
-    apt-get update -y && apt-get --no-install-recommends install -y docker-ce-cli
+ARG TARGETARCH
+ARG NERDCTL_VERSION
+RUN apt-get update -y && apt-get --no-install-recommends install -y fuse3 && \
+    curl -sSL --output /tmp/nerdctl.tgz https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${TARGETARCH:-amd64}.tar.gz && \
+    tar zxvf /tmp/nerdctl.tgz -C /usr/local/bin && \
+    rm -f /tmp/nerdctl.tgz
 COPY --from=containerd-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/
 COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
 
@@ -70,15 +158,80 @@ FROM containerd-base AS snapshotter-base
 COPY --from=snapshotter-dev /out/* /usr/local/bin/
 RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr
 
-# Image which can be used as all-in-one single node demo environment
-FROM snapshotter-base AS cind
-COPY ./script/config/ /
-COPY ./script/cind/ /
-VOLUME /var/lib/containerd
-VOLUME /var/lib/containerd-stargz-grpc
-VOLUME /run/containerd-stargz-grpc
-ENV CONTAINERD_SNAPSHOTTER=stargz
-ENTRYPOINT [ "/entrypoint.sh" ]
+# Base image which contains containerd with builtin stargz snapshotter
+FROM golang-base AS containerd-snapshotter-base
+ARG TARGETARCH
+ARG NERDCTL_VERSION
+RUN apt-get update -y && apt-get --no-install-recommends install -y fuse3 && \
+    curl -sSL --output /tmp/nerdctl.tgz https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${TARGETARCH:-amd64}.tar.gz && \
+    tar zxvf /tmp/nerdctl.tgz -C /usr/local/bin && \
+    rm -f /tmp/nerdctl.tgz
+COPY --from=containerd-snapshotter-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/
+COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
+COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/
+RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr
+
+# Base image which contains podman with stargz-store
+FROM ubuntu:24.04 AS podman-base
+ARG TARGETARCH
+ARG CNI_PLUGINS_VERSION
+ARG PODMAN_VERSION
+ARG NETAVARK_VERSION
+RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y fuse3 libgpgme-dev \
+                         iptables libyajl-dev curl ca-certificates libglib2.0 libseccomp-dev wget && \
+    # Make CNI plugins manipulate iptables instead of nftables
+    # as this test runs in a Docker container that network is configured with iptables.
+    # c.f. https://github.com/moby/moby/issues/26824
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    mkdir -p /etc/containers /etc/cni/net.d /opt/cni/bin && \
+    curl -qsSL https://raw.githubusercontent.com/containers/podman/${PODMAN_VERSION}/cni/87-podman-bridge.conflist | tee /etc/cni/net.d/87-podman-bridge.conflist && \
+    curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin
+
+RUN mkdir /tmp/netavark ; \
+    wget -O /tmp/netavark/netavark.gz https://github.com/containers/netavark/releases/download/${NETAVARK_VERSION}/netavark.gz ; \
+    gunzip /tmp/netavark/netavark.gz ; \
+    mkdir -p /usr/local/libexec/podman ; \
+    mv /tmp/netavark/netavark /usr/local/libexec/podman/ ; \
+    chmod 0755 /usr/local/libexec/podman/netavark
+
+COPY --from=podman-dev /out/bin/* /usr/local/bin/
+COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
+COPY --from=conmon-dev /out/bin/* /usr/local/bin/
+COPY --from=containers-common-dev /out/seccomp.json /usr/share/containers/
+COPY --from=stargz-store-dev /out/* /usr/local/bin/
+
+# Image for testing rootless Podman with Stargz Store.
+# This takes the same approach as nerdctl CI: https://github.com/containerd/nerdctl/blob/6341c8320984f7148b92dd33472d8eaca6dba756/Dockerfile#L302-L326
+FROM podman-base AS podman-rootless
+ARG CONTAINERIZED_SYSTEMD_VERSION
+ARG SLIRP4NETNS_VERSION
+RUN apt-get update -y && apt-get install -y \
+                         systemd systemd-sysv dbus dbus-user-session \
+                         openssh-server openssh-client uidmap
+RUN curl -o /usr/local/bin/slirp4netns --fail -L https://github.com/rootless-containers/slirp4netns/releases/download/${SLIRP4NETNS_VERSION}/slirp4netns-$(uname -m) && \
+    chmod +x /usr/local/bin/slirp4netns && \
+    curl -L -o /docker-entrypoint.sh https://raw.githubusercontent.com/AkihiroSuda/containerized-systemd/${CONTAINERIZED_SYSTEMD_VERSION}/docker-entrypoint.sh && \
+    chmod +x /docker-entrypoint.sh && \
+    curl -L -o /etc/containers/policy.json https://raw.githubusercontent.com/containers/skopeo/master/default-policy.json
+# storage.conf plugs Stargz Store into Podman as an Additional Layer Store
+COPY ./script/podman/config/storage.conf /home/rootless/.config/containers/storage.conf
+# Stargz Store systemd service for rootless Podman
+COPY ./script/podman/config/podman-rootless-stargz-store.service /home/rootless/.config/systemd/user/
+
+COPY ./script/podman/config/containers.conf /home/rootless/.config/containers/containers.conf
+
+# test-podman-rootless.sh logins to the user via SSH
+COPY ./script/podman/config/test-podman-rootless.sh /test-podman-rootless.sh
+RUN ssh-keygen -q -t rsa -f /root/.ssh/id_rsa -N '' && \
+    useradd -m -s /bin/bash rootless && \
+    mkdir -p -m 0700 /home/rootless/.ssh && \
+    cp -a /root/.ssh/id_rsa.pub /home/rootless/.ssh/authorized_keys && \
+    mkdir -p /home/rootless/.local/share /home/rootless/.local/share/stargz-store/store && \
+    chown -R rootless:rootless /home/rootless
+VOLUME /home/rootless/.local/share
+
+ENTRYPOINT ["/docker-entrypoint.sh", "/test-podman-rootless.sh"]
+CMD ["/bin/bash", "--login", "-i"]
 
 # Image which can be used for interactive demo environment
 FROM containerd-base AS demo
@@ -92,22 +245,52 @@ RUN apt-get update && apt-get install -y iptables && \
     mkdir -p /opt/cni/bin && \
     curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin
 
-# Image which can be used as containerized `ctr-remote images optimize` command
-FROM ubuntu:20.04 AS oind
-
-RUN apt-get update -y && \
-    apt-get --no-install-recommends install -y fuse runc ca-certificates
-
+# Image which can be used as a node image for KinD (containerd with builtin snapshotter)
+FROM kindest/node:v1.33.2 AS kind-builtin-snapshotter
+COPY --from=containerd-snapshotter-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/
 COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/
-COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
+COPY ./script/config/ /
+RUN apt-get update -y && apt-get install --no-install-recommends -y fuse3
+ENTRYPOINT [ "/usr/local/bin/kind-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init" ]
 
-ENTRYPOINT [ "/usr/local/bin/ctr-remote", "images", "optimize" ]
+# Image for testing CRI-O with Stargz Store.
+# NOTE: This cannot be used for the node image of KinD.
+FROM ubuntu:24.04 AS crio-stargz-store
+ARG CNI_PLUGINS_VERSION
+ARG CRIO_TEST_PAUSE_IMAGE_NAME
+ENV container docker
+RUN apt-get update -y && apt-get install --no-install-recommends -y \
+                         ca-certificates fuse3 libgpgme-dev libglib2.0-dev curl \
+                         iptables conntrack systemd systemd-sysv && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y tzdata && \
+    # Make CNI plugins manipulate iptables instead of nftables
+    # as this test runs in a Docker container that network is configured with iptables.
+    # c.f. https://github.com/moby/moby/issues/26824
+    update-alternatives --set iptables /usr/sbin/iptables-legacy && \
+    mkdir -p /opt/cni/bin && \
+    curl -sSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin && \
+    echo ${CRIO_TEST_PAUSE_IMAGE_NAME} > /pause_name && \
+    mkdir -p /etc/sysconfig && \
+    echo CRIO_RUNTIME_OPTIONS=--pause-image=${CRIO_TEST_PAUSE_IMAGE_NAME} > /etc/sysconfig/crio && \
+    # Necessary to pass CRI tests: https://github.com/kubernetes-sigs/cri-tools/pull/905
+    mkdir -p /etc/crio/crio.conf.d && \
+    printf '[crio.runtime]\nseccomp_use_default_when_empty = false\n' > /etc/crio/crio.conf.d/02-seccomp.conf
+
+COPY --from=stargz-store-dev /out/* /usr/local/bin/
+COPY --from=cri-o-dev /out/bin/* /usr/local/bin/
+COPY --from=cri-o-dev /out/crio.service /etc/systemd/system/
+COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
+COPY --from=conmon-dev /out/bin/* /usr/local/bin/
+COPY --from=containers-common-dev /out/seccomp.json /usr/share/containers/
+COPY ./script/config-cri-o/ /
+
+ENTRYPOINT [ "/usr/local/bin/entrypoint" ]
 
 # Image which can be used as a node image for KinD
-FROM kindest/node:v1.20.0
+FROM kindest/node:v1.33.2
 COPY --from=containerd-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/
 COPY --from=snapshotter-dev /out/* /usr/local/bin/
 COPY ./script/config/ /
-RUN apt-get update -y && apt-get install --no-install-recommends -y fuse && \
+RUN apt-get update -y && apt-get install --no-install-recommends -y fuse3 && \
     systemctl enable stargz-snapshotter
-ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
+ENTRYPOINT [ "/usr/local/bin/kind-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init" ]

MAINTAINERS

@@ -1,8 +1,8 @@
 # stargz-snapshotter maintainers
 #
-# As a containerd sub-project, containerd maintainers are also included from https://github.com/containerd/project/blob/master/MAINTAINERS.
-# See https://github.com/containerd/project/blob/master/GOVERNANCE.md for description of maintainer role
+# As a containerd sub-project, containerd maintainers are also included from https://github.com/containerd/project/blob/main/MAINTAINERS.
+# See https://github.com/containerd/project/blob/main/GOVERNANCE.md for description of maintainer role
 #
-# MAINTAINERS
+# COMMITTERS
 # GitHub ID, Name, Email address
 ktock, Kohei Tokunaga, ktokunaga.mail@gmail.com

Makefile

@@ -16,13 +16,19 @@
 # Base path used to install.
 CMD_DESTDIR ?= /usr/local
 GO111MODULE_VALUE=auto
-PREFIX ?= out/
+PREFIX ?= $(CURDIR)/out/
 
-CMD=containerd-stargz-grpc ctr-remote
+PKG=github.com/containerd/stargz-snapshotter
+VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
+REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
+GO_BUILD_LDFLAGS ?= -s -w
+GO_LD_FLAGS=-ldflags '$(GO_BUILD_LDFLAGS) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(GO_EXTRA_LDFLAGS)'
+
+CMD=containerd-stargz-grpc ctr-remote stargz-store stargz-fuse-manager
 
 CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))
 
-.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-pullsecrets test-cri
+.PHONY: all build check install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow vendor
 
 all: build
 
@@ -31,18 +37,26 @@ build: $(CMD)
 FORCE:
 
 containerd-stargz-grpc: FORCE
-	GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) -v ./cmd/containerd-stargz-grpc
+	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./containerd-stargz-grpc
 
 ctr-remote: FORCE
-	GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) -v ./cmd/ctr-remote
+	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./ctr-remote
+
+stargz-store: FORCE
+	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store
+
+stargz-store-helper: FORCE
+	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store/helper
+
+stargz-fuse-manager: FORCE
+	cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-fuse-manager
 
 check:
 	@echo "$@"
-	@GO111MODULE=$(GO111MODULE_VALUE) golangci-lint run
-	@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) golangci-lint run
-
-install-check-tools:
-	@curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.19.1
+	@GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
+	@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
+	@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
+	@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
 
 install:
 	@echo "$@"
@@ -57,10 +71,24 @@ clean:
 	@echo "$@"
 	@rm -f $(CMD_BINARIES)
 
+generate:
+	@./script/generated-files/generate.sh update
+
+validate-generated:
+	@./script/generated-files/generate.sh validate
+
+vendor:
+	@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
+	@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
+	@GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
+	@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
+
 test:
 	@echo "$@"
 	@GO111MODULE=$(GO111MODULE_VALUE) go test -race ./...
-	@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -race ./...
+	@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 30m -race ./...
+	@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
+	@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
 
 test-root:
 	@echo "$@"
@@ -77,8 +105,33 @@ test-optimize:
 benchmark:
 	@./script/benchmark/test.sh
 
-test-pullsecrets:
-	@./script/pullsecrets/test.sh
+test-kind:
+	@./script/kind/test.sh
 
-test-cri:
-	@./script/cri/test.sh
+test-cri-containerd:
+	@./script/cri-containerd/test.sh
+
+test-cri-o:
+	@./script/cri-o/test.sh
+
+test-podman:
+	@./script/podman/test.sh
+
+test-criauth:
+	@./script/criauth/test.sh
+
+test-k3s:
+	@./script/k3s/test.sh
+
+test-k3s-argo-workflow:
+	@./script/k3s-argo-workflow/run.sh
+
+test-ipfs:
+	@./script/ipfs/test.sh
+
+validate-vendor:
+	$(eval TMPDIR := $(shell mktemp -d))
+	@cp -R $(CURDIR) ${TMPDIR}
+	@(cd ${TMPDIR}/stargz-snapshotter && make vendor)
+	@diff -r -u -q $(CURDIR) ${TMPDIR}/stargz-snapshotter
+	@rm -rf ${TMPDIR}

README.md

@@ -1,7 +1,14 @@
+[[⬇️ **Download**]](https://github.com/containerd/stargz-snapshotter/releases)
+[[📔**Browse images**]](./docs/pre-converted-images.md)
+[[☸**Quick Start (Kubernetes)**]](#quick-start-with-kubernetes)
+[[🤓**Quick Start (nerdctl)**]](https://github.com/containerd/nerdctl/blob/master/docs/stargz.md)
+[[🔆**Install**]](./docs/INSTALL.md)
+
 # Stargz Snapshotter
 
-[![Tests Status](https://github.com/containerd/stargz-snapshotter/workflows/Tests/badge.svg)](https://github.com/containerd/stargz-snapshotter/actions?query=workflow%3ATests+branch%3Amaster)
-[![Benchmarking](https://github.com/containerd/stargz-snapshotter/workflows/Benchmark/badge.svg)](https://github.com/containerd/stargz-snapshotter/actions?query=workflow%3ABenchmark+branch%3Amaster)
+[![Tests Status](https://github.com/containerd/stargz-snapshotter/workflows/Tests/badge.svg)](https://github.com/containerd/stargz-snapshotter/actions?query=workflow%3ATests+branch%3Amain)
+[![Benchmarking](https://github.com/containerd/stargz-snapshotter/workflows/Benchmark/badge.svg)](https://github.com/containerd/stargz-snapshotter/actions?query=workflow%3ABenchmark+branch%3Amain)
+[![Nightly](https://github.com/containerd/stargz-snapshotter/workflows/Nightly/badge.svg)](https://github.com/containerd/stargz-snapshotter/actions?query=workflow%3ANightly+branch%3Amain)
 
 Read also introductory blog: [Startup Containers in Lightning Speed with Lazy Image Distribution on Containerd](https://medium.com/nttlabs/startup-containers-in-lightning-speed-with-lazy-image-distribution-on-containerd-243d94522361)
 
@@ -32,11 +39,15 @@ We are constantly measuring the performance of this snapshotter so you can get t
 Please note that we sometimes see dispersion among the results because of the NW condition on the internet and the location of the instance in the Github Actions, etc.
 Our benchmarking method is based on [HelloBench](https://github.com/Tintri/hello-bench).
 
+:nerd_face: You can also run containers on IPFS with lazy pulling. This is an experimental feature. See [`./docs/ipfs.md`](./docs/ipfs.md) for more details.
+
 Stargz Snapshotter is a **non-core** sub-project of containerd.
 
 ## Quick Start with Kubernetes
 
 - For more details about stargz snapshotter plugin and its configuration, refer to [Containerd Stargz Snapshotter Plugin Overview](/docs/overview.md).
+- For more details about setup lazy pulling of eStargz with containerd, CRI-O, Podman, systemd, etc., refer to [Install Stargz Snapshotter and Stargz Store](./docs/INSTALL.md).
+- For more details about integration status of eStargz with tools in commuinty, refer to [Integration of eStargz with other tools](./docs/integration.md)
 
 For using stargz snapshotter on kubernetes nodes, you need the following configuration to containerd as well as run stargz snapshotter daemon on the node.
 We assume that you are using containerd (> v1.4.2) as a CRI runtime.
@@ -51,6 +62,8 @@ version = 2
   [proxy_plugins.stargz]
     type = "snapshot"
     address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
 
 # Use stargz snapshotter through CRI
 [plugins."io.containerd.grpc.v1.cri".containerd]
@@ -58,18 +71,18 @@ version = 2
   disable_snapshot_annotations = false
 ```
 
-**Note that `disable_snapshot_annotations = false` is required since containerd > v1.4.2**
-
-This repo contains [a Dockerfile as a KinD node image](/Dockerfile) which includes the above configuration.
-You can use it with [KinD](https://github.com/kubernetes-sigs/kind) like the following,
+You can try our [prebuilt](/Dockerfile) [KinD](https://github.com/kubernetes-sigs/kind) node image that contains the above configuration.
 
 ```console
-$ docker build -t stargz-kind-node https://github.com/containerd/stargz-snapshotter.git
-$ kind create cluster --name stargz-demo --image stargz-kind-node
+$ kind create cluster --name stargz-demo --image ghcr.io/containerd/stargz-snapshotter:0.12.1-kind
 ```
 
+:information_source: kind binary v0.16.x or newer is recommended for `ghcr.io/containerd/stargz-snapshotter:0.12.1-kind`.
+
+:information_source: You can get latest node images from [`ghcr.io/containerd/stargz-snapshotter:${VERSION}-kind`](https://github.com/orgs/containerd/packages/container/package/stargz-snapshotter) namespace.
+
 Then you can create eStargz pods on the cluster.
-In this example, we create a stargz-converted Node.js pod (`ghcr.io/stargz-containers/node:13.13.0-esgz`) as a demo.
+In this example, we create a stargz-converted Node.js pod (`ghcr.io/stargz-containers/node:17.8.0-esgz`) as a demo.
 
 ```yaml
 apiVersion: v1
@@ -79,7 +92,7 @@ metadata:
 spec:
   containers:
   - name: nodejs-stargz
-    image: ghcr.io/stargz-containers/node:13.13.0-esgz
+    image: ghcr.io/stargz-containers/node:17.8.0-esgz
     command: ["node"]
     args:
     - -e
@@ -92,10 +105,10 @@ spec:
     - containerPort: 80
 ```
 
-The following command lazily pulls `ghcr.io/stargz-containers/node:13.13.0-esgz` from Github Container Registry and creates the pod so the time to take for it is shorter than the original image `library/node:13.13`.
+The following command lazily pulls `ghcr.io/stargz-containers/node:17.8.0-esgz` from Github Container Registry and creates the pod so the time to take for it is shorter than the original image `library/node:13.13`.
 
 ```console
-$ kubectl --context kind-stargz-demo apply -f stargz-pod.yaml && kubectl get po nodejs -w
+$ kubectl --context kind-stargz-demo apply -f stargz-pod.yaml && kubectl --context kind-stargz-demo get po nodejs -w
 $ kubectl --context kind-stargz-demo port-forward nodejs 8080:80 &
 $ curl 127.0.0.1:8080
 Hello World!
@@ -103,16 +116,74 @@ Hello World!
 
 Stargz snapshotter also supports [further configuration](/docs/overview.md) including private registry authentication, mirror registries, etc.
 
-## Creating eStargz images with optimization
+## Getting eStargz images
 
 - For more examples and details about the image converter `ctr-remote`, refer to [Optimize Images with `ctr-remote image optimize`](/docs/ctr-remote.md).
 - For more details about eStargz format, refer to [eStargz: Standard-Compatible Extensions to Tar.gz Layers for Lazy Pulling Container Images](/docs/stargz-estargz.md)
 
 For lazy pulling images, you need to prepare eStargz images first.
-You can use [`ctr-remote`](/docs/ctr-remote.md) command for do this.
-You can also try our pre-converted images listed in [Trying pre-converted images](/docs/pre-converted-images.md).
+There are several ways to achieve that.
+This section describes some of them.
 
-In this section, we introduce `ctr-remote` command for converting images into eStargz with optimization for reading files.
+### Trying pre-built eStargz images
+
+You can try our pre-converted eStargz images on ghcr.io listed in [Trying pre-converted images](/docs/pre-converted-images.md).
+
+### Building eStargz images using BuildKit
+
+BuildKit supports building eStargz image since v0.10.
+
+You can try it using [Docker Buildx](https://docs.docker.com/buildx/working-with-buildx/).
+The following command builds an eStargz image and push it to `ghcr.io/ktock/hello:esgz`.
+Flags `oci-mediatypes=true,compression=estargz` enable to build eStargz.
+
+```
+$ docker buildx build -t ghcr.io/ktock/hello:esgz \
+    -o type=registry,oci-mediatypes=true,compression=estargz,force-compression=true \
+    /tmp/buildctx/
+```
+
+> NOTE1: `force-compression=true` isn't needed if the base image is already eStargz.
+
+> NOTE2: Docker still does not support lazy pulling of eStargz.
+
+eStargz-enabled BuildKit (v0.10) will be [included to Docker v22.XX](https://github.com/moby/moby/blob/v22.06.0-beta.0/vendor.mod#L51) however you can build eStargz images with the prior version using Buildx [driver](https://github.com/docker/buildx/blob/master/docs/reference/buildx_create.md#-set-the-builder-driver-to-use---driver) feature.
+You can enable the specific version of BuildKit using [`docker buildx create`](https://docs.docker.com/engine/reference/commandline/buildx_create/) (this example specifies `v0.10.3`).
+
+```
+$ docker buildx create --use --name v0.10.3 --driver docker-container --driver-opt image=moby/buildkit:v0.10.3
+$ docker buildx inspect --bootstrap v0.10.3
+```
+
+### Building eStargz images using Kaniko
+
+[Kaniko](https://github.com/GoogleContainerTools/kaniko) is an image builder runnable in containers and Kubernetes.
+Since v1.5.0, it experimentally supports building eStargz.
+`GGCR_EXPERIMENT_ESTARGZ=1` is needed.
+
+```console
+$ docker run --rm -e GGCR_EXPERIMENT_ESTARGZ=1 \
+    -v /tmp/buildctx:/workspace -v ~/.docker/config.json:/kaniko/.docker/config.json:ro \
+    gcr.io/kaniko-project/executor:v1.8.1 --destination ghcr.io/ktock/hello:esgz
+```
+
+### Building eStargz images using nerdctl
+
+[nerdctl](https://github.com/containerd/nerdctl), Docker-compatible CLI of containerd, supports building eStargz images.
+
+```console
+$ nerdctl build -t ghcr.io/ktock/hello:1 /tmp/buildctx
+$ nerdctl image convert --estargz --oci ghcr.io/ktock/hello:1 ghcr.io/ktock/hello:esgz
+$ nerdctl push ghcr.io/ktock/hello:esgz
+```
+
+> NOTE: `--estargz` should be specified in conjunction with `--oci`
+
+Please refer to nerdctl document for details for further information (e.g. lazy pulling): https://github.com/containerd/nerdctl/blob/master/docs/stargz.md
+
+### Creating eStargz images using `ctr-remote`
+
+[`ctr-remote`](/docs/ctr-remote.md) allows converting an image into eStargz with optimizing it.
 As shown in the above benchmarking result, on-demand lazy pulling improves the performance of pull but causes runtime performance penalty because reading files induce remotely downloading contents.
 For solving this, `ctr-remote` has *workload-based* optimization for images.
 
@@ -132,34 +203,48 @@ Generally, container images are built with purpose and the *workloads* are defin
 By default, `ctr-remote` optimizes the performance of reading files that are most likely accessed in the workload defined in the Dockerfile.
 [You can also specify the custom workload using options if needed](/docs/ctr-remote.md).
 
-The following example converts the legacy `library/ubuntu:18.04` image into eStargz.
+The following example converts the legacy `library/ubuntu:20.04` image into eStargz.
 The command also optimizes the image for the workload of executing `ls` on `/bin/bash`.
 The thing actually done is it runs the specified workload in a temporary container and profiles all file accesses with marking them as *likely accessed* also during runtime.
-Then it pushes the converted image to the local container registry (`registry2:5000`).
 The converted image is still **docker-compatible** so you can run it with eStargz-agnostic runtimes (e.g. Docker).
 
 ```console
-# ctr-remote image optimize --plain-http --entrypoint='[ "/bin/bash", "-c" ]' --args='[ "ls" ]' ubuntu:18.04 http://registry2:5000/ubuntu:18.04
+# ctr-remote image pull docker.io/library/ubuntu:20.04
+# ctr-remote image optimize --oci --entrypoint='[ "/bin/bash", "-c" ]' --args='[ "ls" ]' docker.io/library/ubuntu:20.04 registry2:5000/ubuntu:20.04
+# ctr-remote image push --plain-http registry2:5000/ubuntu:20.04
 ```
 
-Finally, the following commands pull the eStargz image lazily.
+Finally, the following commands clear the local cache then pull the eStargz image lazily.
 Stargz snapshotter prefetches files that are most likely accessed in the optimized workload, which hopefully increases the cache hit rate for that workload and mitigates runtime overheads as shown in the benchmarking result shown top of this doc.
 
 ```console
-# ctr-remote images rpull --plain-http registry2:5000/ubuntu:18.04
-fetching sha256:728332a6... application/vnd.docker.distribution.manifest.v2+json
-fetching sha256:80026893... application/vnd.docker.container.image.v1+json
-# ctr-remote run --rm -t --snapshotter=stargz registry2:5000/ubuntu:18.04 test /bin/bash
-root@8dab301bd68d:/# ls
-bin  boot  dev  etc  home  lib  lib64  media  mnt  opt  proc  root  run  sbin  srv  sys  tmp  usr  var
+# ctr-remote image rm --sync registry2:5000/ubuntu:20.04
+# ctr-remote images rpull --plain-http registry2:5000/ubuntu:20.04
+fetching sha256:610399d1... application/vnd.oci.image.index.v1+json
+fetching sha256:0b4a26b4... application/vnd.oci.image.manifest.v1+json
+fetching sha256:8d8d9dbe... application/vnd.oci.image.config.v1+json
+# ctr-remote run --rm -t --snapshotter=stargz registry2:5000/ubuntu:20.04 test /bin/bash
+root@8eabb871a9bd:/# ls
+bin  boot  dev  etc  home  lib  lib32  lib64  libx32  media  mnt  opt  proc  root  run  sbin  srv  sys  tmp  usr  var
 ```
 
+> NOTE: You can perform lazy pulling from any OCI-compatible registries (e.g. docker.io, ghcr.io, etc) as long as the image is formatted as eStargz.
+
+## Importing Stargz Snapshotter as go module
+
+Currently, Stargz Snapshotter repository contains two Go modules as the following and both of them need to be imported.
+
+- `github.com/containerd/stargz-snapshotter`
+- `github.com/containerd/stargz-snapshotter/estargz`
+
+Please make sure you import the both of them and they point to *the same commit version*.
+
 ## Project details
 
 Stargz Snapshotter is a containerd **non-core** sub-project, licensed under the [Apache 2.0 license](./LICENSE).
 As a containerd non-core sub-project, you will find the:
- * [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md),
+ * [Project governance](https://github.com/containerd/project/blob/main/GOVERNANCE.md),
  * [Maintainers](./MAINTAINERS),
- * and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md)
+ * and [Contributing guidelines](https://github.com/containerd/project/blob/main/CONTRIBUTING.md)
 
 information in our [`containerd/project`](https://github.com/containerd/project) repository.

analyzer/analyzer.go

@@ -0,0 +1,440 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package analyzer
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"github.com/containerd/console"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands/tasks"
+	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/containerd/v2/pkg/cio"
+	"github.com/containerd/containerd/v2/pkg/oci"
+	"github.com/containerd/errdefs"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
+	"github.com/containerd/stargz-snapshotter/analyzer/fanotify"
+	"github.com/containerd/stargz-snapshotter/analyzer/recorder"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/identity"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/rs/xid"
+)
+
+var defaultPeriod = 10 * time.Second
+
+// Analyze analyzes the passed image then store the record of prioritized files into
+// containerd's content store. This function returns the digest of that record file.
+// This digest can be used to read record from the content store.
+func Analyze(ctx context.Context, client *containerd.Client, ref string, opts ...Option) (digest.Digest, error) {
+	var aOpts analyzerOpts
+	for _, o := range opts {
+		o(&aOpts)
+	}
+	if aOpts.terminal && aOpts.waitOnSignal {
+		return "", fmt.Errorf("wait-on-signal option cannot be used with terminal option")
+	}
+
+	target, err := os.MkdirTemp("", "target")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(target)
+
+	cs := client.ContentStore()
+	is := client.ImageService()
+	ss := client.SnapshotService(aOpts.snapshotter)
+
+	img, err := is.Get(ctx, ref)
+	if err != nil {
+		return "", err
+	}
+	platformImg := containerd.NewImageWithPlatform(client, img, platforms.Default())
+
+	// Mount the target image
+	// NOTE: We cannot let containerd prepare the rootfs. We create mount namespace *before*
+	//       creating the container so containerd's bundle preparation (mounting snapshots to
+	//       the bundle directory, etc.) is invisible inside the pre-unshared mount namespace.
+	//       This leads to runc using empty directory as the rootfs.
+	if unpacked, err := platformImg.IsUnpacked(ctx, aOpts.snapshotter); err != nil {
+		return "", err
+	} else if !unpacked {
+		if err := platformImg.Unpack(ctx, aOpts.snapshotter); err != nil {
+			return "", err
+		}
+	}
+	cleanup, err := mountImage(ctx, ss, platformImg, target)
+	if err != nil {
+		return "", err
+	}
+	defer cleanup()
+
+	// Spawn a fanotifier process in a new mount namespace and setup recorder.
+	fanotifier, err := fanotify.SpawnFanotifier("/proc/self/exe")
+	if err != nil {
+		return "", fmt.Errorf("failed to spawn fanotifier: %w", err)
+	}
+	defer func() {
+		if err := fanotifier.Close(); err != nil {
+			log.G(ctx).WithError(err).Warnf("failed to close fanotifier")
+		}
+	}()
+
+	// Prepare the spec based on the specified image and runtime options.
+	var sOpts []oci.SpecOpts
+	if aOpts.specOpts != nil {
+		gotOpts, done, err := aOpts.specOpts(platformImg, target)
+		if err != nil {
+			return "", err
+		}
+		defer func() {
+			if err := done(); err != nil {
+				log.G(ctx).WithError(err).Warnf("failed to cleanup container")
+				return
+			}
+		}()
+		sOpts = append(sOpts, gotOpts...)
+	} else {
+		sOpts = append(sOpts,
+			oci.WithDefaultSpec(),
+			oci.WithDefaultUnixDevices,
+			oci.WithRootFSPath(target),
+			oci.WithImageConfig(platformImg),
+		)
+	}
+	sOpts = append(sOpts, oci.WithLinuxNamespace(runtimespec.LinuxNamespace{
+		Type: runtimespec.MountNamespace,
+		Path: fanotifier.MountNamespacePath(), // use mount namespace that the fanotifier created
+	}))
+
+	// Create the container and the task
+	var container containerd.Container
+	for i := 0; i < 3; i++ {
+		id := xid.New().String()
+		var s runtimespec.Spec
+		container, err = client.NewContainer(ctx, id,
+			containerd.WithImage(platformImg),
+			containerd.WithSnapshotter(aOpts.snapshotter),
+			containerd.WithImageStopSignal(platformImg, "SIGKILL"),
+
+			// WithImageConfig depends on WithImage and WithSnapshotter for resolving
+			// username (accesses to /etc/{passwd,group} files on the rootfs)
+			containerd.WithSpec(&s, sOpts...),
+		)
+		if err != nil {
+			if errdefs.IsAlreadyExists(err) {
+				log.G(ctx).WithError(err).Warnf("failed to create container")
+				continue
+			}
+			return "", err
+		}
+		break
+	}
+	if container == nil {
+		return "", fmt.Errorf("failed to create container")
+	}
+	defer container.Delete(ctx, containerd.WithSnapshotCleanup)
+	var ioCreator cio.Creator
+	var con console.Console
+	waitLine := newLineWaiter(aOpts.waitLineOut)
+	stdinC := newLazyReadCloser(os.Stdin)
+	if aOpts.terminal {
+		if !aOpts.stdin {
+			return "", fmt.Errorf("terminal cannot be used if stdin isn't enabled")
+		}
+		con = console.Current()
+		defer con.Reset()
+		if err := con.SetRaw(); err != nil {
+			return "", err
+		}
+		// On terminal mode, the "stderr" field is unused.
+		ioCreator = cio.NewCreator(cio.WithStreams(con, waitLine.registerWriter(con), nil), cio.WithTerminal)
+	} else if aOpts.stdin {
+		ioCreator = cio.NewCreator(cio.WithStreams(stdinC, waitLine.registerWriter(os.Stdout), os.Stderr))
+	} else {
+		ioCreator = cio.NewCreator(cio.WithStreams(nil, waitLine.registerWriter(os.Stdout), os.Stderr))
+	}
+	task, err := container.NewTask(ctx, ioCreator)
+	if err != nil {
+		return "", err
+	}
+	stdinC.registerCloser(func() { // Ensure to close IO when stdin get EOF
+		task.CloseIO(ctx, containerd.WithStdinCloser)
+	})
+
+	// Start to monitor "/" and run the task.
+	rc, err := recorder.NewImageRecorder(ctx, cs, img, platforms.Default())
+	if err != nil {
+		return "", err
+	}
+	defer rc.Close()
+	if err := fanotifier.Start(); err != nil {
+		return "", fmt.Errorf("failed to start fanotifier: %w", err)
+	}
+	var fanotifierClosed bool
+	var fanotifierClosedMu sync.Mutex
+	go func() {
+		var successCount int
+		defer func() {
+			log.G(ctx).Debugf("success record %d path", successCount)
+		}()
+		for {
+			path, err := fanotifier.GetPath()
+			if err != nil {
+				if err == io.EOF {
+					fanotifierClosedMu.Lock()
+					isFanotifierClosed := fanotifierClosed
+					fanotifierClosedMu.Unlock()
+					if isFanotifierClosed {
+						break
+					}
+				}
+				log.G(ctx).WithError(err).Error("failed to get notified path")
+				break
+			}
+			if err := rc.Record(path); err != nil {
+				log.G(ctx).WithError(err).Debugf("failed to record %q", path)
+			}
+			successCount++
+		}
+	}()
+	if err := task.Start(ctx); err != nil {
+		return "", err
+	}
+	if aOpts.terminal {
+		if err := tasks.HandleConsoleResize(ctx, task, con); err != nil {
+			log.G(ctx).WithError(err).Error("failed to resize console")
+		}
+	} else {
+		sigc := commands.ForwardAllSignals(ctx, task)
+		defer commands.StopCatch(sigc)
+	}
+
+	// Wait until the task exit
+	var status containerd.ExitStatus
+	var killOk bool
+	if aOpts.waitOnSignal { // NOTE: not functional with `terminal` option
+		log.G(ctx).Infof("press Ctrl+C to terminate the container")
+		status, killOk, err = waitOnSignal(ctx, container, task)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		if aOpts.period <= 0 {
+			aOpts.period = defaultPeriod
+		}
+		log.G(ctx).Infof("waiting for %v ...", aOpts.period)
+		status, killOk, err = waitOnTimeout(ctx, container, task, aOpts.period, waitLine)
+		if err != nil {
+			return "", err
+		}
+	}
+	if !killOk {
+		log.G(ctx).Warnf("failed to exit task %v; manually kill it", task.ID())
+	} else {
+		code, _, err := status.Result()
+		if err != nil {
+			return "", err
+		}
+		log.G(ctx).Infof("container exit with code %v", code)
+		if _, err := task.Delete(ctx); err != nil {
+			return "", err
+		}
+	}
+
+	// ensure no record comes in
+	fanotifierClosedMu.Lock()
+	fanotifierClosed = true
+	fanotifierClosedMu.Unlock()
+	if err := fanotifier.Close(); err != nil {
+		log.G(ctx).WithError(err).Warnf("failed to cleanup fanotifier")
+	}
+
+	// Finish recording
+	return rc.Commit(ctx)
+}
+
+func mountImage(ctx context.Context, ss snapshots.Snapshotter, image containerd.Image, mountpoint string) (func(), error) {
+	diffIDs, err := image.RootFS(ctx)
+	if err != nil {
+		return nil, err
+	}
+	mounts, err := ss.Prepare(ctx, mountpoint, identity.ChainID(diffIDs).String())
+	if err != nil {
+		return nil, err
+	}
+	if err := mount.All(mounts, mountpoint); err != nil {
+		if err := ss.Remove(ctx, mountpoint); err != nil && !errdefs.IsNotFound(err) {
+			log.G(ctx).WithError(err).Warnf("failed to cleanup snapshot after mount error")
+		}
+		return nil, fmt.Errorf("failed to mount rootfs at %q: %w", mountpoint, err)
+	}
+	return func() {
+		if err := mount.UnmountAll(mountpoint, 0); err != nil {
+			log.G(ctx).WithError(err).Warnf("failed to unmount snapshot")
+		}
+		if err := ss.Remove(ctx, mountpoint); err != nil && !errdefs.IsNotFound(err) {
+			log.G(ctx).WithError(err).Warnf("failed to cleanup snapshot")
+		}
+	}, nil
+}
+
+func waitOnSignal(ctx context.Context, container containerd.Container, task containerd.Task) (containerd.ExitStatus, bool, error) {
+	statusC, err := task.Wait(ctx)
+	if err != nil {
+		return containerd.ExitStatus{}, false, err
+	}
+	sc := make(chan os.Signal, 1)
+	signal.Notify(sc, syscall.SIGINT)
+	defer signal.Stop(sc)
+	select {
+	case status := <-statusC:
+		return status, true, nil
+	case <-sc:
+		log.G(ctx).Info("signal detected")
+		status, err := killTask(ctx, container, task, statusC)
+		if err != nil {
+			log.G(ctx).WithError(err).Warnf("failed to kill container")
+			return containerd.ExitStatus{}, false, nil
+		}
+		return status, true, nil
+	}
+}
+
+func waitOnTimeout(ctx context.Context, container containerd.Container, task containerd.Task, period time.Duration, line *lineWaiter) (containerd.ExitStatus, bool, error) {
+	statusC, err := task.Wait(ctx)
+	if err != nil {
+		return containerd.ExitStatus{}, false, err
+	}
+	select {
+	case status := <-statusC:
+		return status, true, nil
+	case l := <-line.waitCh:
+		log.G(ctx).Infof("Waiting line detected %q; killing task", l)
+	case <-time.After(period):
+		log.G(ctx).Warnf("killing task. the time period to monitor access log (%s) has timed out", period.String())
+	}
+	status, err := killTask(ctx, container, task, statusC)
+	if err != nil {
+		log.G(ctx).WithError(err).Warnf("failed to kill container")
+		return containerd.ExitStatus{}, false, nil
+	}
+	return status, true, nil
+}
+
+func killTask(ctx context.Context, container containerd.Container, task containerd.Task, statusC <-chan containerd.ExitStatus) (containerd.ExitStatus, error) {
+	sig, err := containerd.GetStopSignal(ctx, container, syscall.SIGKILL)
+	if err != nil {
+		return containerd.ExitStatus{}, err
+	}
+	if err := task.Kill(ctx, sig, containerd.WithKillAll); err != nil && !errdefs.IsNotFound(err) {
+		return containerd.ExitStatus{}, fmt.Errorf("forward SIGKILL: %w", err)
+	}
+	select {
+	case status := <-statusC:
+		return status, nil
+	case <-time.After(5 * time.Second):
+		return containerd.ExitStatus{}, fmt.Errorf("timeout")
+	}
+}
+
+type lazyReadCloser struct {
+	reader      io.Reader
+	closer      func()
+	closerMu    sync.Mutex
+	initCond    *sync.Cond
+	initialized int64
+}
+
+func newLazyReadCloser(r io.Reader) *lazyReadCloser {
+	rc := &lazyReadCloser{reader: r, initCond: sync.NewCond(&sync.Mutex{})}
+	return rc
+}
+
+func (s *lazyReadCloser) registerCloser(closer func()) {
+	s.closerMu.Lock()
+	s.closer = closer
+	s.closerMu.Unlock()
+	atomic.AddInt64(&s.initialized, 1)
+	s.initCond.Broadcast()
+}
+
+func (s *lazyReadCloser) Read(p []byte) (int, error) {
+	if atomic.LoadInt64(&s.initialized) <= 0 {
+		// wait until initialized
+		s.initCond.L.Lock()
+		if atomic.LoadInt64(&s.initialized) <= 0 {
+			s.initCond.Wait()
+		}
+		s.initCond.L.Unlock()
+	}
+
+	n, err := s.reader.Read(p)
+	if err == io.EOF {
+		s.closerMu.Lock()
+		s.closer()
+		s.closerMu.Unlock()
+	}
+	return n, err
+}
+
+func newLineWaiter(s string) *lineWaiter {
+	return &lineWaiter{
+		waitCh:   make(chan string),
+		waitLine: s,
+	}
+}
+
+type lineWaiter struct {
+	waitCh   chan string
+	waitLine string
+}
+
+func (lw *lineWaiter) registerWriter(w io.Writer) io.Writer {
+	if lw.waitLine == "" {
+		return w
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		scanner := bufio.NewScanner(pr)
+		for scanner.Scan() {
+			if strings.Contains(scanner.Text(), lw.waitLine) {
+				lw.waitCh <- lw.waitLine
+			}
+		}
+		if _, err := io.Copy(io.Discard, pr); err != nil {
+			pr.CloseWithError(err)
+			return
+		}
+	}()
+
+	return io.MultiWriter(w, pw)
+}

analyzer/fanotify/conn/conn.go

@@ -0,0 +1,148 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package conn
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	mesStart    = "start"
+	mesStarted  = "started"
+	mesAck      = "ack"
+	mesFdPrefix = "fd:"
+)
+
+// Client is the client to talk to the fanotifier.
+type Client struct {
+	r            io.Reader
+	w            io.Writer
+	notification *bufio.Scanner
+	servicePid   int
+	timeout      time.Duration
+}
+
+// NewClient returns the client to talk to the fanotifier.
+func NewClient(r io.Reader, w io.Writer, servicePid int, timeout time.Duration) *Client {
+	return &Client{r, w, bufio.NewScanner(r), servicePid, timeout}
+}
+
+// Start lets the notirier start to monitor the filesystem.
+func (nc *Client) Start() error {
+	if err := writeMessage(nc.w, mesStart); err != nil {
+		return err
+	}
+	if mes, err := scanWithTimeout(nc.notification, nc.timeout); err != nil {
+		return err
+	} else if mes != mesStarted {
+		return fmt.Errorf("non-started message got from fanotifier service: %q", mes)
+	}
+	return nil
+}
+
+// GetPath returns notified path from the fanotifier. This blocks until new path is notified
+// from the service
+func (nc *Client) GetPath() (string, error) {
+	if !nc.notification.Scan() { // NOTE: no timeout
+		return "", io.EOF
+	}
+	mes := nc.notification.Text()
+	if !strings.HasPrefix(mes, mesFdPrefix) {
+		return "", fmt.Errorf("unexpected prefix for message %q", mes)
+	}
+	fd, err := strconv.ParseInt(mes[len(mesFdPrefix):], 10, 32)
+	if err != nil {
+		return "", fmt.Errorf("invalid fd %q: %w", mes, err)
+	}
+	path, err := os.Readlink(fmt.Sprintf("/proc/%d/fd/%d", nc.servicePid, fd))
+	if err != nil {
+		return "", fmt.Errorf("failed to get link from fd %q: %w", mes, err)
+	}
+	return path, writeMessage(nc.w, mesAck)
+}
+
+// Service is the service end of the fanotifier.
+type Service struct {
+	r       io.Reader
+	w       io.Writer
+	client  *bufio.Scanner
+	timeout time.Duration
+}
+
+// NewService is the service end of the fanotifier.
+func NewService(r io.Reader, w io.Writer, timeout time.Duration) *Service {
+	return &Service{r, w, bufio.NewScanner(r), timeout}
+}
+
+// WaitStart waits "start" message from the client.
+func (ns *Service) WaitStart() error {
+	if !ns.client.Scan() { // NOTE: no timeout
+		return io.EOF
+	}
+	if mes := ns.client.Text(); mes != mesStart {
+		return fmt.Errorf("non-start message got from fanotifier client: %q", mes)
+	}
+	return nil
+}
+
+// SendStarted tells client that the service has been started.
+func (ns *Service) SendStarted() error {
+	return writeMessage(ns.w, mesStarted)
+}
+
+// SendFd send file descriptor to the client.
+func (ns *Service) SendFd(fd int) error {
+	if err := writeMessage(ns.w, fmt.Sprintf("%s%d", mesFdPrefix, fd)); err != nil {
+		return err
+	}
+	if mes, err := scanWithTimeout(ns.client, ns.timeout); err != nil {
+		return err
+	} else if mes != mesAck {
+		return fmt.Errorf("non-ack message got from fanotifier client: %q", mes)
+	}
+	return nil
+}
+
+func scanWithTimeout(sc *bufio.Scanner, timeout time.Duration) (string, error) {
+	notifyCh := make(chan string)
+	errCh := make(chan error)
+	go func() {
+		if !sc.Scan() {
+			errCh <- io.EOF
+		}
+		notifyCh <- sc.Text()
+	}()
+	select {
+	case mes := <-notifyCh:
+		return mes, nil
+	case err := <-errCh:
+		return "", err
+	case <-time.After(timeout):
+		return "", fmt.Errorf("timeout")
+	}
+}
+
+func writeMessage(w io.Writer, mes string) error {
+	_, err := io.WriteString(w, mes+"\n")
+	return err
+}

analyzer/fanotify/fanotify.go

@@ -0,0 +1,100 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fanotify
+
+import (
+	"errors"
+	"fmt"
+	"os/exec"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/containerd/stargz-snapshotter/analyzer/fanotify/conn"
+)
+
+// Fanotifier monitors "/" mountpoint of a new mount namespace and notifies all
+// accessed files.
+type Fanotifier struct {
+	cmd       *exec.Cmd
+	conn      *conn.Client
+	closeOnce sync.Once
+	closeFunc func() error
+}
+
+func SpawnFanotifier(fanotifierBin string) (*Fanotifier, error) {
+	// Run fanotifier that monitor "/" in a new mount namespace
+	cmd := exec.Command(fanotifierBin, "fanotify", "/")
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Cloneflags: syscall.CLONE_NEWNS,
+	}
+	notifyR, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	notifyW, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	return &Fanotifier{
+		cmd: cmd,
+
+		// Connect to the spawned fanotifier over stdio
+		conn: conn.NewClient(notifyR, notifyW, cmd.Process.Pid, 5*time.Second),
+		closeFunc: func() error {
+			var errs []error
+			if err := notifyR.Close(); err != nil {
+				errs = append(errs, err)
+			}
+			if err := notifyW.Close(); err != nil {
+				errs = append(errs, err)
+			}
+			return errors.Join(errs...)
+		},
+	}, nil
+}
+
+// Start starts fanotifier in the new mount namespace
+func (n *Fanotifier) Start() error {
+	return n.conn.Start()
+}
+
+// GetPath gets path notified by the fanotifier.
+func (n *Fanotifier) GetPath() (string, error) {
+	return n.conn.GetPath()
+}
+
+// MountNamespacePath returns the path to the mount namespace where
+// the fanotifier is monitoring.
+func (n *Fanotifier) MountNamespacePath() string {
+	return fmt.Sprintf("/proc/%d/ns/mnt", n.cmd.Process.Pid)
+}
+
+// Close kills fanotifier process and closes the connection.
+func (n *Fanotifier) Close() (err error) {
+	n.closeOnce.Do(func() {
+		if err = n.cmd.Process.Kill(); err != nil {
+			return
+		}
+		err = n.closeFunc()
+	})
+	return
+}

analyzer/fanotify/service/service.go

@@ -0,0 +1,97 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package service
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"github.com/containerd/stargz-snapshotter/analyzer/fanotify/conn"
+	"golang.org/x/sys/unix"
+)
+
+// Serve starts fanotify at target directory and notifies all accessed files.
+// Passed io.Reader an io.Writer are used for communicating with the client.
+func Serve(target string, r io.Reader, w io.Writer) error {
+	sConn := conn.NewService(r, w, 5*time.Second)
+
+	fd, err := unix.FanotifyInit(unix.FAN_CLASS_NOTIF, unix.O_RDONLY)
+	if err != nil {
+		return fmt.Errorf("fanotify_init: %w", err)
+	}
+
+	// This blocks until the client tells us to start monitoring the target mountpoint.
+	if err := sConn.WaitStart(); err != nil {
+		return fmt.Errorf("waiting for start inst: %w", err)
+	}
+
+	// Start monitoring the target mountpoint.
+	if err := unix.FanotifyMark(fd,
+		unix.FAN_MARK_ADD|unix.FAN_MARK_MOUNT,
+		unix.FAN_ACCESS|unix.FAN_OPEN,
+		unix.AT_FDCWD,
+		target,
+	); err != nil {
+		return fmt.Errorf("fanotify_mark: %w", err)
+	}
+
+	// Notify "started" state to the client.
+	if err := sConn.SendStarted(); err != nil {
+		return fmt.Errorf("failed to send started message: %w", err)
+	}
+
+	nr := bufio.NewReader(os.NewFile(uintptr(fd), ""))
+	for {
+		event := &unix.FanotifyEventMetadata{}
+		if err := binary.Read(nr, binary.LittleEndian, event); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return fmt.Errorf("read fanotify fd: %w", err)
+		}
+		if event.Vers != unix.FANOTIFY_METADATA_VERSION {
+			return fmt.Errorf("fanotify version mismatch %d(got) != %d(want)",
+				event.Vers, unix.FANOTIFY_METADATA_VERSION)
+		}
+		if event.Fd < 0 {
+			// queue overflow
+			// TODO: do we need something special?
+			fmt.Fprintf(os.Stderr, "Warn: queue overflow")
+			continue
+		}
+
+		// Notify file descriptor.
+		// NOTE: There is no guarantee that we have /proc in this mount namespace
+		// (the target container's rootfs is mounted on "/") so we send file
+		// descriptor and let the client resolve the path of this file using /proc of
+		// this process.
+		if err := sConn.SendFd(int(event.Fd)); err != nil {
+			return fmt.Errorf("failed to send fd %d to client: %w", fd, err)
+		}
+		if err := unix.Close(int(event.Fd)); err != nil {
+			return fmt.Errorf("Close(fd): %w", err)
+		}
+
+		continue
+	}
+
+	return nil
+}

analyzer/option.go

@@ -0,0 +1,90 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package analyzer
+
+import (
+	"time"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/pkg/oci"
+)
+
+type analyzerOpts struct {
+	period       time.Duration
+	waitOnSignal bool
+	snapshotter  string
+	specOpts     SpecOpts
+	terminal     bool
+	stdin        bool
+	waitLineOut  string
+}
+
+// Option is runtime configuration of analyzer container
+type Option func(opts *analyzerOpts)
+
+// SpecOpts returns runtime configuration based on the passed image and rootfs path.
+type SpecOpts func(image containerd.Image, rootfs string) (opts []oci.SpecOpts, done func() error, err error)
+
+// WithSpecOpts is the runtime configuration
+func WithSpecOpts(specOpts SpecOpts) Option {
+	return func(opts *analyzerOpts) {
+		opts.specOpts = specOpts
+	}
+}
+
+// WithTerminal enable terminal for the container. This must be specified with WithStdin().
+func WithTerminal() Option {
+	return func(opts *analyzerOpts) {
+		opts.terminal = true
+	}
+}
+
+// WithStdin attaches stdin to the container
+func WithStdin() Option {
+	return func(opts *analyzerOpts) {
+		opts.stdin = true
+	}
+}
+
+// WithPeriod is the period to run runtime
+func WithPeriod(period time.Duration) Option {
+	return func(opts *analyzerOpts) {
+		opts.period = period
+	}
+}
+
+// WithWaitOnSignal disables timeout
+func WithWaitOnSignal() Option {
+	return func(opts *analyzerOpts) {
+		opts.waitOnSignal = true
+	}
+}
+
+// WithSnapshotter is the snapshotter to use
+func WithSnapshotter(snapshotter string) Option {
+	return func(opts *analyzerOpts) {
+		opts.snapshotter = snapshotter
+	}
+}
+
+// WithWaitLineOut specifies a substring of a stdout line to be waited.
+// When this line is detected, the container will be killed.
+func WithWaitLineOut(s string) Option {
+	return func(opts *analyzerOpts) {
+		opts.waitLineOut = s
+	}
+}

analyzer/recorder/recorder.go

@@ -0,0 +1,187 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package recorder
+
+import (
+	"archive/tar"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"sync"
+
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/images/converter/uncompress"
+	"github.com/containerd/containerd/v2/pkg/archive/compression"
+	"github.com/containerd/errdefs"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
+	"github.com/containerd/stargz-snapshotter/recorder"
+	"github.com/containerd/stargz-snapshotter/util/containerdutil"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/rs/xid"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	whiteoutPrefix    = ".wh."
+	whiteoutOpaqueDir = whiteoutPrefix + whiteoutPrefix + ".opq"
+)
+
+// ImageRecorder is a wrapper of recorder.Recroder. This holds the relationship
+// between files and layer index in the specified image. So the client can record
+// files without knowing about which layer this file belongs to.
+type ImageRecorder struct {
+	r              *recorder.Recorder
+	index          []map[string]struct{}
+	manifestDigest digest.Digest
+	recordW        content.Writer
+	recordWMu      sync.Mutex
+}
+
+func NewImageRecorder(ctx context.Context, cs content.Store, img images.Image, platformMC platforms.MatchComparer) (*ImageRecorder, error) {
+	manifestDesc, err := containerdutil.ManifestDesc(ctx, cs, img.Target, platformMC)
+	if err != nil {
+		return nil, err
+	}
+	p, err := content.ReadBlob(ctx, cs, manifestDesc)
+	if err != nil {
+		return nil, err
+	}
+	var manifest ocispec.Manifest
+	if err := json.Unmarshal(p, &manifest); err != nil {
+		return nil, err
+	}
+	return imageRecorderFromManifest(ctx, cs, manifestDesc, manifest)
+}
+
+func imageRecorderFromManifest(ctx context.Context, cs content.Store, manifestDesc ocispec.Descriptor, manifest ocispec.Manifest) (*ImageRecorder, error) {
+	var eg errgroup.Group
+	filesMap := make([]map[string]struct{}, len(manifest.Layers))
+	for i, desc := range manifest.Layers {
+		i, desc := i, desc
+		filesMap[i] = make(map[string]struct{})
+
+		// Create the index from the layer blob.
+		// TODO: During optimization, we uncompress the blob several times (here and during
+		//       creating eStargz layer). We should unify this process for better optimization
+		//       performance.
+		log.G(ctx).Infof("analyzing blob %q", desc.Digest)
+		readerAt, err := cs.ReaderAt(ctx, desc)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get reader blob %v: %w", desc.Digest, err)
+		}
+		defer readerAt.Close()
+		r := io.Reader(io.NewSectionReader(readerAt, 0, desc.Size))
+		if !uncompress.IsUncompressedType(desc.MediaType) {
+			r, err = compression.DecompressStream(r)
+			if err != nil {
+				return nil, fmt.Errorf("cannot decompress layer %v: %w", desc.Digest, err)
+			}
+		}
+		eg.Go(func() error {
+			tr := tar.NewReader(r)
+			for {
+				h, err := tr.Next()
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+					return err
+				}
+				filesMap[i][cleanEntryName(h.Name)] = struct{}{}
+			}
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return nil, err
+	}
+
+	recordW, err := content.OpenWriter(ctx, cs,
+		content.WithRef(fmt.Sprintf("recorder-%v", xid.New().String())))
+	if err != nil {
+		return nil, fmt.Errorf("failed to open writer for recorder: %w", err)
+	}
+	return &ImageRecorder{
+		r:              recorder.New(recordW),
+		index:          filesMap,
+		recordW:        recordW,
+		manifestDigest: manifestDesc.Digest,
+	}, nil
+}
+
+func (r *ImageRecorder) Record(name string) error {
+	if name == "" {
+		return nil
+	}
+	r.recordWMu.Lock()
+	defer r.recordWMu.Unlock()
+
+	name = cleanEntryName(name)
+	index := -1
+	for i := len(r.index) - 1; i >= 0; i-- {
+		if _, ok := r.index[i][name]; ok {
+			index = i
+			break
+		}
+
+		// If this is deleted by a whiteout file or directory, return error.
+		wh := cleanEntryName(path.Join(path.Dir("/"+name), whiteoutPrefix+path.Base("/"+name)))
+		if _, ok := r.index[i][wh]; ok {
+			return fmt.Errorf("%q is a deleted file", name)
+		}
+		whDir := cleanEntryName(path.Join(path.Dir("/"+name), whiteoutOpaqueDir))
+		if _, ok := r.index[i][whDir]; ok {
+			return fmt.Errorf("parent dir of %q is a deleted directory", name)
+		}
+	}
+	if index < 0 {
+		return fmt.Errorf("file %q not found in index", name)
+	}
+	return r.r.Record(&recorder.Entry{
+		Path:           name,
+		ManifestDigest: r.manifestDigest.String(),
+		LayerIndex:     &index,
+	})
+}
+
+func (r *ImageRecorder) Commit(ctx context.Context) (digest.Digest, error) {
+	r.recordWMu.Lock()
+	defer r.recordWMu.Unlock()
+
+	if err := r.recordW.Commit(ctx, 0, ""); err != nil && !errdefs.IsAlreadyExists(err) {
+		return "", err
+	}
+	return r.recordW.Digest(), nil
+}
+
+func (r *ImageRecorder) Close() error {
+	r.recordWMu.Lock()
+	defer r.recordWMu.Unlock()
+
+	return r.recordW.Close()
+}
+
+func cleanEntryName(name string) string {
+	// Use path.Clean to consistently deal with path separators across platforms.
+	return strings.TrimPrefix(path.Clean("/"+name), "/")
+}

analyzer/recorder/recorder_test.go

@@ -0,0 +1,327 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package recorder
+
+import (
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/plugins/content/local"
+	"github.com/containerd/errdefs"
+	"github.com/containerd/stargz-snapshotter/recorder"
+	"github.com/containerd/stargz-snapshotter/util/testutil"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/rs/xid"
+)
+
+var allowedPrefix = [4]string{"", "./", "/", "../"}
+
+func TestNodeIndex(t *testing.T) {
+	type recordEntry struct {
+		name  string
+		index int
+	}
+	tests := []struct {
+		name     string
+		in       [][]testutil.TarEntry
+		record   []string
+		want     []recordEntry
+		wantFail bool
+	}{
+		{
+			name: "single layer",
+			in: [][]testutil.TarEntry{
+				{
+					testutil.File("foo", "foo"),
+					testutil.File("bar", "bar"),
+				},
+			},
+			record: []string{"bar", "foo"},
+			want: []recordEntry{
+				{"bar", 0},
+				{"foo", 0},
+			},
+		},
+		{
+			name: "overlay",
+			in: [][]testutil.TarEntry{
+				{
+					testutil.File("foo", "foo"),
+					testutil.Dir("bar/"),
+					testutil.Dir("bar/barbar/"),
+					testutil.File("bar/barbar/bar2", "bar2"),
+					testutil.File("bar/barbar/foo2", "foo2"),
+					testutil.File("x", "x"),
+					testutil.File("y", "y"),
+				},
+				{
+					testutil.File("foo", "foo2"),
+					testutil.File("baz", "baz"),
+					testutil.Dir("bar/"),
+					testutil.Dir("bar/barbar/"),
+					testutil.File("bar/barbar/foo2", "foo2-upper"),
+				},
+				{
+					testutil.File("foo", "foo3"),
+					testutil.Dir("a/"),
+					testutil.File("a/aaaaaa", "a"),
+					testutil.File("y", "y"),
+				},
+			},
+			record: []string{
+				"baz",
+				"foo",
+				"bar/barbar/foo2",
+				"a/aaaaaa",
+				"bar/barbar/bar2",
+				"y"},
+			want: []recordEntry{
+				{"baz", 1},
+				{"foo", 2},
+				{"bar/barbar/foo2", 1},
+				{"a/aaaaaa", 2},
+				{"bar/barbar/bar2", 0},
+				{"y", 2},
+			},
+		},
+		{
+			name: "various files",
+			in: [][]testutil.TarEntry{
+				// All file path should be interpreted as the path relative to root.
+				{
+					testutil.File("foo", "foo"),
+					testutil.Dir("bar/"),
+					testutil.Symlink("bar/foosym", "foo"),
+					testutil.Dir("bar/barbar/"),
+					testutil.File("./bar/barbar/bar2", "bar2"),
+					testutil.File("../bar/barbar/foo2", "foo2"),
+					testutil.File("/x", "x"),
+					testutil.Chardev("chardev", 1, 10),
+					testutil.Fifo("bar/fifo1"),
+				},
+				{
+					testutil.File("./foo", "foo2"),
+					testutil.File("baz", "baz"),
+					testutil.Dir("bar/"),
+					testutil.Dir("./bar/barbar/"),
+					testutil.File("bar/barbar/bazlink", "baz"),
+					testutil.File("../bar/barbar/foo2", "foo2-upper"),
+					testutil.Chardev("chardev", 10, 100),
+					testutil.Chardev("./blockdev", 100, 1),
+				},
+			},
+			record: []string{
+				// All file path should be interpreted as the path relative to root.
+				"./bar/foosym",
+				"bar/barbar/bazlink",
+				"/bar/barbar/foo2",
+				"chardev",
+				"../blockdev",
+				"bar/fifo1",
+				"./bar/barbar/bar2",
+			},
+			want: []recordEntry{
+				{"bar/foosym", 0},
+				{"bar/barbar/bazlink", 1},
+				{"bar/barbar/foo2", 1},
+				{"chardev", 1},
+				{"blockdev", 1},
+				{"bar/fifo1", 0},
+				{"bar/barbar/bar2", 0},
+			},
+		},
+		{
+			name: "whiteout file",
+			in: [][]testutil.TarEntry{
+				{
+					testutil.Dir("bar/"),
+					testutil.File("bar/barfile", "bar"),
+				},
+				{
+					testutil.Dir("bar/"),
+					testutil.File(path.Join("bar", whiteoutPrefix+"barfile"), "bar"),
+				},
+			},
+			record:   []string{"bar/barfile"},
+			want:     []recordEntry{},
+			wantFail: true,
+		},
+		{
+			name: "whiteout dir",
+			in: [][]testutil.TarEntry{
+				{
+					testutil.Dir("bar/"),
+					testutil.File("bar/barfile", "bar"),
+				},
+				{
+					testutil.Dir("bar/"),
+					testutil.Dir(path.Join("bar", whiteoutOpaqueDir) + "/"),
+				},
+			},
+			record:   []string{"bar/barfile"},
+			want:     []recordEntry{},
+			wantFail: true,
+		},
+		{
+			name: "whiteout dir",
+			in: [][]testutil.TarEntry{
+				{
+					testutil.Dir("bar/"),
+					testutil.File("bar/barfile", "bar"),
+				},
+				{
+					testutil.Dir("bar/"),
+					testutil.Dir(path.Join("bar", whiteoutOpaqueDir) + "/"),
+				},
+			},
+			record:   []string{"bar/barfile"},
+			want:     []recordEntry{},
+			wantFail: true,
+		},
+	}
+
+	tempDir, err := os.MkdirTemp("", "test-recorder")
+	if err != nil {
+		t.Fatalf("failed to prepare content store dir: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+	cs, err := local.NewStore(tempDir)
+	if err != nil {
+		t.Fatalf("failed to prepare content store: %v", err)
+	}
+	compressWrappers := map[string]func(r io.Reader) io.Reader{
+		ocispec.MediaTypeImageLayer:     func(r io.Reader) io.Reader { return r }, // nop (uncompressed)
+		ocispec.MediaTypeImageLayerGzip: gzipCompress,                             // gzip compression
+	}
+	ctx := context.Background()
+	for _, tt := range tests {
+		for _, prefix := range allowedPrefix {
+			prefix := prefix
+			for mediatype, cWrapper := range compressWrappers {
+				t.Run(tt.name+":"+mediatype+",prefix="+prefix, func(t *testing.T) {
+					var layers []ocispec.Descriptor
+					for _, es := range tt.in {
+						ref := fmt.Sprintf("recorder-test-%v", xid.New().String())
+						lw, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
+						if err != nil {
+							t.Errorf("failed to open writer: %v", err)
+							return
+						}
+						tarR := testutil.BuildTar(es, testutil.WithPrefix(prefix))
+						if _, err := io.Copy(lw, cWrapper(tarR)); err != nil {
+							t.Errorf("failed to copy layer: %v", err)
+							return
+						}
+						if err := lw.Commit(ctx, 0, ""); err != nil && !errdefs.IsAlreadyExists(err) {
+							t.Errorf("failed to commit layer: %v", err)
+							return
+						}
+						info, err := cs.Info(ctx, lw.Digest())
+						if err != nil {
+							t.Errorf("failed to get layer info: %v", err)
+							return
+						}
+						// TODO: check compressed version as well
+						layers = append(layers, ocispec.Descriptor{
+							Digest:    info.Digest,
+							Size:      info.Size,
+							MediaType: mediatype,
+						})
+					}
+					ir, err := imageRecorderFromManifest(ctx, cs,
+						ocispec.Descriptor{}, ocispec.Manifest{Layers: layers})
+					if err != nil {
+						t.Errorf("failed to get recorder: %v", err)
+						return
+					}
+					defer ir.Close()
+					fail := false
+					for _, name := range tt.record {
+						if err := ir.Record(name); err != nil {
+							fail = true
+							t.Logf("failed to record: %q: %v", name, err)
+							break
+						}
+					}
+					if tt.wantFail != fail {
+						t.Errorf("unexpected record result: fail = %v; wantFail = %v", fail, tt.wantFail)
+						return
+					}
+					if tt.wantFail {
+						return // no need to check the record out
+					}
+					recordOut, err := ir.Commit(ctx)
+					if err != nil {
+						t.Errorf("failed to commit record: %v", err)
+						return
+					}
+
+					ra, err := cs.ReaderAt(ctx, ocispec.Descriptor{Digest: recordOut})
+					if err != nil {
+						t.Errorf("failed to get record out: %v", err)
+						return
+					}
+					defer ra.Close()
+					dec := json.NewDecoder(io.NewSectionReader(ra, 0, ra.Size()))
+					i := 0
+					for dec.More() {
+						var e recorder.Entry
+						if err := dec.Decode(&e); err != nil {
+							t.Errorf("failed to decord record: %v", err)
+							return
+						}
+						if len(tt.want) <= i {
+							t.Errorf("too many records: %d th but want %d", i, len(tt.want))
+							return
+						}
+						if e.Path != tt.want[i].name || *e.LayerIndex != tt.want[i].index {
+							t.Errorf("unexpected entry { name = %q, index = %d }; want { name = %q, index = %d }", e.Path, *e.LayerIndex, tt.want[i].name, tt.want[i].index)
+							return
+						}
+						i++
+					}
+					if i < len(tt.want) {
+						t.Errorf("record is too short: %d; but want %d", i, len(tt.want))
+						return
+					}
+				})
+			}
+		}
+	}
+}
+
+func gzipCompress(r io.Reader) io.Reader {
+	pr, pw := io.Pipe()
+	go func() {
+		gw := gzip.NewWriter(pw)
+		if _, err := io.Copy(gw, r); err != nil {
+			gw.Close()
+			pw.CloseWithError(err)
+			return
+		}
+		gw.Close()
+		pw.Close()
+	}()
+	return pr
+}

cache/cache.go

@@ -18,14 +18,16 @@ package cache
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"sync"
 
-	"github.com/golang/groupcache/lru"
-	"github.com/pkg/errors"
+	"github.com/containerd/stargz-snapshotter/util/cacheutil"
+	"github.com/containerd/stargz-snapshotter/util/namedmutex"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -34,28 +36,81 @@ const (
 )
 
 type DirectoryCacheConfig struct {
+
+	// Number of entries of LRU cache (default: 10).
+	// This won't be used when DataCache is specified.
 	MaxLRUCacheEntry int
-	MaxCacheFds      int
-	SyncAdd          bool
+
+	// Number of file descriptors to cache (default: 10).
+	// This won't be used when FdCache is specified.
+	MaxCacheFds int
+
+	// On Add, wait until the data is fully written to the cache directory.
+	SyncAdd bool
+
+	// DataCache is an on-memory cache of the data.
+	// OnEvicted will be overridden and replaced for internal use.
+	DataCache *cacheutil.LRUCache
+
+	// FdCache is a cache for opened file descriptors.
+	// OnEvicted will be overridden and replaced for internal use.
+	FdCache *cacheutil.LRUCache
+
+	// BufPool will be used for pooling bytes.Buffer.
+	BufPool *sync.Pool
+
+	// Direct forcefully enables direct mode for all operation in cache.
+	// Thus operation won't use on-memory caches.
+	Direct bool
+
+	// FadvDontNeed forcefully clean fscache pagecache for saving memory.
+	FadvDontNeed bool
 }
 
 // TODO: contents validation.
 
+// BlobCache represents a cache for bytes data
 type BlobCache interface {
-	Add(key string, p []byte, opts ...Option)
-	FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error)
+	// Add returns a writer to add contents to cache
+	Add(key string, opts ...Option) (Writer, error)
+
+	// Get returns a reader to read the specified contents
+	// from cache
+	Get(key string, opts ...Option) (Reader, error)
+
+	// Close closes the cache
+	Close() error
+}
+
+// Reader provides the data cached.
+type Reader interface {
+	io.ReaderAt
+	Close() error
+
+	// If a blob is backed by a file, it should return *os.File so that it can be used for FUSE passthrough
+	GetReaderAt() io.ReaderAt
+}
+
+// Writer enables the client to cache byte data. Commit() must be
+// called after data is fully written to Write(). To abort the written
+// data, Abort() must be called.
+type Writer interface {
+	io.WriteCloser
+	Commit() error
+	Abort() error
 }
 
 type cacheOpt struct {
-	direct bool
+	direct      bool
+	passThrough bool
 }
 
 type Option func(o *cacheOpt) *cacheOpt
 
-// When Direct option is specified for FetchAt and Add methods, these operation
-// won't use on-memory caches. When you know that the targeting value won't be
-// used immediately, you can prevent the limited space of on-memory caches from
-// being polluted by these unimportant values.
+// Direct option lets FetchAt and Add methods not to use on-memory caches. When
+// you know that the targeting value won't be  used immediately, you can prevent
+// the limited space of on-memory caches from being polluted by these unimportant
+// values.
 func Direct() Option {
 	return func(o *cacheOpt) *cacheOpt {
 		o.direct = true
@@ -63,34 +118,66 @@ func Direct() Option {
 	}
 }
 
+// PassThrough option indicates whether to enable FUSE passthrough mode
+// to improve local file read performance.
+func PassThrough() Option {
+	return func(o *cacheOpt) *cacheOpt {
+		o.passThrough = true
+		return o
+	}
+}
+
 func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache, error) {
-	maxEntry := config.MaxLRUCacheEntry
-	if maxEntry == 0 {
-		maxEntry = defaultMaxLRUCacheEntry
+	if !filepath.IsAbs(directory) {
+		return nil, fmt.Errorf("dir cache path must be an absolute path; got %q", directory)
 	}
-	maxFds := config.MaxCacheFds
-	if maxFds == 0 {
-		maxFds = defaultMaxCacheFds
-	}
-	if err := os.MkdirAll(directory, os.ModePerm); err != nil {
-		return nil, err
-	}
-	dc := &directoryCache{
-		cache:     newObjectCache(maxEntry),
-		fileCache: newObjectCache(maxFds),
-		wipLock:   &namedLock{},
-		directory: directory,
-		bufPool: sync.Pool{
+	bufPool := config.BufPool
+	if bufPool == nil {
+		bufPool = &sync.Pool{
 			New: func() interface{} {
 				return new(bytes.Buffer)
 			},
-		},
+		}
 	}
-	dc.cache.finalize = func(value interface{}) {
-		dc.bufPool.Put(value)
+	dataCache := config.DataCache
+	if dataCache == nil {
+		maxEntry := config.MaxLRUCacheEntry
+		if maxEntry == 0 {
+			maxEntry = defaultMaxLRUCacheEntry
+		}
+		dataCache = cacheutil.NewLRUCache(maxEntry)
+		dataCache.OnEvicted = func(key string, value interface{}) {
+			value.(*bytes.Buffer).Reset()
+			bufPool.Put(value)
+		}
 	}
-	dc.fileCache.finalize = func(value interface{}) {
-		value.(*os.File).Close()
+	fdCache := config.FdCache
+	if fdCache == nil {
+		maxEntry := config.MaxCacheFds
+		if maxEntry == 0 {
+			maxEntry = defaultMaxCacheFds
+		}
+		fdCache = cacheutil.NewLRUCache(maxEntry)
+		fdCache.OnEvicted = func(key string, value interface{}) {
+			value.(*os.File).Close()
+		}
+	}
+	if err := os.MkdirAll(directory, 0700); err != nil {
+		return nil, err
+	}
+	wipdir := filepath.Join(directory, "wip")
+	if err := os.MkdirAll(wipdir, 0700); err != nil {
+		return nil, err
+	}
+	dc := &directoryCache{
+		cache:        dataCache,
+		fileCache:    fdCache,
+		wipLock:      new(namedmutex.NamedMutex),
+		directory:    directory,
+		wipDirectory: wipdir,
+		bufPool:      bufPool,
+		direct:       config.Direct,
+		fadvDontNeed: config.FadvDontNeed,
 	}
 	dc.syncAdd = config.SyncAdd
 	return dc, nil
@@ -98,38 +185,53 @@ func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache
 
 // directoryCache is a cache implementation which backend is a directory.
 type directoryCache struct {
-	cache     *objectCache
-	fileCache *objectCache
-	directory string
-	wipLock   *namedLock
+	cache        *cacheutil.LRUCache
+	fileCache    *cacheutil.LRUCache
+	wipDirectory string
+	directory    string
+	wipLock      *namedmutex.NamedMutex
 
-	bufPool sync.Pool
+	bufPool *sync.Pool
 
-	syncAdd bool
+	syncAdd      bool
+	direct       bool
+	fadvDontNeed bool
+
+	closed   bool
+	closedMu sync.Mutex
 }
 
-func (dc *directoryCache) FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error) {
+func (dc *directoryCache) Get(key string, opts ...Option) (Reader, error) {
+	if dc.isClosed() {
+		return nil, fmt.Errorf("cache is already closed")
+	}
+
 	opt := &cacheOpt{}
 	for _, o := range opts {
 		opt = o(opt)
 	}
 
-	if !opt.direct {
+	if !dc.direct && !opt.direct {
 		// Get data from memory
-		if b, done, ok := dc.cache.get(key); ok {
-			defer done()
-			data := b.(*bytes.Buffer).Bytes()
-			if int64(len(data)) < offset {
-				return 0, fmt.Errorf("invalid offset %d exceeds chunk size %d",
-					offset, len(data))
-			}
-			return copy(p, data[offset:]), nil
+		if b, done, ok := dc.cache.Get(key); ok {
+			return &reader{
+				ReaderAt: bytes.NewReader(b.(*bytes.Buffer).Bytes()),
+				closeFunc: func() error {
+					done()
+					return nil
+				},
+			}, nil
 		}
 
 		// Get data from disk. If the file is already opened, use it.
-		if f, done, ok := dc.fileCache.get(key); ok {
-			defer done()
-			return f.(*os.File).ReadAt(p, offset)
+		if f, done, ok := dc.fileCache.Get(key); ok {
+			return &reader{
+				ReaderAt: f.(*os.File),
+				closeFunc: func() error {
+					done() // file will be closed when it's evicted from the cache
+					return nil
+				},
+			}, nil
 		}
 	}
 
@@ -138,257 +240,258 @@ func (dc *directoryCache) FetchAt(key string, offset int64, p []byte, opts ...Op
 	//       or simply report the cache miss?
 	file, err := os.Open(dc.cachePath(key))
 	if err != nil {
-		return 0, errors.Wrapf(err, "failed to open blob file for %q", key)
-	}
-	if n, err = file.ReadAt(p, offset); err == io.EOF {
-		err = nil
+		return nil, fmt.Errorf("failed to open blob file for %q: %w", key, err)
 	}
 
-	// Cache the opened file for future use. If "direct" option is specified, this
-	// won't be done. This option is useful for preventing file cache from being
-	// polluted by data that won't be accessed immediately.
-	if opt.direct || !dc.fileCache.add(key, file) {
-		file.Close()
+	// If "direct" option is specified, do not cache the file on memory.
+	// This option is useful for preventing memory cache from being polluted by data
+	// that won't be accessed immediately.
+	if dc.direct || opt.direct {
+		return &reader{
+			ReaderAt: file,
+			closeFunc: func() error {
+				if dc.fadvDontNeed {
+					if err := dropFilePageCache(file); err != nil {
+						fmt.Printf("Warning: failed to drop page cache: %v\n", err)
+					}
+				}
+
+				// In passthough model, close will be toke over by go-fuse
+				// If "passThrough" option is specified, "direct" option also will
+				// be specified, so adding this branch here is enough
+				if opt.passThrough {
+					return nil
+				}
+				return file.Close()
+			},
+		}, nil
 	}
 
 	// TODO: should we cache the entire file data on memory?
 	//       but making I/O (possibly huge) on every fetching
 	//       might be costly.
-
-	return n, err
+	return &reader{
+		ReaderAt: file,
+		closeFunc: func() error {
+			_, done, added := dc.fileCache.Add(key, file)
+			defer done() // Release it immediately. Cleaned up on eviction.
+			if !added {
+				return file.Close() // file already exists in the cache. close it.
+			}
+			return nil
+		},
+	}, nil
 }
 
-func (dc *directoryCache) Add(key string, p []byte, opts ...Option) {
+func (dc *directoryCache) Add(key string, opts ...Option) (Writer, error) {
+	if dc.isClosed() {
+		return nil, fmt.Errorf("cache is already closed")
+	}
+
 	opt := &cacheOpt{}
 	for _, o := range opts {
 		opt = o(opt)
 	}
 
-	if !opt.direct {
-		// Cache the passed data on memory. This enables to serve this data even
-		// during writing it to the disk. If "direct" option is specified, this
-		// won't be done. This option is useful for preventing memory cache from being
-		// polluted by data that won't be accessed immediately.
-		b := dc.bufPool.Get().(*bytes.Buffer)
-		b.Reset()
-		b.Write(p)
-		if !dc.cache.add(key, b) {
-			dc.bufPool.Put(b) // Already exists. No need to cache.
-		}
+	wip, err := dc.wipFile(key)
+	if err != nil {
+		return nil, err
+	}
+	w := &writer{
+		WriteCloser: wip,
+		commitFunc: func() error {
+			if dc.isClosed() {
+				return fmt.Errorf("cache is already closed")
+			}
+			// Commit the cache contents
+			c := dc.cachePath(key)
+			if err := os.MkdirAll(filepath.Dir(c), os.ModePerm); err != nil {
+				var errs []error
+				if err := os.Remove(wip.Name()); err != nil {
+					errs = append(errs, err)
+				}
+				errs = append(errs, fmt.Errorf("failed to create cache directory %q: %w", c, err))
+				return errors.Join(errs...)
+			}
+
+			if dc.fadvDontNeed {
+				if err := dropFilePageCache(wip); err != nil {
+					fmt.Printf("Warning: failed to drop page cache: %v\n", err)
+				}
+			}
+
+			return os.Rename(wip.Name(), c)
+		},
+		abortFunc: func() error {
+			return os.Remove(wip.Name())
+		},
 	}
 
-	// Cache the passed data to disk.
-	b2 := dc.bufPool.Get().(*bytes.Buffer)
-	b2.Reset()
-	b2.Write(p)
-	addFunc := func() {
-		defer dc.bufPool.Put(b2)
-
-		var (
-			c   = dc.cachePath(key)
-			wip = dc.wipPath(key)
-		)
-
-		dc.wipLock.lock(key)
-		if _, err := os.Stat(wip); err == nil {
-			dc.wipLock.unlock(key)
-			return // Write in progress
-		}
-		if _, err := os.Stat(c); err == nil {
-			dc.wipLock.unlock(key)
-			return // Already exists.
-		}
-
-		// Write the contents to a temporary file
-		if err := os.MkdirAll(filepath.Dir(wip), os.ModePerm); err != nil {
-			fmt.Printf("Warning: Failed to Create blob cache directory %q: %v\n", c, err)
-			dc.wipLock.unlock(key)
-			return
-		}
-		wipfile, err := os.Create(wip)
-		if err != nil {
-			fmt.Printf("Warning: failed to prepare temp file for storing cache %q", key)
-			dc.wipLock.unlock(key)
-			return
-		}
-		dc.wipLock.unlock(key)
-
-		defer func() {
-			wipfile.Close()
-			os.Remove(wipfile.Name())
-		}()
-		want := b2.Len()
-		if _, err := io.CopyN(wipfile, b2, int64(want)); err != nil {
-			fmt.Printf("Warning: failed to write cache: %v\n", err)
-			return
-		}
-
-		// Commit the cache contents
-		if err := os.MkdirAll(filepath.Dir(c), os.ModePerm); err != nil {
-			fmt.Printf("Warning: Failed to Create blob cache directory %q: %v\n", c, err)
-			return
-		}
-		if err := os.Rename(wipfile.Name(), c); err != nil {
-			fmt.Printf("Warning: failed to commit cache to %q: %v\n", c, err)
-			return
-		}
-		file, err := os.Open(c)
-		if err != nil {
-			fmt.Printf("Warning: failed to open cache on %q: %v\n", c, err)
-			return
-		}
-
-		// Cache the opened file for future use. If "direct" option is specified, this
-		// won't be done. This option is useful for preventing file cache from being
-		// polluted by data that won't be accessed immediately.
-		if opt.direct || !dc.fileCache.add(key, file) {
-			file.Close()
-		}
+	// If "direct" option is specified, do not cache the passed data on memory.
+	// This option is useful for preventing memory cache from being polluted by data
+	// that won't be accessed immediately.
+	if dc.direct || opt.direct {
+		return w, nil
 	}
 
-	if dc.syncAdd {
-		addFunc()
-	} else {
-		go addFunc()
+	b := dc.bufPool.Get().(*bytes.Buffer)
+	memW := &writer{
+		WriteCloser: nopWriteCloser(io.Writer(b)),
+		commitFunc: func() error {
+			if dc.isClosed() {
+				w.Close()
+				return fmt.Errorf("cache is already closed")
+			}
+			cached, done, added := dc.cache.Add(key, b)
+			if !added {
+				dc.putBuffer(b) // already exists in the cache. abort it.
+			}
+			commit := func() error {
+				defer done()
+				defer w.Close()
+				n, err := w.Write(cached.(*bytes.Buffer).Bytes())
+				if err != nil || n != cached.(*bytes.Buffer).Len() {
+					w.Abort()
+					return err
+				}
+				return w.Commit()
+			}
+			if dc.syncAdd {
+				return commit()
+			}
+			go func() {
+				if err := commit(); err != nil {
+					fmt.Println("failed to commit to file:", err)
+				}
+			}()
+			return nil
+		},
+		abortFunc: func() error {
+			defer w.Close()
+			defer w.Abort()
+			dc.putBuffer(b) // abort it.
+			return nil
+		},
 	}
+
+	return memW, nil
+}
+
+func (dc *directoryCache) putBuffer(b *bytes.Buffer) {
+	b.Reset()
+	dc.bufPool.Put(b)
+}
+
+func (dc *directoryCache) Close() error {
+	dc.closedMu.Lock()
+	defer dc.closedMu.Unlock()
+	if dc.closed {
+		return nil
+	}
+	dc.closed = true
+	return os.RemoveAll(dc.directory)
+}
+
+func (dc *directoryCache) isClosed() bool {
+	dc.closedMu.Lock()
+	closed := dc.closed
+	dc.closedMu.Unlock()
+	return closed
 }
 
 func (dc *directoryCache) cachePath(key string) string {
 	return filepath.Join(dc.directory, key[:2], key)
 }
 
-func (dc *directoryCache) wipPath(key string) string {
-	return filepath.Join(dc.directory, key[:2], "w", key)
-}
-
-type namedLock struct {
-	muMap  map[string]*sync.Mutex
-	refMap map[string]int
-
-	mu sync.Mutex
-}
-
-func (nl *namedLock) lock(name string) {
-	nl.mu.Lock()
-	if nl.muMap == nil {
-		nl.muMap = make(map[string]*sync.Mutex)
-	}
-	if nl.refMap == nil {
-		nl.refMap = make(map[string]int)
-	}
-	if _, ok := nl.muMap[name]; !ok {
-		nl.muMap[name] = &sync.Mutex{}
-	}
-	mu := nl.muMap[name]
-	nl.refMap[name]++
-	nl.mu.Unlock()
-	mu.Lock()
-}
-
-func (nl *namedLock) unlock(name string) {
-	nl.mu.Lock()
-	mu := nl.muMap[name]
-	nl.refMap[name]--
-	if nl.refMap[name] <= 0 {
-		delete(nl.muMap, name)
-		delete(nl.refMap, name)
-	}
-	nl.mu.Unlock()
-	mu.Unlock()
-}
-
-func newObjectCache(maxEntries int) *objectCache {
-	oc := &objectCache{
-		cache: lru.New(maxEntries),
-	}
-	oc.cache.OnEvicted = func(key lru.Key, value interface{}) {
-		value.(*object).release() // Decrease ref count incremented in add operation.
-	}
-	return oc
-}
-
-type objectCache struct {
-	cache    *lru.Cache
-	cacheMu  sync.Mutex
-	finalize func(interface{})
-}
-
-func (oc *objectCache) get(key string) (value interface{}, done func(), ok bool) {
-	oc.cacheMu.Lock()
-	defer oc.cacheMu.Unlock()
-	o, ok := oc.cache.Get(key)
-	if !ok {
-		return nil, nil, false
-	}
-	o.(*object).use()
-	return o.(*object).v, func() { o.(*object).release() }, true
-}
-
-func (oc *objectCache) add(key string, value interface{}) bool {
-	oc.cacheMu.Lock()
-	defer oc.cacheMu.Unlock()
-	if _, ok := oc.cache.Get(key); ok {
-		return false // TODO: should we swap the object?
-	}
-	o := &object{
-		v:        value,
-		finalize: oc.finalize,
-	}
-	o.use() // Keep this object having at least 1 ref count (will be decreased on eviction)
-	oc.cache.Add(key, o)
-	return true
-}
-
-type object struct {
-	v interface{}
-
-	refCounts int64
-	finalize  func(interface{})
-
-	mu sync.Mutex
-}
-
-func (o *object) use() {
-	o.mu.Lock()
-	defer o.mu.Unlock()
-	o.refCounts++
-}
-
-func (o *object) release() {
-	o.mu.Lock()
-	defer o.mu.Unlock()
-	o.refCounts--
-	if o.refCounts <= 0 && o.finalize != nil {
-		// nobody will refer this object
-		o.finalize(o.v)
-	}
+func (dc *directoryCache) wipFile(key string) (*os.File, error) {
+	return os.CreateTemp(dc.wipDirectory, key+"-*")
 }
 
 func NewMemoryCache() BlobCache {
-	return &memoryCache{
-		membuf: map[string]string{},
+	return &MemoryCache{
+		Membuf: map[string]*bytes.Buffer{},
 	}
 }
 
-// memoryCache is a cache implementation which backend is a memory.
-type memoryCache struct {
-	membuf map[string]string // read-only []byte map is more ideal but we don't have it in golang...
+// MemoryCache is a cache implementation which backend is a memory.
+type MemoryCache struct {
+	Membuf map[string]*bytes.Buffer
 	mu     sync.Mutex
 }
 
-func (mc *memoryCache) FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error) {
+func (mc *MemoryCache) Get(key string, opts ...Option) (Reader, error) {
 	mc.mu.Lock()
 	defer mc.mu.Unlock()
-
-	cache, ok := mc.membuf[key]
+	b, ok := mc.Membuf[key]
 	if !ok {
-		return 0, fmt.Errorf("Missed cache: %q", key)
+		return nil, fmt.Errorf("missed cache: %q", key)
 	}
-	return copy(p, cache[offset:]), nil
+	return &reader{bytes.NewReader(b.Bytes()), func() error { return nil }}, nil
 }
 
-func (mc *memoryCache) Add(key string, p []byte, opts ...Option) {
-	mc.mu.Lock()
-	defer mc.mu.Unlock()
-	mc.membuf[key] = string(p)
+func (mc *MemoryCache) Add(key string, opts ...Option) (Writer, error) {
+	b := new(bytes.Buffer)
+	return &writer{
+		WriteCloser: nopWriteCloser(io.Writer(b)),
+		commitFunc: func() error {
+			mc.mu.Lock()
+			defer mc.mu.Unlock()
+			mc.Membuf[key] = b
+			return nil
+		},
+		abortFunc: func() error { return nil },
+	}, nil
+}
+
+func (mc *MemoryCache) Close() error {
+	return nil
+}
+
+type reader struct {
+	io.ReaderAt
+	closeFunc func() error
+}
+
+func (r *reader) Close() error { return r.closeFunc() }
+
+func (r *reader) GetReaderAt() io.ReaderAt {
+	return r.ReaderAt
+}
+
+type writer struct {
+	io.WriteCloser
+	commitFunc func() error
+	abortFunc  func() error
+}
+
+func (w *writer) Commit() error {
+	return w.commitFunc()
+}
+
+func (w *writer) Abort() error {
+	return w.abortFunc()
+}
+
+type writeCloser struct {
+	io.Writer
+	closeFunc func() error
+}
+
+func (w *writeCloser) Close() error { return w.closeFunc() }
+
+func nopWriteCloser(w io.Writer) io.WriteCloser {
+	return &writeCloser{w, func() error { return nil }}
+}
+
+func dropFilePageCache(file *os.File) error {
+	if file == nil {
+		return nil
+	}
+
+	fd := file.Fd()
+	err := unix.Fadvise(int(fd), 0, 0, unix.FADV_DONTNEED)
+	if err != nil {
+		return fmt.Errorf("posix_fadvise failed, ret=%d", err)
+	}
+	return nil
 }

cache/cache_test.go

@@ -25,7 +25,7 @@ package cache
 import (
 	"crypto/sha256"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"testing"
 )
@@ -38,7 +38,7 @@ func TestDirectoryCache(t *testing.T) {
 
 	// with enough memory cache
 	newCache := func() (BlobCache, cleanFunc) {
-		tmp, err := ioutil.TempDir("", "testcache")
+		tmp, err := os.MkdirTemp("", "testcache")
 		if err != nil {
 			t.Fatalf("failed to make tempdir: %v", err)
 		}
@@ -55,7 +55,7 @@ func TestDirectoryCache(t *testing.T) {
 
 	// with smaller memory cache
 	newCache = func() (BlobCache, cleanFunc) {
-		tmp, err := ioutil.TempDir("", "testcache")
+		tmp, err := os.MkdirTemp("", "testcache")
 		if err != nil {
 			t.Fatalf("failed to make tempdir: %v", err)
 		}
@@ -132,7 +132,19 @@ func testCache(t *testing.T, name string, newCache func() (BlobCache, cleanFunc)
 			defer clean()
 			for _, blob := range tt.blobs {
 				d := digestFor(blob)
-				c.Add(d, []byte(blob))
+				w, err := c.Add(d)
+				if err != nil {
+					t.Fatalf("failed to add %v: %v", d, err)
+				}
+				if n, err := w.Write([]byte(blob)); err != nil || n != len(blob) {
+					w.Close()
+					t.Fatalf("failed to write %v (len:%d): %v", d, len(blob), err)
+				}
+				if err := w.Commit(); err != nil {
+					w.Close()
+					t.Fatalf("failed to commit %v (len:%d): %v", d, len(blob), err)
+				}
+				w.Close()
 			}
 			for _, check := range tt.checks {
 				check(t, c)
@@ -162,7 +174,12 @@ func hit(sample string) check {
 
 func testChunk(t *testing.T, c BlobCache, key string, offset int64, sample string) {
 	p := make([]byte, len(sample))
-	if n, err := c.FetchAt(key, offset, p); err != nil {
+	r, err := c.Get(key)
+	if err != nil {
+		t.Errorf("missed %v", key)
+		return
+	}
+	if n, err := r.ReadAt(p, offset); err != nil && err != io.EOF {
 		t.Errorf("failed to fetch blob %q: %v", key, err)
 		return
 	} else if n != len(sample) {
@@ -177,9 +194,7 @@ func testChunk(t *testing.T, c BlobCache, key string, offset int64, sample strin
 func miss(sample string) check {
 	return func(t *testing.T, c BlobCache) {
 		d := digestFor(sample)
-		p := make([]byte, len(sample))
-		_, err := c.FetchAt(d, 0, p)
-		if err == nil {
+		if _, err := c.Get(d); err == nil {
 			t.Errorf("hit blob %q but must be missed: %v", d, err)
 			return
 		}

cmd/containerd-stargz-grpc/config.go

@@ -1,47 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package main
-
-import "github.com/containerd/stargz-snapshotter/fs/config"
-
-type Config struct {
-	config.Config
-
-	// KubeconfigKeychainConfig is config for kubeconfig-based keychain.
-	KubeconfigKeychainConfig `toml:"kubeconfig_keychain"`
-
-	// ResolverConfig is config for resolving registries.
-	ResolverConfig `toml:"resolver"`
-}
-
-type KubeconfigKeychainConfig struct {
-	EnableKeychain bool   `toml:"enable_keychain"`
-	KubeconfigPath string `toml:"kubeconfig_path"`
-}
-
-type ResolverConfig struct {
-	Host map[string]HostConfig `toml:"host"`
-}
-
-type HostConfig struct {
-	Mirrors []MirrorConfig `toml:"mirrors"`
-}
-
-type MirrorConfig struct {
-	Host     string `toml:"host"`
-	Insecure bool   `toml:"insecure"`
-}

cmd/containerd-stargz-grpc/cri.go

@@ -1,90 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes/docker"
-	"github.com/containerd/stargz-snapshotter/fs/source"
-	digest "github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-)
-
-const (
-	// targetRefLabel is a label which contains image reference passed from CRI plugin.
-	targetRefLabel = "containerd.io/snapshot/cri.image-ref"
-
-	// targetDigestLabel is a label which contains layer digest passed from CRI plugin.
-	targetDigestLabel = "containerd.io/snapshot/cri.layer-digest"
-
-	// targetImageLayersLabel is a label which contains layer digests contained in
-	// the target image and is passed from CRI plugin.
-	targetImageLayersLabel = "containerd.io/snapshot/cri.image-layers"
-)
-
-func sourceFromCRILabels(hosts docker.RegistryHosts) source.GetSources {
-	return func(labels map[string]string) ([]source.Source, error) {
-		refStr, ok := labels[targetRefLabel]
-		if !ok {
-			return nil, fmt.Errorf("reference hasn't been passed")
-		}
-		refspec, err := reference.Parse(refStr)
-		if err != nil {
-			return nil, err
-		}
-
-		digestStr, ok := labels[targetDigestLabel]
-		if !ok {
-			return nil, fmt.Errorf("digest hasn't been passed")
-		}
-		target, err := digest.Parse(digestStr)
-		if err != nil {
-			return nil, err
-		}
-
-		var layersDgst []digest.Digest
-		if l, ok := labels[targetImageLayersLabel]; ok {
-			layersStr := strings.Split(l, ",")
-			for _, l := range layersStr {
-				d, err := digest.Parse(l)
-				if err != nil {
-					return nil, err
-				}
-				if d.String() != target.String() {
-					layersDgst = append(layersDgst, d)
-				}
-			}
-		}
-
-		var layers []ocispec.Descriptor
-		for _, dgst := range append([]digest.Digest{target}, layersDgst...) {
-			layers = append(layers, ocispec.Descriptor{Digest: dgst})
-		}
-
-		return []source.Source{
-			{
-				Hosts:    hosts,
-				Name:     refspec,
-				Target:   ocispec.Descriptor{Digest: target},
-				Manifest: ocispec.Manifest{Layers: layers},
-			},
-		}, nil
-	}
-}

cmd/containerd-stargz-grpc/db/db.go

@@ -0,0 +1,553 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package db
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/containerd/stargz-snapshotter/metadata"
+	bolt "go.etcd.io/bbolt"
+)
+
+// Metadata package stores filesystem metadata in the following schema.
+//
+// - filesystems
+//   - *filesystem id*                    : bucket for each filesystem keyed by a unique string.
+//     - nodes
+//       - *node id*                      : bucket for each node keyed by a uniqe uint64.
+//         - size : <varint>              : size of the regular node.
+//         - modtime : <varint>           : modification time of the node.
+//         - linkName : <string>          : link target of symlink
+//         - mode : <uvarint>             : permission and mode bits (os.FileMode).
+//         - uid : <varint>               : uid of the owner.
+//         - gid : <varint>               : gid of the owner.
+//         - devMajor : <varint>          : the major device number for device
+//         - devMinor : <varint>          : the minor device number for device
+//         - xattrKey : <string>          : key of the first extended attribute.
+//         - xattrValue : <string>        : value of the first extended attribute
+//         - xattrsExtra                  : 2nd and the following extended attribute.
+//           - *key* : <string>           : map of key to value string
+//         - numLink : <varint>           : the number of links pointing to this node.
+//     - metadata
+//       - *node id*                      : bucket for each node keyed by a uniqe uint64.
+//         - childName : <string>         : base name of the first child
+//         - childID   : <node id>        : id of the first child
+//         - childrenExtra                : 2nd and following child nodes of directory.
+//           - *basename* : <node id>     : map of basename string to the child node id
+//         - chunk : <encoded>            : information of the first chunkn
+//         - chunksExtra                  : 2nd and following chunks (this is rarely used so we can avoid the cost of creating the bucket)
+//           - *chunk offset* : <encoded> : keyed by chunk offset (varint) in the estargz file to the chunk.
+//         - nextOffset : <varint>        : the offset of the next node with a non-zero offset.
+//     - stream
+//       - *offset*                       : bucket for each chunk stream that have multiple inner chunks.
+//         - *innerOffset* : node id      : node id that has the contents at the keyed innerOffset.
+
+var (
+	bucketKeyFilesystems = []byte("filesystems")
+
+	bucketKeyNodes       = []byte("nodes")
+	bucketKeySize        = []byte("size")
+	bucketKeyModTime     = []byte("modtime")
+	bucketKeyLinkName    = []byte("linkName")
+	bucketKeyMode        = []byte("mode")
+	bucketKeyUID         = []byte("uid")
+	bucketKeyGID         = []byte("gid")
+	bucketKeyDevMajor    = []byte("devMajor")
+	bucketKeyDevMinor    = []byte("devMinor")
+	bucketKeyXattrKey    = []byte("xattrKey")
+	bucketKeyXattrValue  = []byte("xattrValue")
+	bucketKeyXattrsExtra = []byte("xattrsExtra")
+	bucketKeyNumLink     = []byte("numLink")
+
+	bucketKeyMetadata      = []byte("metadata")
+	bucketKeyChildName     = []byte("childName")
+	bucketKeyChildID       = []byte("childID")
+	bucketKeyChildrenExtra = []byte("childrenExtra")
+	bucketKeyChunk         = []byte("chunk")
+	bucketKeyChunksExtra   = []byte("chunksExtra")
+	bucketKeyNextOffset    = []byte("nextOffset")
+
+	bucketKeyStream = []byte("stream")
+)
+
+type childEntry struct {
+	base string
+	id   uint32
+}
+
+type chunkEntry struct {
+	offset      int64
+	chunkOffset int64
+	chunkSize   int64
+	chunkDigest string
+	innerOffset int64 // -1 indicates that no following chunks in the stream.
+}
+
+type metadataEntry struct {
+	children   map[string]childEntry
+	chunks     []chunkEntry
+	nextOffset int64
+}
+
+func getNodes(tx *bolt.Tx, fsID string) (*bolt.Bucket, error) {
+	filesystems := tx.Bucket(bucketKeyFilesystems)
+	if filesystems == nil {
+		return nil, fmt.Errorf("fs %q not found: no fs is registered", fsID)
+	}
+	lbkt := filesystems.Bucket([]byte(fsID))
+	if lbkt == nil {
+		return nil, fmt.Errorf("fs bucket for %q not found", fsID)
+	}
+	nodes := lbkt.Bucket(bucketKeyNodes)
+	if nodes == nil {
+		return nil, fmt.Errorf("nodes bucket for %q not found", fsID)
+	}
+	return nodes, nil
+}
+
+func getMetadata(tx *bolt.Tx, fsID string) (*bolt.Bucket, error) {
+	filesystems := tx.Bucket(bucketKeyFilesystems)
+	if filesystems == nil {
+		return nil, fmt.Errorf("fs %q not found: no fs is registered", fsID)
+	}
+	lbkt := filesystems.Bucket([]byte(fsID))
+	if lbkt == nil {
+		return nil, fmt.Errorf("fs bucket for %q not found", fsID)
+	}
+	md := lbkt.Bucket(bucketKeyMetadata)
+	if md == nil {
+		return nil, fmt.Errorf("metadata bucket for fs %q not found", fsID)
+	}
+	return md, nil
+}
+
+func getStream(tx *bolt.Tx, fsID string) (*bolt.Bucket, error) {
+	filesystems := tx.Bucket(bucketKeyFilesystems)
+	if filesystems == nil {
+		return nil, fmt.Errorf("fs %q not found: no fs is registered", fsID)
+	}
+	lbkt := filesystems.Bucket([]byte(fsID))
+	if lbkt == nil {
+		return nil, fmt.Errorf("fs bucket for %q not found", fsID)
+	}
+	st := lbkt.Bucket(bucketKeyStream)
+	if st == nil {
+		return nil, fmt.Errorf("stream bucket for fs %q not found", fsID)
+	}
+	return st, nil
+}
+
+func getNodeBucketByID(nodes *bolt.Bucket, id uint32) (*bolt.Bucket, error) {
+	b := nodes.Bucket(encodeID(id))
+	if b == nil {
+		return nil, fmt.Errorf("node bucket for %d not found", id)
+	}
+	return b, nil
+}
+
+func getMetadataBucketByID(md *bolt.Bucket, id uint32) (*bolt.Bucket, error) {
+	b := md.Bucket(encodeID(id))
+	if b == nil {
+		return nil, fmt.Errorf("metadata bucket for %d not found", id)
+	}
+	return b, nil
+}
+
+func writeAttr(b *bolt.Bucket, attr *metadata.Attr) error {
+	for _, v := range []struct {
+		key []byte
+		val int64
+	}{
+		{bucketKeySize, attr.Size},
+		{bucketKeyUID, int64(attr.UID)},
+		{bucketKeyGID, int64(attr.GID)},
+		{bucketKeyDevMajor, int64(attr.DevMajor)},
+		{bucketKeyDevMinor, int64(attr.DevMinor)},
+		{bucketKeyNumLink, int64(attr.NumLink - 1)}, // numLink = 0 means num link = 1 in DB
+	} {
+		if v.val != 0 {
+			val, err := encodeInt(v.val)
+			if err != nil {
+				return err
+			}
+			if err := b.Put(v.key, val); err != nil {
+				return err
+			}
+		}
+	}
+	if !attr.ModTime.IsZero() {
+		te, err := attr.ModTime.GobEncode()
+		if err != nil {
+			return err
+		}
+		if err := b.Put(bucketKeyModTime, te); err != nil {
+			return err
+		}
+	}
+	if len(attr.LinkName) > 0 {
+		if err := b.Put(bucketKeyLinkName, []byte(attr.LinkName)); err != nil {
+			return err
+		}
+	}
+	if attr.Mode != 0 {
+		val, err := encodeUint(uint64(attr.Mode))
+		if err != nil {
+			return err
+		}
+		if err := b.Put(bucketKeyMode, val); err != nil {
+			return err
+		}
+	}
+	if len(attr.Xattrs) > 0 {
+		var firstK string
+		var firstV []byte
+		for k, v := range attr.Xattrs {
+			firstK, firstV = k, v
+			break
+		}
+		if err := b.Put(bucketKeyXattrKey, []byte(firstK)); err != nil {
+			return err
+		}
+		if err := b.Put(bucketKeyXattrValue, firstV); err != nil {
+			return err
+		}
+		var xbkt *bolt.Bucket
+		for k, v := range attr.Xattrs {
+			if k == firstK || len(v) == 0 {
+				continue
+			}
+			if xbkt == nil {
+				if xbkt := b.Bucket(bucketKeyXattrsExtra); xbkt != nil {
+					// Reset
+					if err := b.DeleteBucket(bucketKeyXattrsExtra); err != nil {
+						return err
+					}
+				}
+				var err error
+				xbkt, err = b.CreateBucket(bucketKeyXattrsExtra)
+				if err != nil {
+					return err
+				}
+			}
+			if err := xbkt.Put([]byte(k), v); err != nil {
+				return fmt.Errorf("failed to set xattr %q=%q: %w", k, string(v), err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func readAttr(b *bolt.Bucket, attr *metadata.Attr) error {
+	return b.ForEach(func(k, v []byte) error {
+		switch string(k) {
+		case string(bucketKeySize):
+			attr.Size, _ = binary.Varint(v)
+		case string(bucketKeyModTime):
+			if err := (&attr.ModTime).GobDecode(v); err != nil {
+				return err
+			}
+		case string(bucketKeyLinkName):
+			attr.LinkName = string(v)
+		case string(bucketKeyMode):
+			mode, _ := binary.Uvarint(v)
+			attr.Mode = os.FileMode(uint32(mode))
+		case string(bucketKeyUID):
+			i, _ := binary.Varint(v)
+			attr.UID = int(i)
+		case string(bucketKeyGID):
+			i, _ := binary.Varint(v)
+			attr.GID = int(i)
+		case string(bucketKeyDevMajor):
+			i, _ := binary.Varint(v)
+			attr.DevMajor = int(i)
+		case string(bucketKeyDevMinor):
+			i, _ := binary.Varint(v)
+			attr.DevMinor = int(i)
+		case string(bucketKeyNumLink):
+			i, _ := binary.Varint(v)
+			attr.NumLink = int(i) + 1 // numLink = 0 means num link = 1 in DB
+		case string(bucketKeyXattrKey):
+			if attr.Xattrs == nil {
+				attr.Xattrs = make(map[string][]byte)
+			}
+			attr.Xattrs[string(v)] = b.Get(bucketKeyXattrValue)
+		case string(bucketKeyXattrsExtra):
+			if err := b.Bucket(k).ForEach(func(k, v []byte) error {
+				if attr.Xattrs == nil {
+					attr.Xattrs = make(map[string][]byte)
+				}
+				attr.Xattrs[string(k)] = v
+				return nil
+			}); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func readNumLink(b *bolt.Bucket) int {
+	// numLink = 0 means num link = 1 in BD
+	numLink, _ := binary.Varint(b.Get(bucketKeyNumLink))
+	return int(numLink) + 1
+}
+
+func readChunks(b *bolt.Bucket, size int64) (chunks []chunkEntry, err error) {
+	if chunk := b.Get(bucketKeyChunk); len(chunk) > 0 {
+		e, err := decodeChunkEntry(chunk)
+		if err != nil {
+			return nil, err
+		}
+		chunks = append(chunks, e)
+	}
+	if chbkt := b.Bucket(bucketKeyChunksExtra); chbkt != nil {
+		if err := chbkt.ForEach(func(_, v []byte) error {
+			e, err := decodeChunkEntry(v)
+			if err != nil {
+				return err
+			}
+			chunks = append(chunks, e)
+			return nil
+		}); err != nil {
+			return nil, err
+		}
+		sort.Slice(chunks, func(i, j int) bool {
+			return chunks[i].chunkOffset < chunks[j].chunkOffset
+		})
+	}
+	nextOffset := size
+	for i := len(chunks) - 1; i >= 0; i-- {
+		chunks[i].chunkSize = nextOffset - chunks[i].chunkOffset
+		nextOffset = chunks[i].chunkOffset
+	}
+	return
+}
+
+type chunkEntryWithID struct {
+	chunkEntry
+	id uint32
+}
+
+func readInnerChunks(tx *bolt.Tx, fsID string, off int64) (chunks []chunkEntryWithID, err error) {
+	sb, err := getStream(tx, fsID)
+	if err != nil {
+		return nil, err
+	}
+	offEncoded, err := encodeInt(off)
+	if err != nil {
+		return nil, err
+	}
+	ob := sb.Bucket(offEncoded)
+	if ob == nil {
+		return nil, fmt.Errorf("inner chunk bucket for %d not found", off)
+	}
+	nodes, err := getNodes(tx, fsID)
+	if err != nil {
+		return nil, fmt.Errorf("nodes bucket of %q not found: %w", fsID, err)
+	}
+	metadataEntries, err := getMetadata(tx, fsID)
+	if err != nil {
+		return nil, fmt.Errorf("metadata bucket of %q not found: %w", fsID, err)
+	}
+	if err := ob.ForEach(func(_, v []byte) error {
+		nodeid := decodeID(v)
+		b, err := getNodeBucketByID(nodes, nodeid)
+		if err != nil {
+			return fmt.Errorf("failed to get file bucket %d: %w", nodeid, err)
+		}
+		size, _ := binary.Varint(b.Get(bucketKeySize))
+		if md, err := getMetadataBucketByID(metadataEntries, nodeid); err == nil {
+			nodeChunks, err := readChunks(md, size)
+			if err != nil {
+				return fmt.Errorf("failed to get chunks: %w", err)
+			}
+			for _, e := range nodeChunks {
+				if e.offset == off {
+					chunks = append(chunks, chunkEntryWithID{e, nodeid})
+				}
+			}
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	sort.Slice(chunks, func(i, j int) bool {
+		return chunks[i].innerOffset < chunks[j].innerOffset
+	})
+	return chunks, nil
+}
+
+func readChild(md *bolt.Bucket, base string) (uint32, error) {
+	if base == string(md.Get(bucketKeyChildName)) {
+		return decodeID(md.Get(bucketKeyChildID)), nil
+	}
+	cbkt := md.Bucket(bucketKeyChildrenExtra)
+	if cbkt == nil {
+		return 0, fmt.Errorf("extra children not found")
+	}
+	eid := cbkt.Get([]byte(base))
+	if len(eid) == 0 {
+		return 0, fmt.Errorf("children %q not found", base)
+	}
+	return decodeID(eid), nil
+}
+
+func writeMetadataEntry(md *bolt.Bucket, m *metadataEntry) error {
+	if len(m.children) > 0 {
+		var firstChildName string
+		var firstChild childEntry
+		for name, child := range m.children {
+			firstChildName, firstChild = name, child
+			break
+		}
+		if err := md.Put(bucketKeyChildID, encodeID(firstChild.id)); err != nil {
+			return fmt.Errorf("failed to put id of first child %q: %w", firstChildName, err)
+		}
+		if err := md.Put(bucketKeyChildName, []byte(firstChildName)); err != nil {
+			return fmt.Errorf("failed to put name first child %q: %w", firstChildName, err)
+		}
+		if len(m.children) > 1 {
+			var cbkt *bolt.Bucket
+			for k, c := range m.children {
+				if k == firstChildName {
+					continue
+				}
+				if cbkt == nil {
+					if cbkt := md.Bucket(bucketKeyChildrenExtra); cbkt != nil {
+						// Reset
+						if err := md.DeleteBucket(bucketKeyChildrenExtra); err != nil {
+							return err
+						}
+					}
+					var err error
+					cbkt, err = md.CreateBucket(bucketKeyChildrenExtra)
+					if err != nil {
+						return err
+					}
+				}
+				if err := cbkt.Put([]byte(c.base), encodeID(c.id)); err != nil {
+					return fmt.Errorf("failed to add child ID %q: %w", c.id, err)
+				}
+			}
+		}
+	}
+	if len(m.chunks) > 0 {
+		first := m.chunks[0]
+		if err := md.Put(bucketKeyChunk, encodeChunkEntry(first)); err != nil {
+			return fmt.Errorf("failed to set chunk %q: %w", first.offset, err)
+		}
+		var cbkt *bolt.Bucket
+		for _, e := range m.chunks[1:] {
+			if cbkt == nil {
+				if cbkt := md.Bucket(bucketKeyChunksExtra); cbkt != nil {
+					// Reset
+					if err := md.DeleteBucket(bucketKeyChunksExtra); err != nil {
+						return err
+					}
+				}
+				var err error
+				cbkt, err = md.CreateBucket(bucketKeyChunksExtra)
+				if err != nil {
+					return err
+				}
+			}
+			ecoff, err := encodeInt(e.chunkOffset)
+			if err != nil {
+				return err
+			}
+			if err := cbkt.Put(ecoff, encodeChunkEntry(e)); err != nil {
+				return err
+			}
+		}
+	}
+	if m.nextOffset > 0 {
+		if err := putInt(md, bucketKeyNextOffset, m.nextOffset); err != nil {
+			return fmt.Errorf("failed to set next offset value %d: %w", m.nextOffset, err)
+		}
+	}
+	return nil
+}
+
+func encodeChunkEntry(e chunkEntry) []byte {
+	eb := make([]byte, 24+len([]byte(e.chunkDigest)))
+	binary.BigEndian.PutUint64(eb[0:8], uint64(e.chunkOffset))
+	binary.BigEndian.PutUint64(eb[8:16], uint64(e.offset))
+	binary.BigEndian.PutUint64(eb[16:24], uint64(e.innerOffset))
+	copy(eb[24:], []byte(e.chunkDigest))
+	return eb
+}
+
+func decodeChunkEntry(d []byte) (e chunkEntry, _ error) {
+	if len(d) < 24 {
+		return e, fmt.Errorf("mulformed chunk entry (len:%d)", len(d))
+	}
+	e.chunkOffset = int64(binary.BigEndian.Uint64(d[0:8]))
+	e.offset = int64(binary.BigEndian.Uint64(d[8:16]))
+	e.innerOffset = int64(binary.BigEndian.Uint64(d[16:24]))
+	if len(d) > 24 {
+		e.chunkDigest = string(d[24:])
+	}
+	return e, nil
+}
+
+func putInt(b *bolt.Bucket, k []byte, v int64) error {
+	i, err := encodeInt(v)
+	if err != nil {
+		return err
+	}
+	return b.Put(k, i)
+}
+
+func encodeID(id uint32) []byte {
+	b := [4]byte{}
+	binary.BigEndian.PutUint32(b[:], id)
+	return b[:]
+}
+
+func decodeID(b []byte) uint32 {
+	return binary.BigEndian.Uint32(b)
+}
+
+func encodeInt(i int64) ([]byte, error) {
+	var (
+		buf      [binary.MaxVarintLen64]byte
+		iEncoded = buf[:]
+	)
+	iEncoded = iEncoded[:binary.PutVarint(iEncoded, i)]
+
+	if len(iEncoded) == 0 {
+		return nil, fmt.Errorf("failed encoding integer = %v", i)
+	}
+	return iEncoded, nil
+}
+
+func encodeUint(i uint64) ([]byte, error) {
+	var (
+		buf      [binary.MaxVarintLen64]byte
+		iEncoded = buf[:]
+	)
+	iEncoded = iEncoded[:binary.PutUvarint(iEncoded, i)]
+
+	if len(iEncoded) == 0 {
+		return nil, fmt.Errorf("failed encoding integer = %v", i)
+	}
+	return iEncoded, nil
+}

cmd/containerd-stargz-grpc/db/reader_test.go

@@ -0,0 +1,153 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package db
+
+import (
+	"io"
+	"os"
+	"testing"
+
+	"github.com/containerd/stargz-snapshotter/fs/layer"
+	fsreader "github.com/containerd/stargz-snapshotter/fs/reader"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	"github.com/containerd/stargz-snapshotter/metadata/testutil"
+	bolt "go.etcd.io/bbolt"
+)
+
+func TestReader(t *testing.T) {
+	testRunner := &testutil.TestRunner{
+		TestingT: t,
+		Runner: func(testingT testutil.TestingT, name string, run func(t testutil.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	testutil.TestReader(testRunner, newTestableReader)
+}
+
+func TestFSReader(t *testing.T) {
+	testRunner := &fsreader.TestRunner{
+		TestingT: t,
+		Runner: func(testingT fsreader.TestingT, name string, run func(t fsreader.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	fsreader.TestSuiteReader(testRunner, newStore)
+}
+
+func TestFSLayer(t *testing.T) {
+	testRunner := &layer.TestRunner{
+		TestingT: t,
+		Runner: func(testingT layer.TestingT, name string, run func(t layer.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	layer.TestSuiteLayer(testRunner, newStore)
+}
+
+func newTestableReader(sr *io.SectionReader, opts ...metadata.Option) (testutil.TestableReader, error) {
+	f, err := os.CreateTemp("", "readertestdb")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+	db, err := bolt.Open(f.Name(), 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+	r, err := NewReader(db, sr, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return &testableReadCloser{
+		TestableReader: r.(*reader),
+		closeFn: func() error {
+			db.Close()
+			return os.Remove(f.Name())
+		},
+	}, nil
+}
+
+func newStore(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error) {
+	f, err := os.CreateTemp("", "readertestdb")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	db, err := bolt.Open(f.Name(), 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+	r, err := NewReader(db, sr, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return &readCloser{
+		Reader: r,
+		closeFn: func() error {
+			db.Close()
+			return os.Remove(f.Name())
+		},
+	}, nil
+}
+
+type readCloser struct {
+	metadata.Reader
+	closeFn func() error
+}
+
+func (r *readCloser) Close() error {
+	r.closeFn()
+	return r.Reader.Close()
+}
+
+type testableReadCloser struct {
+	testutil.TestableReader
+	closeFn func() error
+}
+
+func (r *testableReadCloser) Close() error {
+	r.closeFn()
+	return r.TestableReader.Close()
+}

cmd/containerd-stargz-grpc/fsopts/fsopts.go

@@ -0,0 +1,80 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fsopts
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path/filepath"
+
+	"github.com/containerd/log"
+	dbmetadata "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/db"
+	ipfs "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/ipfs"
+	"github.com/containerd/stargz-snapshotter/fs"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
+	bolt "go.etcd.io/bbolt"
+)
+
+type Config struct {
+	EnableIpfs    bool
+	MetadataStore string
+	OpenBoltDB    func(string) (*bolt.DB, error)
+}
+
+const (
+	memoryMetadataType = "memory"
+	dbMetadataType     = "db"
+)
+
+func ConfigFsOpts(ctx context.Context, rootDir string, config *Config) ([]fs.Option, error) {
+	fsOpts := []fs.Option{fs.WithMetricsLogLevel(log.InfoLevel)}
+
+	if config.EnableIpfs {
+		fsOpts = append(fsOpts, fs.WithResolveHandler("ipfs", new(ipfs.ResolveHandler)))
+	}
+
+	mt, err := getMetadataStore(rootDir, config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure metadata store: %w", err)
+	}
+	fsOpts = append(fsOpts, fs.WithMetadataStore(mt))
+
+	return fsOpts, nil
+}
+
+func getMetadataStore(rootDir string, config *Config) (metadata.Store, error) {
+	switch config.MetadataStore {
+	case "", memoryMetadataType:
+		return memorymetadata.NewReader, nil
+	case dbMetadataType:
+		if config.OpenBoltDB == nil {
+			return nil, fmt.Errorf("bolt DB is not configured")
+		}
+		db, err := config.OpenBoltDB(filepath.Join(rootDir, "metadata.db"))
+		if err != nil {
+			return nil, err
+		}
+		return func(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error) {
+			return dbmetadata.NewReader(db, sr, opts...)
+		}, nil
+	default:
+		return nil, fmt.Errorf("unknown metadata store type: %v; must be %v or %v",
+			config.MetadataStore, memoryMetadataType, dbMetadataType)
+	}
+}

cmd/containerd-stargz-grpc/ipfs/resolvehandler.go

@@ -0,0 +1,79 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ipfs
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/containerd/stargz-snapshotter/fs/remote"
+	"github.com/containerd/stargz-snapshotter/ipfs"
+	ipfsclient "github.com/containerd/stargz-snapshotter/ipfs/client"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+type ResolveHandler struct{}
+
+func (r *ResolveHandler) Handle(ctx context.Context, desc ocispec.Descriptor) (remote.Fetcher, int64, error) {
+	cid, err := ipfs.GetCID(desc)
+	if err != nil {
+		return nil, 0, err
+	}
+	var ipath string
+	if idir := os.Getenv("IPFS_PATH"); idir != "" {
+		ipath = idir
+	}
+	// HTTP is only supported as of now. We can add https support here if needed (e.g. for connecting to it via proxy, etc)
+	iurl, err := ipfsclient.GetIPFSAPIAddress(ipath, "http")
+	if err != nil {
+		return nil, 0, err
+	}
+	client := ipfsclient.New(iurl)
+	info, err := client.StatCID(cid)
+	if err != nil {
+		return nil, 0, err
+	}
+	return &fetcher{cid: cid, size: int64(info.Size), client: client}, int64(info.Size), nil
+}
+
+type fetcher struct {
+	cid  string
+	size int64
+
+	client *ipfsclient.Client
+}
+
+func (f *fetcher) Fetch(ctx context.Context, off int64, size int64) (io.ReadCloser, error) {
+	if off > f.size {
+		return nil, fmt.Errorf("offset is larger than the size of the blob %d(offset) > %d(blob size)", off, f.size)
+	}
+	o, s := int(off), int(size)
+	return f.client.Get("/ipfs/"+f.cid, &o, &s)
+}
+
+func (f *fetcher) Check() error {
+	_, err := f.client.StatCID(f.cid)
+	return err
+}
+
+func (f *fetcher) GenID(off int64, size int64) string {
+	sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", f.cid, off, size)))
+	return fmt.Sprintf("%x", sum)
+}

cmd/containerd-stargz-grpc/main.go

@@ -19,180 +19,350 @@ package main
 import (
 	"context"
 	"flag"
+	"fmt"
+	golog "log"
+	"math/rand"
 	"net"
 	"net/http"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"time"
 
-	"github.com/BurntSushi/toml"
 	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
-	"github.com/containerd/containerd/contrib/snapshotservice"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/remotes/docker"
-	"github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/keychain"
-	stargzfs "github.com/containerd/stargz-snapshotter/fs"
-	"github.com/containerd/stargz-snapshotter/fs/source"
+	"github.com/containerd/containerd/v2/contrib/snapshotservice"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/containerd/v2/pkg/sys"
+	"github.com/containerd/log"
+	"github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/fsopts"
+	"github.com/containerd/stargz-snapshotter/fusemanager"
+	"github.com/containerd/stargz-snapshotter/service"
+	"github.com/containerd/stargz-snapshotter/service/keychain/keychainconfig"
 	snbase "github.com/containerd/stargz-snapshotter/snapshot"
-	"github.com/hashicorp/go-multierror"
-	"github.com/sirupsen/logrus"
+	"github.com/containerd/stargz-snapshotter/version"
+	sddaemon "github.com/coreos/go-systemd/v22/daemon"
+	metrics "github.com/docker/go-metrics"
+	"github.com/pelletier/go-toml"
+	bolt "go.etcd.io/bbolt"
+	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
 )
 
 const (
-	defaultAddress    = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
-	defaultConfigPath = "/etc/containerd-stargz-grpc/config.toml"
-	defaultLogLevel   = logrus.InfoLevel
-	defaultRootDir    = "/var/lib/containerd-stargz-grpc"
+	defaultAddress             = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+	defaultConfigPath          = "/etc/containerd-stargz-grpc/config.toml"
+	defaultLogLevel            = log.InfoLevel
+	defaultRootDir             = "/var/lib/containerd-stargz-grpc"
+	defaultImageServiceAddress = "/run/containerd/containerd.sock"
+	defaultFuseManagerAddress  = "/run/containerd-stargz-grpc/fuse-manager.sock"
+	fuseManagerBin             = "stargz-fuse-manager"
 )
 
 var (
-	address    = flag.String("address", defaultAddress, "address for the snapshotter's GRPC server")
-	configPath = flag.String("config", defaultConfigPath, "path to the configuration file")
-	logLevel   = flag.String("log-level", defaultLogLevel.String(), "set the logging level [trace, debug, info, warn, error, fatal, panic]")
-	rootDir    = flag.String("root", defaultRootDir, "path to the root directory for this snapshotter")
+	address      = flag.String("address", defaultAddress, "address for the snapshotter's GRPC server")
+	configPath   = flag.String("config", defaultConfigPath, "path to the configuration file")
+	logLevel     = flag.String("log-level", defaultLogLevel.String(), "set the logging level [trace, debug, info, warn, error, fatal, panic]")
+	rootDir      = flag.String("root", defaultRootDir, "path to the root directory for this snapshotter")
+	printVersion = flag.Bool("version", false, "print the version")
 )
 
+type snapshotterConfig struct {
+	service.Config
+
+	// MetricsAddress is address for the metrics API
+	MetricsAddress string `toml:"metrics_address" json:"metrics_address"`
+
+	// NoPrometheus is a flag to disable the emission of the metrics
+	NoPrometheus bool `toml:"no_prometheus" json:"no_prometheus"`
+
+	// DebugAddress is a Unix domain socket address where the snapshotter exposes /debug/ endpoints.
+	DebugAddress string `toml:"debug_address" json:"debug_address"`
+
+	// IPFS is a flag to enbale lazy pulling from IPFS.
+	IPFS bool `toml:"ipfs" json:"ipfs"`
+
+	// MetadataStore is the type of the metadata store to use.
+	MetadataStore string `toml:"metadata_store" default:"memory" json:"metadata_store"`
+
+	// FuseManagerConfig is configuration for fusemanager
+	FuseManagerConfig `toml:"fuse_manager" json:"fuse_manager"`
+}
+
+type FuseManagerConfig struct {
+	// Enable is whether detach fusemanager or not
+	Enable bool `toml:"enable" default:"false" json:"enable"`
+
+	// Address is address for the fusemanager's GRPC server (default: "/run/containerd-stargz-grpc/fuse-manager.sock")
+	Address string `toml:"address" json:"address"`
+
+	// Path is path to the fusemanager's executable (default: looking for a binary "stargz-fuse-manager")
+	Path string `toml:"path" json:"path"`
+}
+
 func main() {
+	rand.Seed(time.Now().UnixNano()) //nolint:staticcheck // Global math/rand seed is deprecated, but still used by external dependencies
 	flag.Parse()
-	lvl, err := logrus.ParseLevel(*logLevel)
+	log.SetFormat(log.JSONFormat)
+	err := log.SetLevel(*logLevel)
 	if err != nil {
 		log.L.WithError(err).Fatal("failed to prepare logger")
 	}
-	logrus.SetLevel(lvl)
-	logrus.SetFormatter(&logrus.JSONFormatter{
-		TimestampFormat: log.RFC3339NanoFixed,
-	})
+	if *printVersion {
+		fmt.Println("containerd-stargz-grpc", version.Version, version.Revision)
+		return
+	}
 
 	var (
 		ctx    = log.WithLogger(context.Background(), log.L)
-		config Config
+		config snapshotterConfig
 	)
+	// Streams log of standard lib (go-fuse uses this) into debug log
+	// Snapshotter should use "github.com/containerd/log" otherwize
+	// logs are always printed as "debug" mode.
+	golog.SetOutput(log.G(ctx).WriterLevel(log.DebugLevel))
 
 	// Get configuration from specified file
-	if _, err := toml.DecodeFile(*configPath, &config); err != nil && !(os.IsNotExist(err) && *configPath == defaultConfigPath) {
+	tree, err := toml.LoadFile(*configPath)
+	if err != nil && (!os.IsNotExist(err) || *configPath != defaultConfigPath) {
 		log.G(ctx).WithError(err).Fatalf("failed to load config file %q", *configPath)
 	}
-
-	// Prepare kubeconfig-based keychain if required
-	credsFuncs := []func(string) (string, string, error){keychain.NewDockerconfigKeychain(ctx)}
-	if config.KubeconfigKeychainConfig.EnableKeychain {
-		var opts []keychain.KubeconfigOption
-		if kcp := config.KubeconfigKeychainConfig.KubeconfigPath; kcp != "" {
-			opts = append(opts, keychain.WithKubeconfigPath(kcp))
-		}
-		credsFuncs = append(credsFuncs, keychain.NewKubeconfigKeychain(ctx, opts...))
+	if err := tree.Unmarshal(&config); err != nil {
+		log.G(ctx).WithError(err).Fatalf("failed to unmarshal config file %q", *configPath)
 	}
 
-	// Use RegistryHosts based on ResolverConfig and keychain
-	hosts := hostsFromConfig(config.ResolverConfig, credsFuncs...)
-
-	// Configure filesystem and snapshotter
-	fs, err := stargzfs.NewFilesystem(filepath.Join(*rootDir, "stargz"),
-		config.Config,
-		stargzfs.WithGetSources(sources(
-			sourceFromCRILabels(hosts),      // provides source info based on CRI labels
-			source.FromDefaultLabels(hosts), // provides source info based on default labels
-		)),
-	)
-	if err != nil {
-		log.G(ctx).WithError(err).Fatalf("failed to configure filesystem")
+	if err := service.Supported(*rootDir); err != nil {
+		log.G(ctx).WithError(err).Fatalf("snapshotter is not supported")
 	}
-	rs, err := snbase.NewSnapshotter(ctx, filepath.Join(*rootDir, "snapshotter"), fs, snbase.AsynchronousRemove)
-	if err != nil {
-		log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
-	}
-	defer func() {
-		log.G(ctx).Debug("Closing the snapshotter")
-		rs.Close()
-		log.G(ctx).Info("Exiting")
-	}()
 
 	// Create a gRPC server
 	rpc := grpc.NewServer()
 
+	// Configure FUSE passthrough
+	// Always set Direct to true to ensure that
+	// *directoryCache.Get always return *os.File instead of buffer
+	if config.PassThrough {
+		config.Direct = true
+	}
+
+	// Configure keychain
+	keyChainConfig := keychainconfig.Config{
+		EnableKubeKeychain:         config.KubeconfigKeychainConfig.EnableKeychain,
+		EnableCRIKeychain:          config.CRIKeychainConfig.EnableKeychain,
+		KubeconfigPath:             config.KubeconfigPath,
+		DefaultImageServiceAddress: defaultImageServiceAddress,
+		ImageServicePath:           config.ImageServicePath,
+	}
+
+	var rs snapshots.Snapshotter
+	fuseManagerConfig := config.FuseManagerConfig
+	if fuseManagerConfig.Enable {
+		fmPath := fuseManagerConfig.Path
+		if fmPath == "" {
+			var err error
+			fmPath, err = exec.LookPath(fuseManagerBin)
+			if err != nil {
+				log.G(ctx).WithError(err).Fatalf("failed to find fusemanager bin")
+			}
+		}
+		fmAddr := fuseManagerConfig.Address
+		if fmAddr == "" {
+			fmAddr = defaultFuseManagerAddress
+		}
+
+		if !filepath.IsAbs(fmAddr) {
+			log.G(ctx).WithError(err).Fatalf("fuse manager address must be an absolute path: %s", fmAddr)
+		}
+		managerNewlyStarted, err := fusemanager.StartFuseManager(ctx, fmPath, fmAddr, filepath.Join(*rootDir, "fusestore.db"), *logLevel, filepath.Join(*rootDir, "stargz-fuse-manager.log"))
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to start fusemanager")
+		}
+
+		fuseManagerConfig := fusemanager.Config{
+			Config:                     config.Config,
+			IPFS:                       config.IPFS,
+			MetadataStore:              config.MetadataStore,
+			DefaultImageServiceAddress: defaultImageServiceAddress,
+		}
+
+		fs, err := fusemanager.NewManagerClient(ctx, *rootDir, fmAddr, &fuseManagerConfig)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure fusemanager")
+		}
+		flags := []snbase.Opt{snbase.AsynchronousRemove}
+		// "managerNewlyStarted" being true indicates that the FUSE manager is newly started. To
+		// fully recover the snapshotter and the FUSE manager's state, we need to restore
+		// all snapshot mounts. If managerNewlyStarted is false, the existing FUSE manager maintains
+		// snapshot mounts so we don't need to restore them.
+		if !managerNewlyStarted {
+			flags = append(flags, snbase.NoRestore)
+		}
+		rs, err = snbase.NewSnapshotter(ctx, filepath.Join(*rootDir, "snapshotter"), fs, flags...)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
+		}
+		log.G(ctx).Infof("Start snapshotter with fusemanager mode")
+	} else {
+		crirpc := rpc
+		// For CRI keychain, if listening path is different from stargz-snapshotter's socket, prepare for the dedicated grpc server and the socket.
+		serveCRISocket := config.CRIKeychainConfig.EnableKeychain && config.ListenPath != "" && config.ListenPath != *address
+		if serveCRISocket {
+			crirpc = grpc.NewServer()
+		}
+		credsFuncs, err := keychainconfig.ConfigKeychain(ctx, crirpc, &keyChainConfig)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure keychain")
+		}
+		if serveCRISocket {
+			addr := config.ListenPath
+			// Prepare the directory for the socket
+			if err := os.MkdirAll(filepath.Dir(addr), 0700); err != nil {
+				log.G(ctx).WithError(err).Fatalf("failed to create directory %q", filepath.Dir(addr))
+			}
+
+			// Try to remove the socket file to avoid EADDRINUSE
+			if err := os.RemoveAll(addr); err != nil {
+				log.G(ctx).WithError(err).Fatalf("failed to remove %q", addr)
+			}
+
+			// Listen and serve
+			l, err := net.Listen("unix", addr)
+			if err != nil {
+				log.G(ctx).WithError(err).Fatalf("error on listen socket %q", addr)
+			}
+			go func() {
+				if err := crirpc.Serve(l); err != nil {
+					log.G(ctx).WithError(err).Errorf("error on serving CRI via socket %q", addr)
+				}
+			}()
+		}
+
+		fsConfig := fsopts.Config{
+			EnableIpfs:    config.IPFS,
+			MetadataStore: config.MetadataStore,
+			OpenBoltDB: func(p string) (*bolt.DB, error) {
+				return bolt.Open(p, 0600, &bolt.Options{
+					NoFreelistSync:  true,
+					InitialMmapSize: 64 * 1024 * 1024,
+					FreelistType:    bolt.FreelistMapType,
+				})
+			},
+		}
+		fsOpts, err := fsopts.ConfigFsOpts(ctx, *rootDir, &fsConfig)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure fs config")
+		}
+
+		rs, err = service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config,
+			service.WithCredsFuncs(credsFuncs...), service.WithFilesystemOptions(fsOpts...))
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
+		}
+	}
+
+	cleanup, err := serve(ctx, rpc, *address, rs, config)
+	if err != nil {
+		log.G(ctx).WithError(err).Fatalf("failed to serve snapshotter")
+	}
+
+	// When FUSE manager is disabled, FUSE servers are goroutines in the
+	// contaienrd-stargz-grpc process. So killing containerd-stargz-grpc will
+	// result in all FUSE mount becoming unavailable with leaving all resources
+	// (e.g. temporary cache) on the node. To ensure graceful shutdown, we
+	// should always cleanup mounts and associated resources here.
+	//
+	// When FUSE manager is enabled, those mounts are still under the control by
+	// the FUSE manager so we need to avoid cleaning them up unless explicitly
+	// commanded via SIGINT. The user can use SIGINT to gracefully killing the FUSE
+	// manager before rebooting the node for ensuring that the all snapshots are
+	// unmounted with cleaning up associated temporary resources.
+	if cleanup || !fuseManagerConfig.Enable {
+		log.G(ctx).Debug("Closing the snapshotter")
+		rs.Close()
+	}
+	log.G(ctx).Info("Exiting")
+}
+
+func serve(ctx context.Context, rpc *grpc.Server, addr string, rs snapshots.Snapshotter, config snapshotterConfig) (bool, error) {
 	// Convert the snapshotter to a gRPC service,
-	service := snapshotservice.FromSnapshotter(rs)
+	snsvc := snapshotservice.FromSnapshotter(rs)
 
 	// Register the service with the gRPC server
-	snapshotsapi.RegisterSnapshotsServer(rpc, service)
+	snapshotsapi.RegisterSnapshotsServer(rpc, snsvc)
 
 	// Prepare the directory for the socket
-	if err := os.MkdirAll(filepath.Dir(*address), 0700); err != nil {
-		log.G(ctx).WithError(err).Fatalf("failed to create directory %q", filepath.Dir(*address))
+	if err := os.MkdirAll(filepath.Dir(addr), 0700); err != nil {
+		return false, fmt.Errorf("failed to create directory %q: %w", filepath.Dir(addr), err)
 	}
 
 	// Try to remove the socket file to avoid EADDRINUSE
-	if err := os.RemoveAll(*address); err != nil {
-		log.G(ctx).WithError(err).Fatalf("failed to remove %q", *address)
+	if err := os.RemoveAll(addr); err != nil {
+		return false, fmt.Errorf("failed to remove %q: %w", addr, err)
+	}
+
+	errCh := make(chan error, 1)
+
+	// We need to consider both the existence of MetricsAddress as well as NoPrometheus flag not set
+	if config.MetricsAddress != "" && !config.NoPrometheus {
+		l, err := net.Listen("tcp", config.MetricsAddress)
+		if err != nil {
+			return false, fmt.Errorf("failed to get listener for metrics endpoint: %w", err)
+		}
+		m := http.NewServeMux()
+		m.Handle("/metrics", metrics.Handler())
+		go func() {
+			if err := http.Serve(l, m); err != nil {
+				errCh <- fmt.Errorf("error on serving metrics via socket %q: %w", addr, err)
+			}
+		}()
+	}
+
+	if config.DebugAddress != "" {
+		log.G(ctx).Infof("listen %q for debugging", config.DebugAddress)
+		l, err := sys.GetLocalListener(config.DebugAddress, 0, 0)
+		if err != nil {
+			return false, fmt.Errorf("failed to listen %q: %w", config.DebugAddress, err)
+		}
+		go func() {
+			if err := http.Serve(l, debugServerMux()); err != nil {
+				errCh <- fmt.Errorf("error on serving a debug endpoint via socket %q: %w", addr, err)
+			}
+		}()
 	}
 
 	// Listen and serve
-	l, err := net.Listen("unix", *address)
+	l, err := net.Listen("unix", addr)
 	if err != nil {
-		log.G(ctx).WithError(err).Fatalf("error on listen socket %q", *address)
+		return false, fmt.Errorf("error on listen socket %q: %w", addr, err)
 	}
 	go func() {
 		if err := rpc.Serve(l); err != nil {
-			log.G(ctx).WithError(err).Fatalf("error on serving via socket %q", *address)
+			errCh <- fmt.Errorf("error on serving via socket %q: %w", addr, err)
 		}
 	}()
-	waitForSIGINT()
-	log.G(ctx).Info("Got SIGINT")
-}
 
-func waitForSIGINT() {
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
-	<-c
-}
-
-func hostsFromConfig(cfg ResolverConfig, credsFuncs ...func(string) (string, string, error)) docker.RegistryHosts {
-	return func(host string) (hosts []docker.RegistryHost, _ error) {
-		for _, h := range append(cfg.Host[host].Mirrors, MirrorConfig{
-			Host: host,
-		}) {
-			tr := &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}
-			config := docker.RegistryHost{
-				Client:       tr,
-				Host:         h.Host,
-				Scheme:       "https",
-				Path:         "/v2",
-				Capabilities: docker.HostCapabilityPull,
-				Authorizer: docker.NewDockerAuthorizer(
-					docker.WithAuthClient(tr),
-					docker.WithAuthCreds(func(host string) (string, string, error) {
-						for _, f := range credsFuncs {
-							if username, secret, err := f(host); err != nil {
-								return "", "", err
-							} else if !(username == "" && secret == "") {
-								return username, secret, nil
-							}
-						}
-						return "", "", nil
-					})),
-			}
-			if localhost, _ := docker.MatchLocalhost(config.Host); localhost || h.Insecure {
-				config.Scheme = "http"
-			}
-			if config.Host == "docker.io" {
-				config.Host = "registry-1.docker.io"
-			}
-			hosts = append(hosts, config)
-		}
-		return
+	if os.Getenv("NOTIFY_SOCKET") != "" {
+		notified, notifyErr := sddaemon.SdNotify(false, sddaemon.SdNotifyReady)
+		log.G(ctx).Debugf("SdNotifyReady notified=%v, err=%v", notified, notifyErr)
 	}
-}
-
-func sources(ps ...source.GetSources) source.GetSources {
-	return func(labels map[string]string) (source []source.Source, allErr error) {
-		for _, p := range ps {
-			src, err := p(labels)
-			if err == nil {
-				return src, nil
-			}
-			allErr = multierror.Append(allErr, err)
+	defer func() {
+		if os.Getenv("NOTIFY_SOCKET") != "" {
+			notified, notifyErr := sddaemon.SdNotify(false, sddaemon.SdNotifyStopping)
+			log.G(ctx).Debugf("SdNotifyStopping notified=%v, err=%v", notified, notifyErr)
 		}
-		return
+	}()
+
+	var s os.Signal
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, unix.SIGINT, unix.SIGTERM)
+	select {
+	case s = <-sigCh:
+		log.G(ctx).Infof("Got %v", s)
+	case err := <-errCh:
+		return false, err
 	}
+	if s == unix.SIGINT {
+		return true, nil // do cleanup on SIGINT
+	}
+	return false, nil
 }

cmd/containerd-stargz-grpc/server.go

@@ -0,0 +1,34 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"expvar"
+	"net/http"
+	"net/http/pprof"
+)
+
+func debugServerMux() *http.ServeMux {
+	m := http.NewServeMux()
+	m.Handle("/debug/vars", expvar.Handler())
+	m.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
+	m.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+	m.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+	m.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+	m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+	return m
+}

cmd/ctr-remote/commands/convert.go

@@ -18,24 +18,32 @@ package commands
 
 import (
 	"compress/gzip"
+	gocontext "context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
+	"os/signal"
 
-	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/platforms"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/recorder"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/containerd/containerd/v2/core/images/converter/uncompress"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
 	"github.com/containerd/stargz-snapshotter/estargz"
-	"github.com/containerd/stargz-snapshotter/nativeconverter"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
-	"github.com/containerd/stargz-snapshotter/nativeconverter/uncompress"
+	esgzexternaltocconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz/externaltoc"
+	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
+	"github.com/containerd/stargz-snapshotter/recorder"
+	"github.com/klauspost/compress/zstd"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-var ConvertCommand = cli.Command{
+// ConvertCommand converts an image
+var ConvertCommand = &cli.Command{
 	Name:      "convert",
 	Usage:     "convert an image",
 	ArgsUsage: "[flags] <source_ref> <target_ref>...",
@@ -48,47 +56,79 @@ When '--all-platforms' is given all images in a manifest list must be available.
 `,
 	Flags: []cli.Flag{
 		// estargz flags
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "estargz",
 			Usage: "convert legacy tar(.gz) layers to eStargz for lazy pulling. Should be used in conjunction with '--oci'",
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:  "estargz-record-in",
 			Usage: "Read 'ctr-remote optimize --record-out=<FILE>' record file",
 		},
-		cli.IntFlag{
+		&cli.IntFlag{
 			Name:  "estargz-compression-level",
 			Usage: "eStargz compression level",
 			Value: gzip.BestCompression,
 		},
-		cli.IntFlag{
+		&cli.IntFlag{
 			Name:  "estargz-chunk-size",
 			Usage: "eStargz chunk size",
 			Value: 0,
 		},
+		&cli.IntFlag{
+			Name:  "estargz-min-chunk-size",
+			Usage: "The minimal number of bytes of data must be written in one gzip stream. Note that this adds a TOC property that old reader doesn't understand.",
+			Value: 0,
+		},
+		&cli.BoolFlag{
+			Name:  "estargz-external-toc",
+			Usage: "Separate TOC JSON into another image (called \"TOC image\"). The name of TOC image is the original + \"-esgztoc\" suffix. Both eStargz and the TOC image should be pushed to the same registry. stargz-snapshotter refers to the TOC image when it pulls the result eStargz image.",
+		},
+		&cli.BoolFlag{
+			Name:  "estargz-keep-diff-id",
+			Usage: "convert to esgz without changing diffID (cannot be used in conjunction with '--estargz-record-in'. must be specified with '--estargz-external-toc')",
+		},
+		// zstd:chunked flags
+		&cli.BoolFlag{
+			Name:  "zstdchunked",
+			Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked). Must be used in conjunction with '--oci'.",
+		},
+		&cli.StringFlag{
+			Name:  "zstdchunked-record-in",
+			Usage: "Read 'ctr-remote optimize --record-out=<FILE>' record file",
+		},
+		&cli.IntFlag{
+			Name:  "zstdchunked-compression-level",
+			Usage: "zstd:chunked compression level",
+			Value: 3, // SpeedDefault; see also https://pkg.go.dev/github.com/klauspost/compress/zstd#EncoderLevel
+		},
+		&cli.IntFlag{
+			Name:  "zstdchunked-chunk-size",
+			Usage: "zstd:chunked chunk size",
+			Value: 0,
+		},
 		// generic flags
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "uncompress",
 			Usage: "convert tar.gz layers to uncompressed tar layers",
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "oci",
 			Usage: "convert Docker media types to OCI media types",
 		},
 		// platform flags
-		cli.StringSliceFlag{
+		&cli.StringSliceFlag{
 			Name:  "platform",
 			Usage: "Convert content for a specific platform",
 			Value: &cli.StringSlice{},
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "all-platforms",
 			Usage: "Convert content for all platforms",
 		},
 	},
 	Action: func(context *cli.Context) error {
 		var (
-			convertOpts = []nativeconverter.ConvertOpt{}
+			convertOpts = []converter.Opt{}
 		)
 		srcRef := context.Args().Get(0)
 		targetRef := context.Args().Get(1)
@@ -96,42 +136,89 @@ When '--all-platforms' is given all images in a manifest list must be available.
 			return errors.New("src and target image need to be specified")
 		}
 
-		if !context.Bool("all-platforms") {
+		var platformMC platforms.MatchComparer
+		if context.Bool("all-platforms") {
+			platformMC = platforms.All
+		} else {
 			if pss := context.StringSlice("platform"); len(pss) > 0 {
 				var all []ocispec.Platform
 				for _, ps := range pss {
 					p, err := platforms.Parse(ps)
 					if err != nil {
-						return errors.Wrapf(err, "invalid platform %q", ps)
+						return fmt.Errorf("invalid platform %q: %w", ps, err)
 					}
 					all = append(all, p)
 				}
-				convertOpts = append(convertOpts, nativeconverter.WithPlatform(platforms.Ordered(all...)))
+				platformMC = platforms.Ordered(all...)
 			} else {
-				convertOpts = append(convertOpts, nativeconverter.WithPlatform(platforms.Default()))
+				platformMC = platforms.DefaultStrict()
 			}
 		}
+		convertOpts = append(convertOpts, converter.WithPlatform(platformMC))
 
+		var layerConvertFunc converter.ConvertFunc
+		var finalize func(ctx gocontext.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
 		if context.Bool("estargz") {
 			esgzOpts, err := getESGZConvertOpts(context)
 			if err != nil {
 				return err
 			}
-			convertOpts = append(convertOpts, nativeconverter.WithLayerConvertFunc(estargzconvert.LayerConvertFunc(esgzOpts...)))
+			if context.Bool("estargz-external-toc") {
+				if !context.Bool("estargz-keep-diff-id") {
+					layerConvertFunc, finalize = esgzexternaltocconvert.LayerConvertFunc(esgzOpts, context.Int("estargz-compression-level"))
+				} else {
+					if context.String("estargz-record-in") != "" {
+						return fmt.Errorf("option --estargz-keep-diff-id conflicts with --estargz-record-in")
+					}
+					layerConvertFunc, finalize = esgzexternaltocconvert.LayerConvertLossLessFunc(esgzexternaltocconvert.LayerConvertLossLessConfig{
+						CompressionLevel: context.Int("estargz-compression-level"),
+						ChunkSize:        context.Int("estargz-chunk-size"),
+						MinChunkSize:     context.Int("estargz-min-chunk-size"),
+					})
+				}
+			} else {
+				if context.Bool("estargz-keep-diff-id") {
+					return fmt.Errorf("option --estargz-keep-diff-id must be used with --estargz-external-toc")
+				}
+				layerConvertFunc = estargzconvert.LayerConvertFunc(esgzOpts...)
+			}
 			if !context.Bool("oci") {
-				logrus.Warn("option --estargz should be used in conjunction with --oci")
+				log.L.Warn("option --estargz should be used in conjunction with --oci")
 			}
 			if context.Bool("uncompress") {
 				return errors.New("option --estargz conflicts with --uncompress")
 			}
+			if context.Bool("zstdchunked") {
+				return errors.New("option --estargz conflicts with --zstdchunked")
+			}
+		}
+
+		if context.Bool("zstdchunked") {
+			esgzOpts, err := getZstdchunkedConvertOpts(context)
+			if err != nil {
+				return err
+			}
+			layerConvertFunc = zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(
+				zstd.EncoderLevelFromZstd(context.Int("zstdchunked-compression-level")), esgzOpts...)
+			if !context.Bool("oci") {
+				return errors.New("option --zstdchunked must be used in conjunction with --oci")
+			}
+			if context.Bool("uncompress") {
+				return errors.New("option --zstdchunked conflicts with --uncompress")
+			}
 		}
 
 		if context.Bool("uncompress") {
-			convertOpts = append(convertOpts, nativeconverter.WithLayerConvertFunc(uncompress.LayerConvertFunc))
+			layerConvertFunc = uncompress.LayerConvertFunc
 		}
 
+		if layerConvertFunc == nil {
+			return errors.New("specify layer converter")
+		}
+		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc))
+
 		if context.Bool("oci") {
-			convertOpts = append(convertOpts, nativeconverter.WithDockerToOCI(true))
+			convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
 		}
 
 		client, ctx, cancel, err := commands.NewClient(context)
@@ -140,14 +227,40 @@ When '--all-platforms' is given all images in a manifest list must be available.
 		}
 		defer cancel()
 
-		conv, err := nativeconverter.New(client)
+		ctx, done, err := client.WithLease(ctx)
 		if err != nil {
 			return err
 		}
-		newImg, err := conv.Convert(ctx, targetRef, srcRef, convertOpts...)
+		defer done(ctx)
+
+		sigCh := make(chan os.Signal, 1)
+		signal.Notify(sigCh, os.Interrupt)
+		go func() {
+			// Cleanly cancel conversion
+			select {
+			case s := <-sigCh:
+				log.G(ctx).Infof("Got %v", s)
+				cancel()
+			case <-ctx.Done():
+			}
+		}()
+		newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
 		if err != nil {
 			return err
 		}
+		if finalize != nil {
+			newI, err := finalize(ctx, client.ContentStore(), targetRef, &newImg.Target)
+			if err != nil {
+				return err
+			}
+			is := client.ImageService()
+			_ = is.Delete(ctx, newI.Name)
+			finimg, err := is.Create(ctx, *newI)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(context.App.Writer, "extra image:", finimg.Name)
+		}
 		fmt.Fprintln(context.App.Writer, newImg.Target.Digest.String())
 		return nil
 	},
@@ -157,6 +270,7 @@ func getESGZConvertOpts(context *cli.Context) ([]estargz.Option, error) {
 	esgzOpts := []estargz.Option{
 		estargz.WithCompressionLevel(context.Int("estargz-compression-level")),
 		estargz.WithChunkSize(context.Int("estargz-chunk-size")),
+		estargz.WithMinChunkSize(context.Int("estargz-min-chunk-size")),
 	}
 	if estargzRecordIn := context.String("estargz-record-in"); estargzRecordIn != "" {
 		paths, err := readPathsFromRecordFile(estargzRecordIn)
@@ -170,6 +284,22 @@ func getESGZConvertOpts(context *cli.Context) ([]estargz.Option, error) {
 	return esgzOpts, nil
 }
 
+func getZstdchunkedConvertOpts(context *cli.Context) ([]estargz.Option, error) {
+	esgzOpts := []estargz.Option{
+		estargz.WithChunkSize(context.Int("zstdchunked-chunk-size")),
+	}
+	if zstdchunkedRecordIn := context.String("zstdchunked-record-in"); zstdchunkedRecordIn != "" {
+		paths, err := readPathsFromRecordFile(zstdchunkedRecordIn)
+		if err != nil {
+			return nil, err
+		}
+		esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths))
+		var ignored []string
+		esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored))
+	}
+	return esgzOpts, nil
+}
+
 func readPathsFromRecordFile(filename string) ([]string, error) {
 	r, err := os.Open(filename)
 	if err != nil {

cmd/ctr-remote/commands/flags.go

@@ -0,0 +1,517 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package commands
+
+import (
+	"bytes"
+	gocontext "context"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/contrib/nvidia"
+	"github.com/containerd/containerd/v2/core/containers"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/pkg/netns"
+	"github.com/containerd/containerd/v2/pkg/oci"
+	gocni "github.com/containerd/go-cni"
+	"github.com/containerd/log"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/rs/xid"
+	"github.com/urfave/cli/v2"
+)
+
+const netnsMountDir = "/var/run/netns"
+
+func parseGPUs(gpuStr string) ([]int, bool) {
+	if gpuStr == "" {
+		return nil, false
+	}
+	if gpuStr == "all" {
+		return nil, true
+	}
+	parts := strings.Split(gpuStr, ",")
+	var devices []int
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if device, err := strconv.Atoi(part); err == nil {
+			devices = append(devices, device)
+		}
+	}
+	return devices, false
+}
+
+var samplerFlags = []cli.Flag{
+	&cli.BoolFlag{
+		Name:  "terminal,t",
+		Usage: "enable terminal for sample container. must be specified with i option",
+	},
+	&cli.BoolFlag{
+		Name:  "i",
+		Usage: "attach stdin to the container",
+	},
+	&cli.IntFlag{
+		Name:  "period",
+		Usage: "time period to monitor access log",
+		Value: defaultPeriod,
+	},
+	&cli.StringFlag{
+		Name:  "user",
+		Usage: "user/group name to override image's default config(user[:group])",
+	},
+	&cli.StringFlag{
+		Name:  "cwd",
+		Usage: "working dir to override image's default config",
+	},
+	&cli.StringFlag{
+		Name:  "args",
+		Usage: "command arguments to override image's default config(in JSON array)",
+	},
+	&cli.StringFlag{
+		Name:  "entrypoint",
+		Usage: "entrypoint to override image's default config(in JSON array)",
+	},
+	&cli.StringSliceFlag{
+		Name:  "env",
+		Usage: "environment valulable to add or override to the image's default config",
+	},
+	&cli.StringFlag{
+		Name:  "env-file",
+		Usage: "specify additional container environment variables in a file(i.e. FOO=bar, one per line)",
+	},
+	&cli.StringSliceFlag{
+		Name:  "mount",
+		Usage: "additional mounts for the container (e.g. type=foo,source=/path,destination=/target,options=bind)",
+	},
+	&cli.StringFlag{
+		Name:  "dns-nameservers",
+		Usage: "comma-separated nameservers added to the container's /etc/resolv.conf",
+		Value: "8.8.8.8",
+	},
+	&cli.StringFlag{
+		Name:  "dns-search-domains",
+		Usage: "comma-separated search domains added to the container's /etc/resolv.conf",
+	},
+	&cli.StringFlag{
+		Name:  "dns-options",
+		Usage: "comma-separated options added to the container's /etc/resolv.conf",
+	},
+	&cli.StringFlag{
+		Name:  "add-hosts",
+		Usage: "comma-separated hosts configuration (host:IP) added to container's /etc/hosts",
+	},
+	&cli.BoolFlag{
+		Name:  "cni",
+		Usage: "enable CNI-based networking",
+	},
+	&cli.StringFlag{
+		Name:  "cni-plugin-conf-dir",
+		Usage: "path to the CNI plugins configuration directory",
+	},
+	&cli.StringFlag{
+		Name:  "cni-plugin-dir",
+		Usage: "path to the CNI plugins binary directory",
+	},
+	&cli.StringFlag{
+		Name:  "gpus",
+		Usage: "add gpus to the container (comma-separated list of indices or 'all')",
+	},
+	&cli.BoolFlag{
+		Name:  "net-host",
+		Usage: "enable host networking in the container",
+	},
+}
+
+func getSpecOpts(clicontext *cli.Context) func(image containerd.Image, rootfs string) (opts []oci.SpecOpts, done func() error, rErr error) {
+	return func(image containerd.Image, rootfs string) (opts []oci.SpecOpts, done func() error, rErr error) {
+		var cleanups []func() error
+		done = func() error {
+			var errs []error
+			for i := len(cleanups) - 1; i >= 0; i-- {
+				if err := cleanups[i](); err != nil {
+					errs = append(errs, err)
+				}
+			}
+			return errors.Join(errs...)
+		}
+		defer func() {
+			if rErr != nil {
+				if err := done(); err != nil {
+					rErr = fmt.Errorf("failed to cleanup: %w", rErr)
+				}
+			}
+		}()
+
+		entrypointOpt, err := withEntrypointArgs(clicontext, image)
+		if err != nil {
+			rErr = fmt.Errorf("failed to parse entrypoint and arg flags: %w", err)
+			return
+		}
+		resolverOpt, cleanup, err := withResolveConfig(clicontext)
+		if err != nil {
+			rErr = fmt.Errorf("failed to parse DNS-related flags: %w", err)
+			return
+		}
+		cleanups = append(cleanups, cleanup)
+		var mounts []runtimespec.Mount
+		for _, mount := range clicontext.StringSlice("mount") {
+			m, err := parseMountFlag(mount)
+			if err != nil {
+				rErr = fmt.Errorf("failed to parse mount flag %q: %w", mount, err)
+				return
+			}
+			mounts = append(mounts, m)
+		}
+		opts = append(opts,
+			oci.WithDefaultSpec(),
+			oci.WithDefaultUnixDevices,
+			oci.WithRootFSPath(rootfs),
+			oci.WithImageConfig(image),
+			oci.WithEnv(clicontext.StringSlice("env")),
+			oci.WithMounts(mounts),
+			resolverOpt,
+			entrypointOpt,
+		)
+		if envFile := clicontext.String("env-file"); envFile != "" {
+			opts = append(opts, oci.WithEnvFile(envFile))
+		}
+		if username := clicontext.String("user"); username != "" {
+			opts = append(opts, oci.WithUser(username))
+		}
+		if cwd := clicontext.String("cwd"); cwd != "" {
+			opts = append(opts, oci.WithProcessCwd(cwd))
+		}
+		if clicontext.Bool("terminal") {
+			if !clicontext.Bool("i") {
+				rErr = fmt.Errorf("terminal flag must be specified with \"-i\"")
+				return
+			}
+			opts = append(opts, oci.WithTTY)
+		}
+		if clicontext.Bool("cni") {
+			var nOpt oci.SpecOpts
+			nOpt, cleanup, err = withCNI(clicontext)
+			if err != nil {
+				rErr = fmt.Errorf("failed to parse CNI-related flags: %w", err)
+				return
+			}
+			cleanups = append(cleanups, cleanup)
+			opts = append(opts, nOpt)
+		}
+		if clicontext.Bool("net-host") {
+			if runtime.GOOS == "windows" {
+				log.L.Warn("option --net-host is not supported on Windows")
+			} else {
+				opts = append(opts, oci.WithHostNamespace(runtimespec.NetworkNamespace), oci.WithHostHostsFile, oci.WithHostResolvconf)
+			}
+		}
+		if clicontext.IsSet("gpus") {
+			if runtime.GOOS == "windows" {
+				log.L.Warn("option --gpus is not supported on Windows")
+			} else {
+				devices, useAll := parseGPUs(clicontext.String("gpus"))
+				if useAll {
+					opts = append(opts, nvidia.WithGPUs(nvidia.WithAllCapabilities))
+				} else if len(devices) > 0 {
+					opts = append(opts, nvidia.WithGPUs(nvidia.WithDevices(devices...), nvidia.WithAllCapabilities))
+				}
+			}
+		}
+
+		return
+	}
+}
+
+func withEntrypointArgs(clicontext *cli.Context, image containerd.Image) (oci.SpecOpts, error) {
+	var eFlag []string
+	if eStr := clicontext.String("entrypoint"); eStr != "" {
+		if err := json.Unmarshal([]byte(eStr), &eFlag); err != nil {
+			return nil, fmt.Errorf("invalid option \"entrypoint\": %w", err)
+		}
+	}
+	var aFlag []string
+	if aStr := clicontext.String("args"); aStr != "" {
+		if err := json.Unmarshal([]byte(aStr), &aFlag); err != nil {
+			return nil, fmt.Errorf("invalid option \"args\": %w", err)
+		}
+	}
+	return func(ctx gocontext.Context, client oci.Client, container *containers.Container, s *runtimespec.Spec) error {
+		configDesc, err := image.Config(ctx)
+		if err != nil {
+			return err
+		}
+		var ociimage imagespec.Image
+		switch configDesc.MediaType {
+		case imagespec.MediaTypeImageConfig, images.MediaTypeDockerSchema2Config:
+			p, err := content.ReadBlob(ctx, image.ContentStore(), configDesc)
+			if err != nil {
+				return err
+			}
+			if err := json.Unmarshal(p, &ociimage); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unknown image config media type %s", configDesc.MediaType)
+		}
+		entrypoint := ociimage.Config.Entrypoint
+		if len(eFlag) > 0 {
+			entrypoint = eFlag
+		}
+		args := ociimage.Config.Cmd
+		if len(aFlag) > 0 {
+			args = aFlag
+		}
+		return oci.WithProcessArgs(append(entrypoint, args...)...)(ctx, client, container, s)
+	}, nil
+}
+
+func withCNI(clicontext *cli.Context) (specOpt oci.SpecOpts, done func() error, rErr error) {
+	var cleanups []func() error
+	done = func() error {
+		var errs []error
+		for i := len(cleanups) - 1; i >= 0; i-- {
+			if err := cleanups[i](); err != nil {
+				errs = append(errs, err)
+			}
+		}
+		return errors.Join(errs...)
+	}
+	defer func() {
+		if rErr != nil {
+			if err := done(); err != nil {
+				rErr = fmt.Errorf("failed to cleanup: %w", rErr)
+			}
+		}
+	}()
+
+	// Create a new network namespace for configuring it with CNI plugins
+	ns, err := netns.NewNetNS(netnsMountDir)
+	if err != nil {
+		rErr = fmt.Errorf("failed to prepare netns: %w", err)
+		return
+	}
+	cleanups = append(cleanups, ns.Remove)
+
+	// Configure the namespace with CNI plugins
+	var cniopts []gocni.Opt
+	if cdir := clicontext.String("cni-plugin-conf-dir"); cdir != "" {
+		cniopts = append(cniopts, gocni.WithPluginConfDir(cdir))
+	}
+	if pdir := clicontext.String("cni-plugin-dir"); pdir != "" {
+		cniopts = append(cniopts, gocni.WithPluginDir([]string{pdir}))
+	}
+	// The first-found configration file will be effective
+	// TODO: Should we make the number of reading files configurable?
+	cniopts = append(cniopts, gocni.WithDefaultConf)
+	network, err := gocni.New(cniopts...)
+	if err != nil {
+		rErr = fmt.Errorf("failed to prepare CNI plugins: %w", err)
+		return
+	}
+	id := xid.New().String()
+	ctx := gocontext.Background()
+	if _, err := network.Setup(ctx, id, ns.GetPath()); err != nil {
+		rErr = fmt.Errorf("failed to setup netns with CNI plugins: %w", err)
+		return
+	}
+	cleanups = append(cleanups, func() error {
+		return network.Remove(ctx, id, ns.GetPath())
+	})
+
+	// Make the container use this network namespace
+	return oci.WithLinuxNamespace(runtimespec.LinuxNamespace{
+		Type: runtimespec.NetworkNamespace,
+		Path: ns.GetPath(),
+	}), done, nil
+}
+
+func withResolveConfig(clicontext *cli.Context) (specOpt oci.SpecOpts, cleanup func() error, rErr error) {
+	defer func() {
+		if rErr != nil {
+			if err := cleanup(); err != nil {
+				rErr = fmt.Errorf("failed to cleanup: %w", rErr)
+			}
+		}
+	}()
+
+	extrahosts, nameservers, searches, dnsopts, err := parseResolveFlag(clicontext)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Generate /etc/hosts and /etc/resolv.conf
+	resolvDir, err := os.MkdirTemp("", "tmpetc")
+	if err != nil {
+		return nil, nil, err
+	}
+	cleanup = func() error { return os.RemoveAll(resolvDir) }
+	var (
+		etcHostsPath      = filepath.Join(resolvDir, "hosts")
+		etcResolvConfPath = filepath.Join(resolvDir, "resolv.conf")
+		buf               = new(bytes.Buffer)
+	)
+	for _, n := range nameservers {
+		if _, err := fmt.Fprintf(buf, "nameserver %s\n", n); err != nil {
+			rErr = fmt.Errorf("failed to prepare nameserver of /etc/resolv.conf: %w", err)
+			return
+		}
+	}
+
+	if len(searches) > 0 {
+		_, err := fmt.Fprintf(buf, "search %s\n", strings.Join(searches, " "))
+		if err != nil {
+			rErr = fmt.Errorf("failed to prepare search contents of /etc/resolv.conf: %w", err)
+			return
+		}
+	}
+	if len(dnsopts) > 0 {
+		_, err := fmt.Fprintf(buf, "options %s\n", strings.Join(dnsopts, " "))
+		if err != nil {
+			rErr = fmt.Errorf("failed to prepare options contents of /etc/resolv.conf: %w", err)
+			return
+		}
+	}
+	if err := os.WriteFile(etcResolvConfPath, buf.Bytes(), 0644); err != nil {
+		rErr = fmt.Errorf("failed to write contents to /etc/resolv.conf: %w", err)
+		return
+	}
+	buf.Reset() // Reusing for /etc/hosts
+	for _, h := range []struct {
+		host string
+		ip   string
+	}{
+		// Configuration compatible to docker's config
+		// https://github.com/moby/libnetwork/blob/535ef365dc1dd82a5135803a58bc6198a3b9aa27/etchosts/etchosts.go#L28-L36
+		{"localhost", "127.0.0.1"},
+		{"localhost ip6-localhost ip6-loopback", "::1"},
+		{"ip6-localnet", "fe00::0"},
+		{"ip6-mcastprefix", "ff00::0"},
+		{"ip6-allnodes", "ff02::1"},
+		{"ip6-allrouters", "ff02::2"},
+	} {
+		if _, err := fmt.Fprintf(buf, "%s\t%s\n", h.ip, h.host); err != nil {
+			rErr = fmt.Errorf("failed to write default hosts to /etc/hosts: %w", err)
+			return
+		}
+	}
+	for _, h := range extrahosts {
+		parts := strings.SplitN(h, ":", 2) // host:IP
+		if len(parts) != 2 {
+			rErr = fmt.Errorf("cannot parse %q as extra host; must be \"host:IP\"", h)
+			return
+		}
+		// TODO: Validate them
+		if _, err := fmt.Fprintf(buf, "%s\t%s\n", parts[1], parts[0]); err != nil {
+			rErr = fmt.Errorf("failed to write extra hosts to /etc/hosts: %w", err)
+			return
+		}
+	}
+	if err := os.WriteFile(etcHostsPath, buf.Bytes(), 0644); err != nil {
+		rErr = fmt.Errorf("failed to write contents to /etc/hosts: %w", err)
+		return
+	}
+
+	return oci.WithMounts([]runtimespec.Mount{
+		{
+			Destination: "/etc/resolv.conf",
+			Source:      etcResolvConfPath,
+			Options:     []string{"bind"},
+		},
+		{
+			Destination: "/etc/hosts",
+			Source:      etcHostsPath,
+			Options:     []string{"bind"},
+		},
+	}), cleanup, nil
+
+}
+
+// parseMountFlag parses a mount string in the form "type=foo,source=/path,destination=/target,options=rbind:rw"
+func parseMountFlag(m string) (runtimespec.Mount, error) {
+	mount := runtimespec.Mount{}
+	r := csv.NewReader(strings.NewReader(m))
+
+	fields, err := r.Read()
+	if err != nil {
+		return mount, err
+	}
+
+	for _, field := range fields {
+		v := strings.Split(field, "=")
+		if len(v) != 2 {
+			return mount, fmt.Errorf("invalid mount specification: expected key=val")
+		}
+
+		key := v[0]
+		val := v[1]
+		switch key {
+		case "type":
+			mount.Type = val
+		case "source", "src":
+			mount.Source = val
+		case "destination", "dst":
+			mount.Destination = val
+		case "options":
+			mount.Options = strings.Split(val, ":")
+		default:
+			return mount, fmt.Errorf("mount option %q not supported", key)
+		}
+	}
+
+	return mount, nil
+}
+
+func parseResolveFlag(clicontext *cli.Context) (hosts []string, nameservers []string, searches []string, dnsopts []string, _ error) {
+	if nFlag := clicontext.String("dns-nameservers"); nFlag != "" {
+		fields, err := csv.NewReader(strings.NewReader(nFlag)).Read()
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		nameservers = append(nameservers, fields...)
+	}
+	if sFlag := clicontext.String("dns-search-domains"); sFlag != "" {
+		fields, err := csv.NewReader(strings.NewReader(sFlag)).Read()
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		searches = append(searches, fields...)
+	}
+	if oFlag := clicontext.String("dns-options"); oFlag != "" {
+		fields, err := csv.NewReader(strings.NewReader(oFlag)).Read()
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		dnsopts = append(dnsopts, fields...)
+	}
+	if hFlag := clicontext.String("add-hosts"); hFlag != "" {
+		fields, err := csv.NewReader(strings.NewReader(hFlag)).Read()
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		hosts = append(hosts, fields...)
+	}
+	return
+}

cmd/ctr-remote/commands/get-toc-digest.go

@@ -0,0 +1,110 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package commands
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/estargz/zstdchunked"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/urfave/cli/v2"
+)
+
+// GetTOCDigestCommand outputs TOC info of a layer
+var GetTOCDigestCommand = &cli.Command{
+	Name:      "get-toc-digest",
+	Usage:     "get the digest of TOC of a layer",
+	ArgsUsage: "<layer digest>",
+	Flags: []cli.Flag{
+		// zstd:chunked flags
+		&cli.BoolFlag{
+			Name:  "zstdchunked",
+			Usage: "parse layer as zstd:chunked",
+		},
+		// other flags for debugging
+		&cli.BoolFlag{
+			Name:  "dump-toc",
+			Usage: "dump TOC instead of digest. Note that the dumped TOC might be formatted with indents so may have different digest against the original in the layer",
+		},
+	},
+	Action: func(clicontext *cli.Context) error {
+		layerDgstStr := clicontext.Args().Get(0)
+		if layerDgstStr == "" {
+			return errors.New("layer digest need to be specified")
+		}
+
+		client, ctx, cancel, err := commands.NewClient(clicontext)
+		if err != nil {
+			return err
+		}
+		defer cancel()
+
+		layerDgst, err := digest.Parse(layerDgstStr)
+		if err != nil {
+			return err
+		}
+		ra, err := client.ContentStore().ReaderAt(ctx, ocispec.Descriptor{Digest: layerDgst})
+		if err != nil {
+			return err
+		}
+		defer ra.Close()
+
+		footerSize := estargz.FooterSize
+		if clicontext.Bool("zstdchunked") {
+			footerSize = zstdchunked.FooterSize
+		}
+		footer := make([]byte, footerSize)
+		if _, err := ra.ReadAt(footer, ra.Size()-int64(footerSize)); err != nil {
+			return fmt.Errorf("error reading footer: %w", err)
+		}
+
+		var decompressor estargz.Decompressor
+		decompressor = new(estargz.GzipDecompressor)
+		if clicontext.Bool("zstdchunked") {
+			decompressor = new(zstdchunked.Decompressor)
+		}
+
+		_, tocOff, tocSize, err := decompressor.ParseFooter(footer)
+		if err != nil {
+			return fmt.Errorf("error parsing footer: %w", err)
+		}
+		if tocSize <= 0 {
+			tocSize = ra.Size() - tocOff - int64(footerSize)
+		}
+		toc, tocDgst, err := decompressor.ParseTOC(io.NewSectionReader(ra, tocOff, tocSize))
+		if err != nil {
+			return fmt.Errorf("error parsing TOC: %w", err)
+		}
+
+		if clicontext.Bool("dump-toc") {
+			tocJSON, err := json.MarshalIndent(toc, "", "\t")
+			if err != nil {
+				return fmt.Errorf("failed to marshal toc: %w", err)
+			}
+			fmt.Println(string(tocJSON))
+			return nil
+		}
+		fmt.Println(tocDgst.String())
+		return nil
+	},
+}

cmd/ctr-remote/commands/ipfs-push.go

@@ -0,0 +1,99 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package commands
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
+	"github.com/containerd/stargz-snapshotter/ipfs"
+	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/urfave/cli/v2"
+)
+
+// IPFSPushCommand pushes an image to IPFS
+var IPFSPushCommand = &cli.Command{
+	Name:      "ipfs-push",
+	Usage:     "push an image to IPFS (experimental)",
+	ArgsUsage: "[flags] <image_ref>",
+	Flags: []cli.Flag{
+		// platform flags
+		&cli.StringSliceFlag{
+			Name:  "platform",
+			Usage: "Add content for a specific platform",
+			Value: &cli.StringSlice{},
+		},
+		&cli.BoolFlag{
+			Name:  "all-platforms",
+			Usage: "Add content for all platforms",
+		},
+		&cli.BoolFlag{
+			Name:  "estargz",
+			Value: true,
+			Usage: "Convert the image into eStargz",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		srcRef := context.Args().Get(0)
+		if srcRef == "" {
+			return errors.New("image need to be specified")
+		}
+
+		var platformMC platforms.MatchComparer
+		if context.Bool("all-platforms") {
+			platformMC = platforms.All
+		} else {
+			if pss := context.StringSlice("platform"); len(pss) > 0 {
+				var all []ocispec.Platform
+				for _, ps := range pss {
+					p, err := platforms.Parse(ps)
+					if err != nil {
+						return fmt.Errorf("invalid platform %q: %w", ps, err)
+					}
+					all = append(all, p)
+				}
+				platformMC = platforms.Ordered(all...)
+			} else {
+				platformMC = platforms.DefaultStrict()
+			}
+		}
+
+		client, ctx, cancel, err := commands.NewClient(context)
+		if err != nil {
+			return err
+		}
+		defer cancel()
+
+		var layerConvert converter.ConvertFunc
+		if context.Bool("estargz") {
+			layerConvert = estargzconvert.LayerConvertFunc()
+		}
+		p, err := ipfs.Push(ctx, client, srcRef, layerConvert, platformMC)
+		if err != nil {
+			return err
+		}
+		log.L.WithField("CID", p).Infof("Pushed")
+		fmt.Println(p)
+
+		return nil
+	},
+}

cmd/ctr-remote/commands/notify.go

@@ -0,0 +1,38 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package commands
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/containerd/stargz-snapshotter/analyzer/fanotify/service"
+	"github.com/urfave/cli/v2"
+)
+
+// FanotifyCommand notifies filesystem event under the specified directory.
+var FanotifyCommand = &cli.Command{
+	Name:   "fanotify",
+	Hidden: true,
+	Action: func(context *cli.Context) error {
+		target := context.Args().Get(0)
+		if target == "" {
+			return fmt.Errorf("target must be specified")
+		}
+		return service.Serve(target, os.Stdin, os.Stdout)
+	},
+}

cmd/ctr-remote/commands/optimize.go

@@ -14,328 +14,393 @@
    limitations under the License.
 */
 
-/*
-   Copyright 2019 The Go Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style
-   license that can be found in the NOTICE.md file.
-*/
-
 package commands
 
 import (
-	gocontext "context"
-	"encoding/csv"
+	"compress/gzip"
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"os"
-	"path/filepath"
-	"strings"
+	"os/signal"
 	"time"
 
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/platforms"
-	"github.com/containerd/stargz-snapshotter/converter"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/imageio"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/recorder"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/sampler"
-	"github.com/containerd/stargz-snapshotter/util/tempfiles"
-	reglogs "github.com/google/go-containerregistry/pkg/logs"
-	"github.com/google/go-containerregistry/pkg/name"
-	spec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/core/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/images/converter"
+	"github.com/containerd/log"
+	"github.com/containerd/platforms"
+	"github.com/containerd/stargz-snapshotter/analyzer"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/estargz/zstdchunked"
+	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	esgzexternaltocconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz/externaltoc"
+	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
+	"github.com/containerd/stargz-snapshotter/recorder"
+	"github.com/containerd/stargz-snapshotter/util/containerdutil"
+	"github.com/klauspost/compress/zstd"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/urfave/cli/v2"
 )
 
 const defaultPeriod = 10
 
-var OptimizeCommand = cli.Command{
+// OptimizeCommand converts and optimizes an image
+var OptimizeCommand = &cli.Command{
 	Name:      "optimize",
 	Usage:     "optimize an image with user-specified workload",
-	ArgsUsage: "<input-ref> <output-ref>",
-	Flags: []cli.Flag{
-		cli.BoolFlag{
-			Name:  "plain-http",
-			Usage: "allow HTTP connections to the registry which has the prefix \"http://\"",
-		},
-		cli.BoolFlag{
+	ArgsUsage: "[flags] <source_ref> <target_ref>...",
+	Flags: append([]cli.Flag{
+		&cli.BoolFlag{
 			Name:  "reuse",
 			Usage: "reuse eStargz (already optimized) layers without further conversion",
 		},
-		cli.BoolFlag{
-			Name:  "terminal,t",
-			Usage: "enable terminal for sample container",
-		},
-		cli.BoolFlag{
-			Name:  "wait-on-signal",
-			Usage: "ignore context cancel and keep the container running until it receives signal (Ctrl + C) sent manually",
-		},
-		cli.IntFlag{
-			Name:  "period",
-			Usage: "time period to monitor access log",
-			Value: defaultPeriod,
-		},
-		cli.StringFlag{
-			Name:  "user",
-			Usage: "user name to override image's default config",
-		},
-		cli.StringFlag{
-			Name:  "cwd",
-			Usage: "working dir to override image's default config",
-		},
-		cli.StringFlag{
-			Name:  "args",
-			Usage: "command arguments to override image's default config(in JSON array)",
-		},
-		cli.StringFlag{
-			Name:  "entrypoint",
-			Usage: "entrypoint to override image's default config(in JSON array)",
-		},
-		cli.StringSliceFlag{
-			Name:  "env",
-			Usage: "environment valulable to add or override to the image's default config",
-		},
-		cli.StringSliceFlag{
-			Name:  "mount",
-			Usage: "additional mounts for the container (e.g. type=foo,source=/path,destination=/target,options=bind)",
-		},
-		cli.StringFlag{
-			Name:  "dns-nameservers",
-			Usage: "comma-separated nameservers added to the container's /etc/resolv.conf",
-			Value: "8.8.8.8",
-		},
-		cli.StringFlag{
-			Name:  "dns-search-domains",
-			Usage: "comma-separated search domains added to the container's /etc/resolv.conf",
-		},
-		cli.StringFlag{
-			Name:  "dns-options",
-			Usage: "comma-separated options added to the container's /etc/resolv.conf",
-		},
-		cli.StringFlag{
-			Name:  "add-hosts",
-			Usage: "comma-separated hosts configuration (host:IP) added to container's /etc/hosts",
-		},
-		cli.BoolFlag{
-			Name:  "cni",
-			Usage: "enable CNI-based networking",
-		},
-		cli.StringFlag{
-			Name:  "cni-plugin-conf-dir",
-			Usage: "path to the CNI plugins configuration directory",
-		},
-		cli.StringFlag{
-			Name:  "cni-plugin-dir",
-			Usage: "path to the CNI plugins binary directory",
-		},
-		cli.StringFlag{
+		&cli.StringSliceFlag{
 			Name:  "platform",
-			Usage: "platform specifier of the source image",
+			Usage: "Pull content from a specific platform",
+			Value: &cli.StringSlice{},
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
 			Name:  "all-platforms",
 			Usage: "targeting all platform of the source image",
 		},
-		cli.BoolFlag{
+		&cli.BoolFlag{
+			Name:  "wait-on-signal",
+			Usage: "ignore context cancel and keep the container running until it receives SIGINT (Ctrl + C) sent manually",
+		},
+		&cli.StringFlag{
+			Name:  "wait-on-line",
+			Usage: "Substring of a stdout line to be waited. When this string is detected, the container will be killed.",
+		},
+		&cli.BoolFlag{
 			Name:  "no-optimize",
 			Usage: "convert image without optimization",
 		},
-		cli.StringFlag{
+		&cli.StringFlag{
 			Name:  "record-out",
 			Usage: "record the monitor log to the specified file",
 		},
-		// TODO: add "record-in" to use existing record
-	},
-	Action: func(context *cli.Context) error {
-
-		ctx := gocontext.Background()
-
-		// Set up logs package of ggcr to get useful messages
-		reglogs.Warn.SetOutput(log.G(ctx).WriterLevel(logrus.WarnLevel))
-		reglogs.Progress.SetOutput(log.G(ctx).WriterLevel(logrus.InfoLevel))
-
-		// Parse arguments
-		var (
-			src = context.Args().Get(0)
-			dst = context.Args().Get(1)
-		)
-		if src == "" || dst == "" {
-			return fmt.Errorf("source and destination of the target image must be specified")
-		}
-		opts, err := parseArgs(context)
-		if err != nil {
-			return errors.Wrap(err, "failed to parse args")
+		&cli.BoolFlag{
+			Name:  "oci",
+			Usage: "convert Docker media types to OCI media types",
+		},
+		&cli.IntFlag{
+			Name:  "estargz-compression-level",
+			Usage: "eStargz compression level",
+			Value: gzip.BestCompression,
+		},
+		&cli.BoolFlag{
+			Name:  "estargz-external-toc",
+			Usage: "Separate TOC JSON into another image (called \"TOC image\"). The name of TOC image is the original + \"-esgztoc\" suffix. Both eStargz and the TOC image should be pushed to the same registry. stargz-snapshotter refers to the TOC image when it pulls the result eStargz image.",
+		},
+		&cli.IntFlag{
+			Name:  "estargz-chunk-size",
+			Usage: "eStargz chunk size (not applied to zstd:chunked)",
+			Value: 0,
+		},
+		&cli.IntFlag{
+			Name:  "estargz-min-chunk-size",
+			Usage: "The minimal number of bytes of data must be written in one gzip stream. Note that this adds a TOC property that old reader doesn't understand (not applied to zstd:chunked)",
+			Value: 0,
+		},
+		&cli.BoolFlag{
+			Name:  "zstdchunked",
+			Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked)",
+		},
+		&cli.IntFlag{
+			Name:  "zstdchunked-compression-level",
+			Usage: "zstd:chunked compression level",
+			Value: 3, // SpeedDefault; see also https://pkg.go.dev/github.com/klauspost/compress/zstd#EncoderLevel
+		},
+	}, samplerFlags...),
+	Action: func(clicontext *cli.Context) error {
+		convertOpts := []converter.Opt{}
+		srcRef := clicontext.Args().Get(0)
+		targetRef := clicontext.Args().Get(1)
+		if srcRef == "" || targetRef == "" {
+			return errors.New("src and target image need to be specified")
 		}
 
-		// Parse references
-		srcIO, err := parseReference(src, context)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse source ref %q", src)
-		}
-		dstIO, err := parseReference(dst, context)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse destination ref %q", dst)
-		}
-
-		// Parse platform information
-		var platform *spec.Platform
-		if context.Bool("all-platforms") {
-			platform = nil
-		} else if pStr := context.String("platform"); pStr != "" {
-			p, err := platforms.Parse(pStr)
-			if err != nil {
-				return errors.Wrapf(err, "failed to parse platform %q", pStr)
-			}
-			platform = &p
+		var platformMC platforms.MatchComparer
+		if clicontext.Bool("all-platforms") {
+			platformMC = platforms.All
 		} else {
-			p := platforms.DefaultSpec()
-			platform = &p
+			if pss := clicontext.StringSlice("platform"); len(pss) > 0 {
+				var all []ocispec.Platform
+				for _, ps := range pss {
+					p, err := platforms.Parse(ps)
+					if err != nil {
+						return fmt.Errorf("invalid platform %q: %w", ps, err)
+					}
+					all = append(all, p)
+				}
+				platformMC = platforms.Ordered(all...)
+			} else {
+				platformMC = platforms.DefaultStrict()
+			}
+		}
+		convertOpts = append(convertOpts, converter.WithPlatform(platformMC))
+
+		if clicontext.Bool("oci") {
+			convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
+		} else if clicontext.Bool("zstdchunked") {
+			return errors.New("option --zstdchunked must be used in conjunction with --oci")
 		}
 
-		tf := tempfiles.NewTempFiles()
-		defer func() {
-			if err := tf.CleanupAll(); err != nil {
-				log.G(ctx).WithError(err).Warn("failed to cleanup layer files")
-			}
-		}()
-
-		noOptimize := context.Bool("no-optimize")
-		optimizerOpts := &optimizer.Opts{
-			Reuse:  context.Bool("reuse"),
-			Period: time.Duration(context.Int("period")) * time.Second,
-		}
-
-		var rec *recorder.Recorder
-		if recordOut := context.String("record-out"); recordOut != "" {
-			recordWriter, err := os.Create(recordOut)
-			if err != nil {
-				return err
-			}
-			defer recordWriter.Close()
-			rec = recorder.New(recordWriter)
-		}
-
-		// Convert and push the image
-		srcIndex, err := srcIO.ReadIndex()
-		if err != nil {
-			// No index found. Try to deal it as a thin image.
-			log.G(ctx).Warn("index not found; treating as a thin image with ignoring the platform option")
-			srcImage, err := srcIO.ReadImage()
-			if err != nil {
-				return err
-			}
-			p := platforms.DefaultSpec()
-			dstImage, err := converter.ConvertImage(ctx, noOptimize, optimizerOpts, srcImage, &p, tf, rec, opts...)
-			if err != nil {
-				return err
-			}
-			return dstIO.WriteImage(dstImage)
-		}
-		dstIndex, err := converter.ConvertIndex(ctx, noOptimize, optimizerOpts, srcIndex, platform, tf, rec, opts...)
+		client, ctx, cancel, err := commands.NewClient(clicontext)
 		if err != nil {
 			return err
 		}
-		return dstIO.WriteIndex(dstIndex)
+		defer cancel()
+
+		ctx, done, err := client.WithLease(ctx)
+		if err != nil {
+			return err
+		}
+		defer done(ctx)
+
+		recordOut, esgzOptsPerLayer, wrapper, err := analyze(ctx, clicontext, client, srcRef)
+		if err != nil {
+			return err
+		}
+		if recordOutFile := clicontext.String("record-out"); recordOutFile != "" {
+			if err := writeContentFile(ctx, client, recordOut, recordOutFile); err != nil {
+				return fmt.Errorf("failed output record file: %w", err)
+			}
+		}
+		var f converter.ConvertFunc
+		var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
+		if clicontext.Bool("zstdchunked") {
+			f = zstdchunkedconvert.LayerConvertWithLayerOptsFuncWithCompressionLevel(
+				zstd.EncoderLevelFromZstd(clicontext.Int("zstdchunked-compression-level")), esgzOptsPerLayer)
+		} else if !clicontext.Bool("estargz-external-toc") {
+			f = estargzconvert.LayerConvertWithLayerAndCommonOptsFunc(esgzOptsPerLayer,
+				estargz.WithCompressionLevel(clicontext.Int("estargz-compression-level")),
+				estargz.WithChunkSize(clicontext.Int("estargz-chunk-size")),
+				estargz.WithMinChunkSize(clicontext.Int("estargz-min-chunk-size")))
+		} else {
+			if clicontext.Bool("reuse") {
+				// We require that the layer conversion is triggerd for each layer
+				// to make sure that "finalize" function has the information of all layers.
+				return fmt.Errorf("\"estargz-external-toc\" can't be used with \"reuse\" flag")
+			}
+			f, finalize = esgzexternaltocconvert.LayerConvertWithLayerAndCommonOptsFunc(esgzOptsPerLayer, []estargz.Option{
+				estargz.WithChunkSize(clicontext.Int("estargz-chunk-size")),
+				estargz.WithMinChunkSize(clicontext.Int("estargz-min-chunk-size")),
+			}, clicontext.Int("estargz-compression-level"))
+		}
+		if wrapper != nil {
+			f = wrapper(f)
+		}
+		layerConvertFunc := logWrapper(f)
+
+		sigCh := make(chan os.Signal, 1)
+		signal.Notify(sigCh, os.Interrupt)
+		go func() {
+			// Cleanly cancel conversion
+			select {
+			case s := <-sigCh:
+				log.G(ctx).Infof("Got %v", s)
+				cancel()
+			case <-ctx.Done():
+			}
+		}()
+		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc))
+		newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
+		if err != nil {
+			return err
+		}
+		if finalize != nil {
+			newI, err := finalize(ctx, client.ContentStore(), targetRef, &newImg.Target)
+			if err != nil {
+				return err
+			}
+			is := client.ImageService()
+			_ = is.Delete(ctx, newI.Name)
+			finimg, err := is.Create(ctx, *newI)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(clicontext.App.Writer, "extra image:", finimg.Name)
+		}
+		fmt.Fprintln(clicontext.App.Writer, newImg.Target.Digest.String())
+		return nil
 	},
 }
 
-func parseArgs(clicontext *cli.Context) (opts []sampler.Option, err error) {
-	if env := clicontext.StringSlice("env"); len(env) > 0 {
-		opts = append(opts, sampler.WithEnvs(env))
+func writeContentFile(ctx context.Context, client *containerd.Client, dgst digest.Digest, targetFile string) error {
+	fw, err := os.Create(targetFile)
+	if err != nil {
+		return err
 	}
-	if mounts := clicontext.StringSlice("mount"); len(mounts) > 0 {
-		opts = append(opts, sampler.WithMounts(mounts))
+	defer fw.Close()
+	ra, err := client.ContentStore().ReaderAt(ctx, ocispec.Descriptor{Digest: dgst})
+	if err != nil {
+		return err
 	}
-	if args := clicontext.String("args"); args != "" {
-		var as []string
-		err = json.Unmarshal([]byte(args), &as)
-		if err != nil {
-			return nil, errors.Wrapf(err, "invalid option \"args\"")
+	defer ra.Close()
+	_, err = io.Copy(fw, io.NewSectionReader(ra, 0, ra.Size()))
+	return err
+}
+
+func analyze(ctx context.Context, clicontext *cli.Context, client *containerd.Client, srcRef string) (digest.Digest, map[digest.Digest][]estargz.Option, func(converter.ConvertFunc) converter.ConvertFunc, error) {
+	if clicontext.Bool("no-optimize") {
+		return "", nil, nil, nil
+	}
+
+	// Do analysis only when the target platforms contain the current platform
+	if !clicontext.Bool("all-platforms") {
+		if pss := clicontext.StringSlice("platform"); len(pss) > 0 {
+			containsDefault := false
+			for _, ps := range pss {
+				p, err := platforms.Parse(ps)
+				if err != nil {
+					return "", nil, nil, fmt.Errorf("invalid platform %q: %w", ps, err)
+				}
+				if platforms.DefaultStrict().Match(p) {
+					containsDefault = true
+				}
+			}
+			if !containsDefault {
+				return "", nil, nil, nil // do not run analyzer
+			}
 		}
-		opts = append(opts, sampler.WithArgs(as))
 	}
-	if entrypoint := clicontext.String("entrypoint"); entrypoint != "" {
-		var es []string
-		err = json.Unmarshal([]byte(entrypoint), &es)
-		if err != nil {
-			return nil, errors.Wrapf(err, "invalid option \"entrypoint\"")
-		}
-		opts = append(opts, sampler.WithEntrypoint(es))
-	}
-	if username := clicontext.String("user"); username != "" {
-		opts = append(opts, sampler.WithUser(username))
-	}
-	if cwd := clicontext.String("cwd"); cwd != "" {
-		opts = append(opts, sampler.WithWorkingDir(cwd))
-	}
-	if clicontext.Bool("terminal") {
-		opts = append(opts, sampler.WithTerminal())
+
+	cs := client.ContentStore()
+	is := client.ImageService()
+
+	// Analyze layers and get prioritized files
+	aOpts := []analyzer.Option{analyzer.WithSpecOpts(getSpecOpts(clicontext))}
+	if clicontext.Bool("wait-on-signal") && clicontext.Bool("terminal") {
+		return "", nil, nil, fmt.Errorf("wait-on-signal can't be used with terminal flag")
 	}
 	if clicontext.Bool("wait-on-signal") {
-		opts = append(opts, sampler.WithWaitOnSignal())
+		aOpts = append(aOpts, analyzer.WithWaitOnSignal())
+	} else {
+		aOpts = append(aOpts,
+			analyzer.WithPeriod(time.Duration(clicontext.Int("period"))*time.Second),
+			analyzer.WithWaitLineOut(clicontext.String("wait-on-line")))
 	}
-	if nameservers := clicontext.String("dns-nameservers"); nameservers != "" {
-		fields, err := csv.NewReader(strings.NewReader(nameservers)).Read()
-		if err != nil {
-			return nil, err
+	if clicontext.Bool("terminal") {
+		if !clicontext.Bool("i") {
+			return "", nil, nil, fmt.Errorf("terminal flag must be specified with \"-i\"")
 		}
-		opts = append(opts, sampler.WithDNSNameservers(fields))
+		aOpts = append(aOpts, analyzer.WithTerminal())
 	}
-	if search := clicontext.String("dns-search-domains"); search != "" {
-		fields, err := csv.NewReader(strings.NewReader(search)).Read()
-		if err != nil {
-			return nil, err
-		}
-		opts = append(opts, sampler.WithDNSSearchDomains(fields))
+	if clicontext.Bool("i") {
+		aOpts = append(aOpts, analyzer.WithStdin())
 	}
-	if dnsopts := clicontext.String("dns-options"); dnsopts != "" {
-		fields, err := csv.NewReader(strings.NewReader(dnsopts)).Read()
-		if err != nil {
-			return nil, err
-		}
-		opts = append(opts, sampler.WithDNSOptions(fields))
-	}
-	if hosts := clicontext.String("add-hosts"); hosts != "" {
-		fields, err := csv.NewReader(strings.NewReader(hosts)).Read()
-		if err != nil {
-			return nil, err
-		}
-		opts = append(opts, sampler.WithExtraHosts(fields))
-	}
-	if clicontext.Bool("cni") {
-		opts = append(opts, sampler.WithCNI())
-	}
-	if cniPluginConfDir := clicontext.String("cni-plugin-conf-dir"); cniPluginConfDir != "" {
-		opts = append(opts, sampler.WithCNIPluginConfDir(cniPluginConfDir))
-	}
-	if cniPluginDir := clicontext.String("cni-plugin-dir"); cniPluginDir != "" {
-		opts = append(opts, sampler.WithCNIPluginDir(cniPluginDir))
-	}
-
-	return
-}
-
-func parseReference(ref string, clicontext *cli.Context) (imageio.ImageIO, error) {
-	if strings.HasPrefix(ref, "local://") {
-		abspath, err := filepath.Abs(strings.TrimPrefix(ref, "local://"))
-		if err != nil {
-			return nil, err
-		}
-		return &imageio.LocalImage{LocalPath: abspath}, nil
-	}
-	var opts []name.Option
-	if strings.HasPrefix(ref, "http://") {
-		ref = strings.TrimPrefix(ref, "http://")
-		if clicontext.Bool("plain-http") {
-			opts = append(opts, name.Insecure)
-		} else {
-			return nil, fmt.Errorf("\"--plain-http\" option must be specified to connect to %q using HTTP", ref)
-		}
-	}
-	remoteRef, err := name.ParseReference(ref, opts...)
+	recordOut, err := analyzer.Analyze(ctx, client, srcRef, aOpts...)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse reference %q", ref)
+		return "", nil, nil, err
+	}
+
+	// Parse record file
+	srcImg, err := is.Get(ctx, srcRef)
+	if err != nil {
+		return "", nil, nil, err
+	}
+	manifestDesc, err := containerdutil.ManifestDesc(ctx, cs, srcImg.Target, platforms.DefaultStrict())
+	if err != nil {
+		return "", nil, nil, err
+	}
+	p, err := content.ReadBlob(ctx, cs, manifestDesc)
+	if err != nil {
+		return "", nil, nil, err
+	}
+	var manifest ocispec.Manifest
+	if err := json.Unmarshal(p, &manifest); err != nil {
+		return "", nil, nil, err
+	}
+	// TODO: this should be indexed by layer "index" (not "digest")
+	layerLogs := make(map[digest.Digest][]string, len(manifest.Layers))
+	ra, err := cs.ReaderAt(ctx, ocispec.Descriptor{Digest: recordOut})
+	if err != nil {
+		return "", nil, nil, err
+	}
+	defer ra.Close()
+	dec := json.NewDecoder(io.NewSectionReader(ra, 0, ra.Size()))
+	added := make(map[digest.Digest]map[string]struct{}, len(manifest.Layers))
+	for dec.More() {
+		var e recorder.Entry
+		if err := dec.Decode(&e); err != nil {
+			return "", nil, nil, err
+		}
+		if *e.LayerIndex < len(manifest.Layers) &&
+			e.ManifestDigest == manifestDesc.Digest.String() {
+			dgst := manifest.Layers[*e.LayerIndex].Digest
+			if added[dgst] == nil {
+				added[dgst] = map[string]struct{}{}
+			}
+			if _, ok := added[dgst][e.Path]; !ok {
+				added[dgst][e.Path] = struct{}{}
+				layerLogs[dgst] = append(layerLogs[dgst], e.Path)
+			}
+		}
+	}
+
+	// Create a converter wrapper for skipping layer conversion. This skip occurs
+	// if "reuse" option is specified, the source layer is already valid estargz
+	// and no access occur to that layer.
+	var excludes []digest.Digest
+	layerOpts := make(map[digest.Digest][]estargz.Option, len(manifest.Layers))
+	for _, desc := range manifest.Layers {
+		if layerLog, ok := layerLogs[desc.Digest]; ok && len(layerLog) > 0 {
+			layerOpts[desc.Digest] = []estargz.Option{estargz.WithPrioritizedFiles(layerLog)}
+		} else if clicontext.Bool("reuse") && isReusableESGZLayer(ctx, desc, cs) {
+			excludes = append(excludes, desc.Digest) // reuse layer without conversion
+		}
+	}
+	return recordOut, layerOpts, excludeWrapper(excludes), nil
+}
+
+func isReusableESGZLayer(ctx context.Context, desc ocispec.Descriptor, cs content.Store) bool {
+	dgstStr, ok := desc.Annotations[estargz.TOCJSONDigestAnnotation]
+	if !ok {
+		return false
+	}
+	tocdgst, err := digest.Parse(dgstStr)
+	if err != nil {
+		return false
+	}
+	ra, err := cs.ReaderAt(ctx, desc)
+	if err != nil {
+		return false
+	}
+	defer ra.Close()
+	r, err := estargz.Open(io.NewSectionReader(ra, 0, desc.Size), estargz.WithDecompressors(new(zstdchunked.Decompressor)))
+	if err != nil {
+		return false
+	}
+	if _, err := r.VerifyTOC(tocdgst); err != nil {
+		return false
+	}
+	return true
+}
+
+func excludeWrapper(excludes []digest.Digest) func(converter.ConvertFunc) converter.ConvertFunc {
+	return func(convertFunc converter.ConvertFunc) converter.ConvertFunc {
+		return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
+			for _, e := range excludes {
+				if e == desc.Digest {
+					log.G(ctx).Warnf("reusing %q without conversion", e)
+					return nil, nil
+				}
+			}
+			return convertFunc(ctx, cs, desc)
+		}
+	}
+}
+
+func logWrapper(convertFunc converter.ConvertFunc) converter.ConvertFunc {
+	return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
+		log.G(ctx).WithField("digest", desc.Digest).Infof("converting...")
+		return convertFunc(ctx, cs, desc)
 	}
-	return &imageio.RemoteImage{RemoteRef: remoteRef}, nil
 }

cmd/ctr-remote/commands/rpull.go

@@ -20,16 +20,18 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/cmd/ctr/commands/content"
-	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/snapshots"
+	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands"
+	"github.com/containerd/containerd/v2/cmd/ctr/commands/content"
+	"github.com/containerd/containerd/v2/core/images"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	ctdsnapshotters "github.com/containerd/containerd/v2/pkg/snapshotters"
+	"github.com/containerd/log"
 	fsconfig "github.com/containerd/stargz-snapshotter/fs/config"
 	"github.com/containerd/stargz-snapshotter/fs/source"
+	"github.com/containerd/stargz-snapshotter/ipfs"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -37,21 +39,30 @@ const (
 	skipContentVerifyOpt  = "skip-content-verify"
 )
 
-var RpullCommand = cli.Command{
+// RpullCommand is a subcommand to pull an image from a registry leveraging stargz snapshotter
+var RpullCommand = &cli.Command{
 	Name:      "rpull",
-	Usage:     "pull an image from a registry levaraging stargz snapshotter",
+	Usage:     "pull an image from a registry leveraging stargz snapshotter",
 	ArgsUsage: "[flags] <ref>",
-	Description: `Fetch and prepare an image for use in containerd levaraging stargz snapshotter.
+	Description: `Fetch and prepare an image for use in containerd leveraging stargz snapshotter.
 
 After pulling an image, it should be ready to use the same reference in a run
 command. 
 `,
-	Flags: append(commands.RegistryFlags, commands.LabelFlag,
-		cli.BoolFlag{
+	Flags: append(append(commands.RegistryFlags, commands.LabelFlag,
+		&cli.BoolFlag{
 			Name:  skipContentVerifyOpt,
 			Usage: "Skip content verification for layers contained in this image.",
 		},
-	),
+		&cli.BoolFlag{
+			Name:  "ipfs",
+			Usage: "Pull image from IPFS. Specify an IPFS CID as a reference. (experimental)",
+		},
+		&cli.BoolFlag{
+			Name:  "use-containerd-labels",
+			Usage: "Use labels defined in containerd project",
+		},
+	), commands.SnapshotterFlags...),
 	Action: func(context *cli.Context) error {
 		var (
 			ref    = context.Args().First()
@@ -78,21 +89,35 @@ command.
 			return err
 		}
 		config.FetchConfig = fc
+		config.containerdLabels = context.Bool("use-containerd-labels")
 
 		if context.Bool(skipContentVerifyOpt) {
 			config.skipVerify = true
 		}
 
-		if err := pull(ctx, client, ref, config); err != nil {
-			return err
+		if context.Bool("ipfs") {
+			r, err := ipfs.NewResolver(ipfs.ResolverOptions{
+				Scheme: "ipfs",
+			})
+			if err != nil {
+				return err
+			}
+			config.Resolver = r
 		}
-		return nil
+		config.snapshotter = remoteSnapshotterName
+		if sn := context.String("snapshotter"); sn != "" {
+			config.snapshotter = sn
+		}
+
+		return pull(ctx, client, ref, config)
 	},
 }
 
 type rPullConfig struct {
 	*content.FetchConfig
-	skipVerify bool
+	skipVerify       bool
+	snapshotter      string
+	containerdLabels bool
 }
 
 func pull(ctx context.Context, client *containerd.Client, ref string, config *rPullConfig) error {
@@ -112,16 +137,23 @@ func pull(ctx context.Context, client *containerd.Client, ref string, config *rP
 		}))
 	}
 
+	var labelHandler func(h images.Handler) images.Handler
+	prefetchSize := int64(10 * 1024 * 1024)
+	if config.containerdLabels {
+		labelHandler = source.AppendExtraLabelsHandler(prefetchSize, ctdsnapshotters.AppendInfoHandlerWrapper(ref))
+	} else {
+		labelHandler = source.AppendDefaultLabelsHandlerWrapper(ref, prefetchSize)
+	}
+
 	log.G(pCtx).WithField("image", ref).Debug("fetching")
 	labels := commands.LabelArgs(config.Labels)
 	if _, err := client.Pull(pCtx, ref, []containerd.RemoteOpt{
 		containerd.WithPullLabels(labels),
 		containerd.WithResolver(config.Resolver),
 		containerd.WithImageHandler(h),
-		containerd.WithSchema1Conversion,
 		containerd.WithPullUnpack,
-		containerd.WithPullSnapshotter(remoteSnapshotterName, snOpts...),
-		containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(ref, 10*1024*1024)),
+		containerd.WithPullSnapshotter(config.snapshotter, snOpts...),
+		containerd.WithImageHandlerWrapper(labelHandler),
 	}...); err != nil {
 		return err
 	}

cmd/ctr-remote/main.go

@@ -20,27 +20,47 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/containerd/containerd/cmd/ctr/app"
-	"github.com/containerd/containerd/pkg/seed"
+	"github.com/containerd/containerd/v2/cmd/ctr/app"
 	"github.com/containerd/stargz-snapshotter/cmd/ctr-remote/commands"
-	"github.com/urfave/cli"
+	"github.com/urfave/cli/v2"
 )
 
-func init() {
-	seed.WithTimeAndRand()
-}
-
 func main() {
-	customCommands := []cli.Command{commands.RpullCommand, commands.OptimizeCommand, commands.ConvertCommand}
+	customCommands := []*cli.Command{
+		commands.RpullCommand,
+		commands.OptimizeCommand,
+		commands.ConvertCommand,
+		commands.GetTOCDigestCommand,
+		commands.IPFSPushCommand,
+	}
 	app := app.New()
 	for i := range app.Commands {
 		if app.Commands[i].Name == "images" {
-			app.Commands[i].Subcommands = append(app.Commands[i].Subcommands, customCommands...)
+			sc := map[string]*cli.Command{}
+			for _, subcmd := range customCommands {
+				sc[subcmd.Name] = subcmd
+			}
+
+			// First, replace duplicated subcommands
+			for j := range app.Commands[i].Subcommands {
+				for name, subcmd := range sc {
+					if name == app.Commands[i].Subcommands[j].Name {
+						app.Commands[i].Subcommands[j] = subcmd
+						delete(sc, name)
+					}
+				}
+			}
+
+			// Next, append all new sub commands
+			for _, subcmd := range sc {
+				app.Commands[i].Subcommands = append(app.Commands[i].Subcommands, subcmd)
+			}
 			break
 		}
 	}
+	app.Commands = append(app.Commands, commands.FanotifyCommand)
 	if err := app.Run(os.Args); err != nil {
-		fmt.Fprintf(os.Stderr, "ctr: %v\n", err)
+		fmt.Fprintf(os.Stderr, "ctr-remote: %v\n", err)
 		os.Exit(1)
 	}
 }

cmd/go.mod

@@ -0,0 +1,159 @@
+module github.com/containerd/stargz-snapshotter/cmd
+
+go 1.24.0
+
+toolchain go1.24.2
+
+require (
+	github.com/containerd/containerd/api v1.9.0
+	github.com/containerd/containerd/v2 v2.1.4
+	github.com/containerd/go-cni v1.1.13
+	github.com/containerd/log v0.1.0
+	github.com/containerd/platforms v1.0.0-rc.1
+	github.com/containerd/stargz-snapshotter v0.15.2-0.20240622031358-6405f362966d
+	github.com/containerd/stargz-snapshotter/estargz v0.17.0
+	github.com/containerd/stargz-snapshotter/ipfs v0.15.2-0.20240622031358-6405f362966d
+	github.com/coreos/go-systemd/v22 v22.5.0
+	github.com/docker/go-metrics v0.0.1
+	github.com/goccy/go-json v0.10.5
+	github.com/klauspost/compress v1.18.0
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.1.1
+	github.com/opencontainers/runtime-spec v1.2.1
+	github.com/pelletier/go-toml v1.9.5
+	github.com/rs/xid v1.6.0
+	github.com/urfave/cli/v2 v2.27.7
+	go.etcd.io/bbolt v1.4.2
+	golang.org/x/sync v0.16.0
+	golang.org/x/sys v0.34.0
+	google.golang.org/grpc v1.74.2
+)
+
+require (
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.13.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cilium/ebpf v0.16.0 // indirect
+	github.com/containerd/cgroups/v3 v3.0.5 // indirect
+	github.com/containerd/console v1.0.5 // indirect
+	github.com/containerd/continuity v0.4.5 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/go-runc v1.1.0 // indirect
+	github.com/containerd/plugin v1.0.0 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
+	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/containernetworking/cni v1.3.0 // indirect
+	github.com/containernetworking/plugins v1.7.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/cli v28.3.3+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hanwen/go-fuse/v2 v2.8.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
+	github.com/intel/goresctrl v0.8.0 // indirect
+	github.com/ipfs/go-cid v0.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/mdlayher/vsock v1.2.1 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/symlink v0.3.0 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr v0.16.1 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
+	github.com/opencontainers/selinux v1.12.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.23.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sasha-s/go-deadlock v0.3.5 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
+	github.com/vbatts/tar-split v0.12.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
+	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
+	golang.org/x/mod v0.24.0 // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.33.3 // indirect
+	k8s.io/apimachinery v0.33.3 // indirect
+	k8s.io/client-go v0.33.3 // indirect
+	k8s.io/cri-api v0.33.3 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+	tags.cncf.io/container-device-interface v1.0.1 // indirect
+	tags.cncf.io/container-device-interface/specs-go v1.0.0 // indirect
+)
+
+replace (
+	// Import local packages.
+	github.com/containerd/stargz-snapshotter => ../
+	github.com/containerd/stargz-snapshotter/estargz => ../estargz
+	github.com/containerd/stargz-snapshotter/ipfs => ../ipfs
+)

cmd/go.sum

@@ -0,0 +1,539 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/Microsoft/hcsshim v0.13.0 h1:/BcXOiS6Qi7N9XqUcv27vkIuVOkBEcWstd2pMlWSeaA=
+github.com/Microsoft/hcsshim v0.13.0/go.mod h1:9KWJ/8DgU+QzYGupX4tzMhRQE8h6w90lH6HAaclpEok=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok=
+github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo=
+github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins=
+github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc=
+github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
+github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0=
+github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI=
+github.com/containerd/containerd/v2 v2.1.4 h1:/hXWjiSFd6ftrBOBGfAZ6T30LJcx1dBjdKEeI8xucKQ=
+github.com/containerd/containerd/v2 v2.1.4/go.mod h1:8C5QV9djwsYDNhxfTCFjWtTBZrqjditQ4/ghHSYjnHM=
+github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4=
+github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY=
+github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o=
+github.com/containerd/go-cni v1.1.13 h1:eFSGOKlhoYNxpJ51KRIMHZNlg5UgocXEIEBGkY7Hnis=
+github.com/containerd/go-cni v1.1.13/go.mod h1:nTieub0XDRmvCZ9VI/SBG6PyqT95N4FIhxsauF1vSBI=
+github.com/containerd/go-runc v1.1.0 h1:OX4f+/i2y5sUT7LhmcJH7GYrjjhHa1QI4e8yO0gGleA=
+github.com/containerd/go-runc v1.1.0/go.mod h1:xJv2hFF7GvHtTJd9JqTS2UVxMkULUYw4JN5XAUZqH5U=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/platforms v1.0.0-rc.1 h1:83KIq4yy1erSRgOVHNk1HYdPvzdJ5CnsWaRoJX4C41E=
+github.com/containerd/platforms v1.0.0-rc.1/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4=
+github.com/containerd/plugin v1.0.0 h1:c8Kf1TNl6+e2TtMHZt+39yAPDbouRH9WAToRjex483Y=
+github.com/containerd/plugin v1.0.0/go.mod h1:hQfJe5nmWfImiqT1q8Si3jLv3ynMUIBB47bQ+KexvO8=
+github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ=
+github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
+github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40=
+github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk=
+github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo=
+github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4=
+github.com/containernetworking/plugins v1.7.1 h1:CNAR0jviDj6FS5Vg85NTgKWLDzZPfi/lj+VJfhMDTIs=
+github.com/containernetworking/plugins v1.7.1/go.mod h1:xuMdjuio+a1oVQsHKjr/mgzuZ24leAsqUYRnzGoXHy0=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/cli v28.3.3+incompatible h1:fp9ZHAr1WWPGdIWBM1b3zLtgCF+83gRdVMTJsUeiyAo=
+github.com/docker/cli v28.3.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
+github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
+github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
+github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
+github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
+github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hanwen/go-fuse/v2 v2.8.0 h1:wV8rG7rmCz8XHSOwBZhG5YcVqcYjkzivjmbaMafPlAs=
+github.com/hanwen/go-fuse/v2 v2.8.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
+github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
+github.com/intel/goresctrl v0.8.0 h1:N3shVbS3kA1Hk2AmcbHv8805Hjbv+zqsCIZCGktxx50=
+github.com/intel/goresctrl v0.8.0/go.mod h1:T3ZZnuHSNouwELB5wvOoUJaB7l/4Rm23rJy/wuWJlr0=
+github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0=
+github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
+github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM=
+github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
+github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
+github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
+github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
+github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ=
+github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
+github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
+github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
+github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
+github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg=
+github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
+github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0=
+github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8=
+github.com/moby/sys/symlink v0.3.0 h1:GZX89mEZ9u53f97npBy4Rc3vJKj7JBDj/PN2I22GrNU=
+github.com/moby/sys/symlink v0.3.0/go.mod h1:3eNdhduHmYPcgsJtZXW1W4XUJdZGBIkttZ8xKqPUJq0=
+github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
+github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
+github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
+github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=
+github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
+github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
+github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
+github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw=
+github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0=
+github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
+github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
+github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
+github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
+github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
+github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
+github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
+github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
+github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
+github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
+github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
+github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww=
+github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
+github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
+github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
+github.com/opencontainers/selinux v1.12.0 h1:6n5JV4Cf+4y0KNXW48TLj5DwfXpvWlxXplUkdTrmPb8=
+github.com/opencontainers/selinux v1.12.0/go.mod h1:BTPX+bjVbWGXw7ZZWUbdENt8w0htPSrlgOOysQaU62U=
+github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw=
+github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc=
+github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
+github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU=
+github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
+github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
+github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
+github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo=
+github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=
+github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
+github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I=
+go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ=
+go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
+go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
+go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
+go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
+go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
+go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis=
+go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
+go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
+go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
+go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
+go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
+golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo=
+golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
+golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
+golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
+golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
+golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
+golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
+golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4=
+google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY=
+gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
+k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
+k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
+k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
+k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
+k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
+k8s.io/cri-api v0.33.3 h1:aQvK3UxsaVMul4z71lOiblMHdhw9ROaw3Cgg15xDrD4=
+k8s.io/cri-api v0.33.3/go.mod h1:OLQvT45OpIA+tv91ZrpuFIGY+Y2Ho23poS7n115Aocs=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
+k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
+k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=
+k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
+lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
+sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
+sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
+sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
+sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc=
+tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0=
+tags.cncf.io/container-device-interface/specs-go v1.0.0 h1:8gLw29hH1ZQP9K1YtAzpvkHCjjyIxHZYzBAvlQ+0vD8=
+tags.cncf.io/container-device-interface/specs-go v1.0.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ=

cmd/stargz-fuse-manager/main.go

@@ -0,0 +1,97 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/log"
+
+	"github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/fsopts"
+	fusemanager "github.com/containerd/stargz-snapshotter/fusemanager"
+	"github.com/containerd/stargz-snapshotter/service"
+	"github.com/containerd/stargz-snapshotter/service/keychain/keychainconfig"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	fusemanager.RegisterConfigFunc(func(cc *fusemanager.ConfigContext) ([]service.Option, error) {
+		fsConfig := fsopts.Config{
+			EnableIpfs:    cc.Config.IPFS,
+			MetadataStore: cc.Config.MetadataStore,
+			OpenBoltDB:    cc.OpenBoltDB,
+		}
+		fsOpts, err := fsopts.ConfigFsOpts(cc.Ctx, cc.RootDir, &fsConfig)
+		if err != nil {
+			return nil, err
+		}
+		return []service.Option{service.WithFilesystemOptions(fsOpts...)}, nil
+	})
+
+	fusemanager.RegisterConfigFunc(func(cc *fusemanager.ConfigContext) ([]service.Option, error) {
+		keyChainConfig := keychainconfig.Config{
+			EnableKubeKeychain:         cc.Config.Config.KubeconfigKeychainConfig.EnableKeychain,
+			EnableCRIKeychain:          cc.Config.Config.CRIKeychainConfig.EnableKeychain,
+			KubeconfigPath:             cc.Config.Config.KubeconfigPath,
+			DefaultImageServiceAddress: cc.Config.DefaultImageServiceAddress,
+			ImageServicePath:           cc.Config.Config.ImageServicePath,
+		}
+		if cc.Config.Config.CRIKeychainConfig.EnableKeychain && cc.Config.Config.ListenPath == "" || cc.Config.Config.ListenPath == cc.Address {
+			return nil, fmt.Errorf("listen path of CRI server must be specified as a separated socket from FUSE manager server")
+		}
+		// For CRI keychain, if listening path is different from stargz-snapshotter's socket, prepare for the dedicated grpc server and the socket.
+		serveCRISocket := cc.Config.Config.CRIKeychainConfig.EnableKeychain && cc.Config.Config.ListenPath != "" && cc.Config.Config.ListenPath != cc.Address
+		if serveCRISocket {
+			cc.CRIServer = grpc.NewServer()
+		}
+		credsFuncs, err := keychainconfig.ConfigKeychain(cc.Ctx, cc.CRIServer, &keyChainConfig)
+		if err != nil {
+			return nil, err
+		}
+		if serveCRISocket {
+			addr := cc.Config.Config.ListenPath
+			// Prepare the directory for the socket
+			if err := os.MkdirAll(filepath.Dir(addr), 0700); err != nil {
+				return nil, fmt.Errorf("failed to create directory %q: %w", filepath.Dir(addr), err)
+			}
+
+			// Try to remove the socket file to avoid EADDRINUSE
+			if err := os.RemoveAll(addr); err != nil {
+				return nil, fmt.Errorf("failed to remove %q: %w", addr, err)
+			}
+
+			// Listen and serve
+			l, err := net.Listen("unix", addr)
+			if err != nil {
+				return nil, fmt.Errorf("error on listen socket %q: %w", addr, err)
+			}
+			go func() {
+				if err := cc.CRIServer.Serve(l); err != nil {
+					log.G(cc.Ctx).WithError(err).Errorf("error on serving CRI via socket %q", addr)
+				}
+			}()
+		}
+		return []service.Option{service.WithCredsFuncs(credsFuncs...)}, nil
+	})
+}
+
+func main() {
+	fusemanager.Run()
+}

cmd/stargz-store/helper/main.go

@@ -0,0 +1,66 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"io"
+	"os"
+	"time"
+
+	"github.com/containerd/containerd/v2/defaults"
+	"github.com/containerd/containerd/v2/pkg/dialer"
+	"github.com/containerd/stargz-snapshotter/store/pb"
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func main() {
+	var addr = "/var/lib/stargz-store/store.sock" // default
+	if len(os.Args) >= 2 {
+		addr = os.Args[1]
+	}
+	data, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		panic(err)
+	}
+
+	backoffConfig := backoff.DefaultConfig
+	backoffConfig.MaxDelay = 3 * time.Second
+	connParams := grpc.ConnectParams{
+		Backoff: backoffConfig,
+	}
+	gopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithConnectParams(connParams),
+		grpc.WithContextDialer(dialer.ContextDialer),
+		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
+		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
+	}
+	conn, err := grpc.NewClient(dialer.DialAddress(addr), gopts...)
+	if err != nil {
+		panic(err)
+	}
+	c := pb.NewControllerClient(conn)
+	_, err = c.AddCredential(context.Background(), &pb.AddCredentialRequest{
+		Data: data,
+	})
+	if err != nil {
+		panic(err)
+	}
+}

cmd/stargz-store/main.go

@@ -0,0 +1,294 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	golog "log"
+	"math/rand"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/log"
+	dbmetadata "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/db"
+	"github.com/containerd/stargz-snapshotter/fs/config"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
+	"github.com/containerd/stargz-snapshotter/service/keychain/kubeconfig"
+	"github.com/containerd/stargz-snapshotter/service/resolver"
+	"github.com/containerd/stargz-snapshotter/store"
+	"github.com/containerd/stargz-snapshotter/store/pb"
+	sddaemon "github.com/coreos/go-systemd/v22/daemon"
+	"github.com/pelletier/go-toml"
+	bolt "go.etcd.io/bbolt"
+	grpc "google.golang.org/grpc"
+)
+
+const (
+	defaultLogLevel   = log.InfoLevel
+	defaultConfigPath = "/etc/stargz-store/config.toml"
+	defaultRootDir    = "/var/lib/stargz-store"
+)
+
+var (
+	configPath = flag.String("config", defaultConfigPath, "path to the configuration file")
+	logLevel   = flag.String("log-level", defaultLogLevel.String(), "set the logging level [trace, debug, info, warn, error, fatal, panic]")
+	rootDir    = flag.String("root", defaultRootDir, "path to the root directory for this snapshotter")
+	listenaddr = flag.String("addr", filepath.Join(defaultRootDir, "store.sock"), "path to the socket listened by this snapshotter")
+)
+
+type Config struct {
+	config.Config
+
+	// KubeconfigKeychainConfig is config for kubeconfig-based keychain.
+	KubeconfigKeychainConfig `toml:"kubeconfig_keychain"`
+
+	// ResolverConfig is config for resolving registries.
+	ResolverConfig `toml:"resolver"`
+
+	// MetadataStore is the type of the metadata store to use.
+	MetadataStore string `toml:"metadata_store" default:"memory"`
+}
+
+type KubeconfigKeychainConfig struct {
+	EnableKeychain bool   `toml:"enable_keychain"`
+	KubeconfigPath string `toml:"kubeconfig_path"`
+}
+
+type ResolverConfig resolver.Config
+
+func main() {
+	rand.Seed(time.Now().UnixNano()) //nolint:staticcheck // Global math/rand seed is deprecated, but still used by external dependencies
+	flag.Parse()
+	mountPoint := flag.Arg(0)
+	err := log.SetLevel(*logLevel)
+	if err != nil {
+		log.L.WithError(err).Fatal("failed to prepare logger")
+	}
+	log.SetFormat(log.JSONFormat)
+	var (
+		ctx    = log.WithLogger(context.Background(), log.L)
+		config Config
+	)
+	// Streams log of standard lib (go-fuse uses this) into debug log
+	// Snapshotter should use "github.com/containerd/log" otherwise
+	// logs are always printed as "debug" mode.
+	golog.SetOutput(log.G(ctx).WriterLevel(log.DebugLevel))
+
+	if mountPoint == "" {
+		log.G(ctx).Fatalf("mount point must be specified")
+	}
+
+	// Get configuration from specified file
+	if *configPath != "" {
+		tree, err := toml.LoadFile(*configPath)
+		if err != nil && (!os.IsNotExist(err) || *configPath != defaultConfigPath) {
+			log.G(ctx).WithError(err).Fatalf("failed to load config file %q", *configPath)
+		}
+		if err := tree.Unmarshal(&config); err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to unmarshal config file %q", *configPath)
+		}
+	}
+
+	sk := new(storeKeychain)
+
+	errCh := serveController(*listenaddr, sk)
+
+	// Prepare kubeconfig-based keychain if required
+	credsFuncs := []resolver.Credential{sk.credentials}
+	if config.EnableKeychain {
+		var opts []kubeconfig.Option
+		if kcp := config.KubeconfigPath; kcp != "" {
+			opts = append(opts, kubeconfig.WithKubeconfigPath(kcp))
+		}
+		credsFuncs = append(credsFuncs, kubeconfig.NewKubeconfigKeychain(ctx, opts...))
+	}
+
+	// Use RegistryHosts based on ResolverConfig and keychain
+	hosts := resolver.RegistryHostsFromConfig(resolver.Config(config.ResolverConfig), credsFuncs...)
+
+	// Configure and mount filesystem
+	if _, err := os.Stat(mountPoint); err != nil {
+		if err2 := os.MkdirAll(mountPoint, 0755); err2 != nil && !os.IsExist(err2) {
+			log.G(ctx).WithError(err).WithError(err2).
+				Fatalf("failed to prepare mountpoint %q", mountPoint)
+		}
+	}
+	if config.DisableVerification {
+		log.G(ctx).Fatalf("content verification can't be disabled")
+	}
+	mt, err := getMetadataStore(*rootDir, config)
+	if err != nil {
+		log.G(ctx).WithError(err).Fatalf("failed to configure metadata store")
+	}
+	layerManager, err := store.NewLayerManager(ctx, *rootDir, hosts, mt, config.Config)
+	if err != nil {
+		log.G(ctx).WithError(err).Fatalf("failed to prepare pool")
+	}
+	if err := store.Mount(ctx, mountPoint, layerManager, config.Debug); err != nil {
+		log.G(ctx).WithError(err).Fatalf("failed to mount fs at %q", mountPoint)
+	}
+	defer func() {
+		syscall.Unmount(mountPoint, 0)
+		log.G(ctx).Info("Exiting")
+	}()
+
+	if os.Getenv("NOTIFY_SOCKET") != "" {
+		notified, notifyErr := sddaemon.SdNotify(false, sddaemon.SdNotifyReady)
+		log.G(ctx).Debugf("SdNotifyReady notified=%v, err=%v", notified, notifyErr)
+	}
+	defer func() {
+		if os.Getenv("NOTIFY_SOCKET") != "" {
+			notified, notifyErr := sddaemon.SdNotify(false, sddaemon.SdNotifyStopping)
+			log.G(ctx).Debugf("SdNotifyStopping notified=%v, err=%v", notified, notifyErr)
+		}
+	}()
+
+	if err := waitForSignal(ctx, errCh); err != nil {
+		log.G(ctx).Errorf("error: %v", err)
+		os.Exit(1)
+	}
+}
+
+func waitForSignal(ctx context.Context, errCh <-chan error) error {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	select {
+	case s := <-c:
+		log.G(ctx).Infof("Got %v", s)
+	case err := <-errCh:
+		return err
+	}
+	return nil
+}
+
+const (
+	memoryMetadataType = "memory"
+	dbMetadataType     = "db"
+)
+
+func getMetadataStore(rootDir string, config Config) (metadata.Store, error) {
+	switch config.MetadataStore {
+	case "", memoryMetadataType:
+		return memorymetadata.NewReader, nil
+	case dbMetadataType:
+		bOpts := bolt.Options{
+			NoFreelistSync:  true,
+			InitialMmapSize: 64 * 1024 * 1024,
+			FreelistType:    bolt.FreelistMapType,
+		}
+		db, err := bolt.Open(filepath.Join(rootDir, "metadata.db"), 0600, &bOpts)
+		if err != nil {
+			return nil, err
+		}
+		return func(sr *io.SectionReader, opts ...metadata.Option) (metadata.Reader, error) {
+			return dbmetadata.NewReader(db, sr, opts...)
+		}, nil
+	default:
+		return nil, fmt.Errorf("unknown metadata store type: %v; must be %v or %v",
+			config.MetadataStore, memoryMetadataType, dbMetadataType)
+	}
+}
+
+func newController(addCredentialFunc func(data []byte) error) *controller {
+	return &controller{
+		addCredentialFunc: addCredentialFunc,
+	}
+}
+
+type controller struct {
+	addCredentialFunc func(data []byte) error
+}
+
+func (c *controller) AddCredential(ctx context.Context, req *pb.AddCredentialRequest) (resp *pb.AddCredentialResponse, _ error) {
+	return &pb.AddCredentialResponse{}, c.addCredentialFunc(req.Data)
+}
+
+type authConfig struct {
+	Username      string `json:"username,omitempty"`
+	Password      string `json:"password,omitempty"`
+	IdentityToken string `json:"identityToken,omitempty"`
+}
+
+type storeKeychain struct {
+	config   map[string]authConfig
+	configMu sync.Mutex
+}
+
+func (sk *storeKeychain) add(data []byte) error {
+	conf := make(map[string]authConfig)
+	if err := json.NewDecoder(bytes.NewReader(data)).Decode(&conf); err != nil && !errors.Is(err, io.EOF) {
+		return err
+	}
+	sk.configMu.Lock()
+	if sk.config == nil {
+		sk.config = make(map[string]authConfig)
+	}
+	for k, c := range conf {
+		sk.config[k] = c
+	}
+	sk.configMu.Unlock()
+	return nil
+}
+
+func (sk *storeKeychain) credentials(host string, refspec reference.Spec) (string, string, error) {
+	if host != refspec.Hostname() {
+		return "", "", nil // Do not use creds for mirrors
+	}
+	sk.configMu.Lock()
+	defer sk.configMu.Unlock()
+	if acfg, ok := sk.config[refspec.String()]; ok {
+		if acfg.IdentityToken != "" {
+			return "", acfg.IdentityToken, nil
+		} else if acfg.Username != "" || acfg.Password != "" {
+			return acfg.Username, acfg.Password, nil
+		}
+	}
+	return "", "", nil
+}
+
+func serveController(addr string, sk *storeKeychain) <-chan error {
+	// Try to remove the socket file to avoid EADDRINUSE
+	os.Remove(addr)
+	rpc := grpc.NewServer()
+	c := newController(sk.add)
+	pb.RegisterControllerServer(rpc, c)
+	errCh := make(chan error, 1)
+	go func() {
+		l, err := net.Listen("unix", addr)
+		if err != nil {
+			errCh <- fmt.Errorf("error on listen socket %q: %w", addr, err)
+			return
+		}
+		if err := rpc.Serve(l); err != nil {
+			errCh <- fmt.Errorf("error on serving via socket %q: %w", addr, err)
+		}
+	}()
+	return errCh
+}

converter/converter.go

@@ -1,187 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-/*
-   Copyright 2019 The Go Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style
-   license that can be found in the NOTICE.md file.
-*/
-
-package converter
-
-import (
-	gocontext "context"
-	"fmt"
-	"io"
-	"sync"
-
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/platforms"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/layer"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/recorder"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/sampler"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/util"
-	"github.com/containerd/stargz-snapshotter/estargz"
-	"github.com/containerd/stargz-snapshotter/util/tempfiles"
-	regpkg "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/empty"
-	"github.com/google/go-containerregistry/pkg/v1/mutate"
-	"github.com/google/go-containerregistry/pkg/v1/partial"
-	ocidigest "github.com/opencontainers/go-digest"
-	spec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"golang.org/x/sync/errgroup"
-)
-
-func ConvertIndex(ctx gocontext.Context, noOptimize bool, opts *optimizer.Opts, srcIndex regpkg.ImageIndex, platform *spec.Platform, tf *tempfiles.TempFiles, rec *recorder.Recorder, runopts ...sampler.Option) (regpkg.ImageIndex, error) {
-	var addendums []mutate.IndexAddendum
-	manifest, err := srcIndex.IndexManifest()
-	if err != nil {
-		return nil, err
-	}
-	for _, m := range manifest.Manifests {
-		p := platforms.DefaultSpec()
-		if m.Platform != nil {
-			p = *(specPlatform(m.Platform))
-		}
-		if platform != nil {
-			if !platforms.NewMatcher(*platform).Match(p) {
-				continue
-			}
-		}
-		srcImg, err := srcIndex.Image(m.Digest)
-		if err != nil {
-			return nil, err
-		}
-		cctx := log.WithLogger(ctx, log.G(ctx).WithField("platform", platforms.Format(p)))
-		dstImg, err := ConvertImage(cctx, noOptimize, opts, srcImg, &p, tf, rec, runopts...)
-		if err != nil {
-			return nil, err
-		}
-		desc, err := partial.Descriptor(dstImg)
-		if err != nil {
-			return nil, err
-		}
-		desc.Platform = m.Platform // inherit the platform information
-		addendums = append(addendums, mutate.IndexAddendum{
-			Add:        dstImg,
-			Descriptor: *desc,
-		})
-	}
-	if len(addendums) == 0 {
-		return nil, fmt.Errorf("no target image is specified")
-	}
-
-	// Push the converted image
-	return mutate.AppendManifests(empty.Index, addendums...), nil
-}
-
-func ConvertImage(ctx gocontext.Context, noOptimize bool, opts *optimizer.Opts, srcImg regpkg.Image, platform *spec.Platform, tf *tempfiles.TempFiles, rec *recorder.Recorder, runopts ...sampler.Option) (dstImg regpkg.Image, _ error) {
-	// The order of the list is base layer first, top layer last.
-	layers, err := srcImg.Layers()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get image layers")
-	}
-	addendums := make([]mutate.Addendum, len(layers))
-	if noOptimize || !platforms.NewMatcher(platforms.DefaultSpec()).Match(*platform) {
-		// Do not run the optimization container if the option requires it or
-		// the source image doesn't match to the platform where this command runs on.
-		log.G(ctx).Warn("Platform mismatch or optimization disabled; converting without optimization")
-		// TODO: enable to reuse layers
-		var eg errgroup.Group
-		var addendumsMu sync.Mutex
-		for i, l := range layers {
-			i, l := i, l
-			eg.Go(func() error {
-				newL, jtocDigest, err := buildEStargzLayer(l, tf)
-				if err != nil {
-					return err
-				}
-				addendumsMu.Lock()
-				addendums[i] = mutate.Addendum{
-					Layer: newL,
-					Annotations: map[string]string{
-						estargz.TOCJSONDigestAnnotation: jtocDigest.String(),
-					},
-				}
-				addendumsMu.Unlock()
-				return nil
-			})
-		}
-		if err := eg.Wait(); err != nil {
-			return nil, errors.Wrapf(err, "failed to convert layer to stargz")
-		}
-	} else {
-		addendums, err = optimizer.Optimize(ctx, opts, srcImg, tf, rec, runopts...)
-		if err != nil {
-			return nil, err
-		}
-	}
-	srcCfg, err := srcImg.ConfigFile()
-	if err != nil {
-		return nil, err
-	}
-	srcCfg.RootFS.DiffIDs = []regpkg.Hash{}
-	srcCfg.History = []regpkg.History{}
-	img, err := mutate.ConfigFile(empty.Image, srcCfg)
-	if err != nil {
-		return nil, err
-	}
-
-	return mutate.Append(img, addendums...)
-}
-
-func buildEStargzLayer(uncompressed regpkg.Layer, tf *tempfiles.TempFiles) (regpkg.Layer, ocidigest.Digest, error) {
-	tftmp := tempfiles.NewTempFiles() // Shorter lifetime than tempfiles passed by argument
-	defer tftmp.CleanupAll()
-	r, err := uncompressed.Uncompressed()
-	if err != nil {
-		return nil, "", err
-	}
-	file, err := tftmp.TempFile("", "tmpdata")
-	if err != nil {
-		return nil, "", err
-	}
-	if _, err := io.Copy(file, r); err != nil {
-		return nil, "", err
-	}
-	sr, err := util.FileSectionReader(file)
-	if err != nil {
-		return nil, "", err
-	}
-	rc, err := estargz.Build(sr) // no optimization
-	if err != nil {
-		return nil, "", err
-	}
-	defer rc.Close()
-	l, err := layer.NewStaticCompressedLayer(rc, tf)
-	if err != nil {
-		return nil, "", err
-	}
-	return l, rc.TOCDigest(), err
-}
-
-// specPlatform converts ggcr's platform struct to OCI's struct
-func specPlatform(p *regpkg.Platform) *spec.Platform {
-	return &spec.Platform{
-		Architecture: p.Architecture,
-		OS:           p.OS,
-		OSVersion:    p.OSVersion,
-		OSFeatures:   p.OSFeatures,
-		Variant:      p.Variant,
-	}
-}

converter/optimizer/imageio/imageio.go

@@ -1,109 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-/*
-   Copyright 2019 The Go Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style
-   license that can be found in the NOTICE.md file.
-*/
-
-package imageio
-
-import (
-	"fmt"
-
-	"github.com/google/go-containerregistry/pkg/authn"
-	"github.com/google/go-containerregistry/pkg/name"
-	regpkg "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/empty"
-	"github.com/google/go-containerregistry/pkg/v1/layout"
-	"github.com/google/go-containerregistry/pkg/v1/mutate"
-	"github.com/google/go-containerregistry/pkg/v1/partial"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
-)
-
-// ImageIO is an interface for helpers of reading/writing images to/from somewhere.
-type ImageIO interface {
-	ReadIndex() (regpkg.ImageIndex, error)
-	WriteIndex(index regpkg.ImageIndex) error
-	ReadImage() (regpkg.Image, error)
-	WriteImage(image regpkg.Image) error
-}
-
-// RemoteImage is a helper for reading/writing images stored in the remote registry.
-type RemoteImage struct {
-	RemoteRef name.Reference
-}
-
-func (ri RemoteImage) ReadIndex() (regpkg.ImageIndex, error) {
-	return remote.Index(ri.RemoteRef, remote.WithAuthFromKeychain(authn.DefaultKeychain))
-}
-
-func (ri RemoteImage) WriteIndex(index regpkg.ImageIndex) error {
-	return remote.WriteIndex(ri.RemoteRef, index, remote.WithAuthFromKeychain(authn.DefaultKeychain))
-}
-
-func (ri RemoteImage) ReadImage() (regpkg.Image, error) {
-	desc, err := remote.Get(ri.RemoteRef, remote.WithAuthFromKeychain(authn.DefaultKeychain))
-	if err != nil {
-		return nil, err
-	}
-	return desc.Image()
-}
-
-func (ri RemoteImage) WriteImage(image regpkg.Image) error {
-	return remote.Write(ri.RemoteRef, image, remote.WithAuthFromKeychain(authn.DefaultKeychain))
-}
-
-// LocalImage is a helper for reading/writing images stored in the OCI Image Layout directory.
-type LocalImage struct {
-	LocalPath string
-}
-
-func (li LocalImage) ReadIndex() (regpkg.ImageIndex, error) {
-	lp, err := layout.FromPath(li.LocalPath)
-	if err != nil {
-		return nil, err
-	}
-	return lp.ImageIndex()
-}
-
-func (li LocalImage) WriteIndex(index regpkg.ImageIndex) error {
-	_, err := layout.Write(li.LocalPath, index)
-	return err
-}
-
-func (li LocalImage) ReadImage() (regpkg.Image, error) {
-	// OCI Image Layout doesn't have representation of thin image
-	return nil, fmt.Errorf("thin image cannot be read from local")
-}
-
-func (li LocalImage) WriteImage(image regpkg.Image) error {
-	// OCI layout requires index so create it.
-	// TODO: Should we add platform information here?
-	desc, err := partial.Descriptor(image)
-	if err != nil {
-		return err
-	}
-	_, err = layout.Write(li.LocalPath, mutate.AppendManifests(
-		empty.Index,
-		mutate.IndexAddendum{
-			Add:        image,
-			Descriptor: *desc,
-		},
-	))
-	return err
-}

converter/optimizer/layer/layer.go

@@ -1,106 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-/*
-   Copyright 2019 The Go Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style
-   license that can be found in the NOTICE.md file.
-*/
-
-package layer
-
-import (
-	"compress/gzip"
-	"crypto/sha256"
-	"encoding/hex"
-	"io"
-	"io/ioutil"
-
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/util"
-	"github.com/containerd/stargz-snapshotter/util/tempfiles"
-	regpkg "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/types"
-	"github.com/pkg/errors"
-)
-
-func NewStaticCompressedLayer(compressed io.Reader, tf *tempfiles.TempFiles) (regpkg.Layer, error) {
-	file, err := tf.TempFile("", "layerdata")
-	if err != nil {
-		return nil, err
-	}
-	var (
-		diff = sha256.New()
-		h    = sha256.New()
-	)
-	zr, err := gzip.NewReader(io.TeeReader(compressed, io.MultiWriter(file, h)))
-	if err != nil {
-		return nil, err
-	}
-	defer zr.Close()
-	if _, err := io.Copy(diff, zr); err != nil {
-		return nil, err
-	}
-	sr, err := util.FileSectionReader(file)
-	if err != nil {
-		return nil, err
-	}
-	return StaticCompressedLayer{
-		R: sr,
-		Diff: regpkg.Hash{
-			Algorithm: "sha256",
-			Hex:       hex.EncodeToString(diff.Sum(nil)),
-		},
-		Hash: regpkg.Hash{
-			Algorithm: "sha256",
-			Hex:       hex.EncodeToString(h.Sum(nil)),
-		},
-		SizeVal: sr.Size(),
-	}, nil
-}
-
-type StaticCompressedLayer struct {
-	R       io.Reader
-	Diff    regpkg.Hash
-	Hash    regpkg.Hash
-	SizeVal int64
-}
-
-func (l StaticCompressedLayer) Digest() (regpkg.Hash, error) {
-	return l.Hash, nil
-}
-
-func (l StaticCompressedLayer) Size() (int64, error) {
-	return l.SizeVal, nil
-}
-
-func (l StaticCompressedLayer) DiffID() (regpkg.Hash, error) {
-	return l.Diff, nil
-}
-
-func (l StaticCompressedLayer) MediaType() (types.MediaType, error) {
-	return types.DockerLayer, nil
-}
-
-func (l StaticCompressedLayer) Compressed() (io.ReadCloser, error) {
-	// TODO: We should pass l.closerFunc to ggcr as Close() of io.ReadCloser
-	//       but ggcr currently doesn't call Close() so we close it manually on EOF.
-	//       See also: https://github.com/google/go-containerregistry/pull/768
-	return ioutil.NopCloser(l.R), nil
-}
-
-func (l StaticCompressedLayer) Uncompressed() (io.ReadCloser, error) {
-	return nil, errors.New("unsupported")
-}

converter/optimizer/layerconverter/layerconverter.go

@@ -1,116 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-/*
-   Copyright 2019 The Go Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style
-   license that can be found in the NOTICE.md file.
-*/
-
-package layerconverter
-
-import (
-	"context"
-	"fmt"
-	"io"
-
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/layer"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/logger"
-	"github.com/containerd/stargz-snapshotter/estargz"
-	"github.com/containerd/stargz-snapshotter/util/tempfiles"
-	regpkg "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/mutate"
-	"github.com/hashicorp/go-multierror"
-	ocidigest "github.com/opencontainers/go-digest"
-)
-
-type LayerConverter = func() (mutate.Addendum, error)
-
-func FromTar(ctx context.Context, sr *io.SectionReader, mon logger.Monitor, tf *tempfiles.TempFiles) LayerConverter {
-	return func() (mutate.Addendum, error) {
-		log.G(ctx).Debugf("converting...")
-		defer log.G(ctx).Infof("converted")
-
-		rc, err := estargz.Build(sr, estargz.WithPrioritizedFiles(mon.DumpLog()))
-		if err != nil {
-			return mutate.Addendum{}, err
-		}
-		defer rc.Close()
-		jtocDigest := rc.TOCDigest().String()
-		log.G(ctx).WithField("TOC JSON digest", jtocDigest).Debugf("calculated digest")
-		l, err := layer.NewStaticCompressedLayer(rc, tf)
-		if err != nil {
-			return mutate.Addendum{}, err
-		}
-		return mutate.Addendum{
-			Layer: l,
-			Annotations: map[string]string{
-				estargz.TOCJSONDigestAnnotation: jtocDigest,
-			},
-		}, nil
-	}
-}
-
-func FromEStargz(ctx context.Context, tocdgst ocidigest.Digest, l regpkg.Layer, sr *io.SectionReader, mon logger.Monitor) (LayerConverter, error) {
-	// If the layer is valid eStargz, use this layer without conversion
-	r, err := estargz.Open(sr)
-	if err != nil {
-		return nil, err
-	}
-	if _, err := r.VerifyTOC(tocdgst); err != nil {
-		return nil, err
-	}
-	dgst, err := l.Digest()
-	if err != nil {
-		return nil, err
-	}
-	diff, err := l.DiffID()
-	if err != nil {
-		return nil, err
-	}
-	return func() (mutate.Addendum, error) {
-		if len(mon.DumpLog()) != 0 {
-			// There have been some accesses to this layer. we don't reuse this.
-			return mutate.Addendum{}, fmt.Errorf("unable to reuse accessed layer")
-		}
-		log.G(ctx).Infof("no access occur; copying without conversion")
-		return mutate.Addendum{
-			Layer: layer.StaticCompressedLayer{
-				R:       sr,
-				Diff:    diff,
-				Hash:    dgst,
-				SizeVal: sr.Size(),
-			},
-			Annotations: map[string]string{
-				estargz.TOCJSONDigestAnnotation: tocdgst.String(),
-			},
-		}, nil
-	}, nil
-}
-
-func Compose(cs ...LayerConverter) LayerConverter {
-	return func() (add mutate.Addendum, allErr error) {
-		for _, f := range cs {
-			a, err := f()
-			if err == nil {
-				return a, nil
-			}
-			allErr = multierror.Append(allErr, err)
-		}
-		return
-	}
-}

converter/optimizer/logger/logger.go

@@ -1,522 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package logger
-
-import (
-	"archive/tar"
-	"context"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/stargz-snapshotter/util/positionwatcher"
-	fusefs "github.com/hanwen/go-fuse/v2/fs"
-	"github.com/hanwen/go-fuse/v2/fuse"
-	"github.com/pkg/errors"
-	"golang.org/x/sys/unix"
-)
-
-const (
-	DefaultBlockSize      = 4096
-	whiteoutPrefix        = ".wh."
-	whiteoutOpaqueDir     = whiteoutPrefix + whiteoutPrefix + ".opq"
-	xattrPAXRecordsPrefix = "SCHILY.xattr."
-)
-
-func Mount(mountPoint string, tarfile io.ReaderAt, monitor Monitor) (func() error, error) {
-	// Mount filesystem.
-	root := newRoot(tarfile, monitor)
-	timeSec := time.Second
-	rawFS := fusefs.NewNodeFS(root, &fusefs.Options{
-		AttrTimeout:     &timeSec,
-		EntryTimeout:    &timeSec,
-		NullPermissions: true,
-	})
-	if err := root.InitNodes(); err != nil {
-		return nil, errors.Wrap(err, "failed to init nodes")
-	}
-	server, err := fuse.NewServer(rawFS, mountPoint, &fuse.MountOptions{
-		AllowOther: true,             // allow users other than root&mounter to access fs
-		Options:    []string{"suid"}, // allow setuid inside container
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to prepare filesystem server")
-	}
-	go server.Serve()
-	if err := server.WaitMount(); err != nil {
-		return nil, errors.Wrap(err, "failed to mount filesystem")
-	}
-
-	return func() error { return server.Unmount() }, nil
-}
-
-// Nodes
-func newRoot(tarfile io.ReaderAt, monitor Monitor) *node {
-	root := &node{
-		tr:      tarfile,
-		monitor: monitor,
-	}
-	now := time.Now()
-	root.attr.SetTimes(&now, &now, &now)
-	root.attr.Mode = fuse.S_IFDIR | 0777
-
-	return root
-}
-
-type node struct {
-	fusefs.Inode
-
-	fullname string
-	attr     fuse.Attr
-	r        io.ReaderAt
-	link     string // Symbolic link.
-	xattr    map[string][]byte
-
-	tr io.ReaderAt
-
-	monitor Monitor
-}
-
-var _ = (fusefs.InodeEmbedder)((*node)(nil))
-
-var _ = (fusefs.NodeLookuper)((*node)(nil))
-
-func (n *node) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) {
-	c := n.GetChild(name)
-	if c == nil {
-		return nil, syscall.ENOENT
-	}
-	// All node used in this filesystem must have Getattr method.
-	var a fuse.AttrOut
-	if errno := c.Operations().(fusefs.NodeGetattrer).Getattr(ctx, nil, &a); errno != 0 {
-		return nil, errno
-	}
-	out.Attr = a.Attr
-	n.monitor.OnLookup(filepath.Join(n.fullname, name))
-
-	return c, 0
-}
-
-var _ = (fusefs.NodeReadlinker)((*node)(nil))
-
-func (n *node) Readlink(ctx context.Context) ([]byte, syscall.Errno) {
-	n.monitor.OnReadlink(n.fullname, n.link)
-	return []byte(n.link), 0
-}
-
-var _ = (fusefs.NodeOpener)((*node)(nil))
-
-func (n *node) Open(ctx context.Context, flags uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) {
-	n.monitor.OnOpen(n.fullname)
-	// It is OK to return (nil, OK) here. In that case,
-	// this node need to implement "Read" function.
-	return nil, 0, 0
-}
-
-var _ = (fusefs.NodeReader)((*node)(nil))
-
-func (n *node) Read(ctx context.Context, f fusefs.FileHandle, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) {
-	if _, err := n.r.ReadAt(dest, off); err != nil && err != io.EOF {
-		log.G(ctx).WithError(err).Warnf("failed to read file %q", n.fullname)
-		return nil, syscall.EIO
-	}
-	n.monitor.OnRead(n.fullname, off, int64(len(dest)))
-	return fuse.ReadResultData(dest), 0
-}
-
-var _ = (fusefs.NodeGetattrer)((*node)(nil))
-
-func (n *node) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno {
-	copyAttr(&out.Attr, n.attr)
-	n.monitor.OnGetAttr(n.fullname)
-	return 0
-}
-
-var _ = (fusefs.NodeGetxattrer)((*node)(nil))
-
-func (n *node) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) {
-	if v, ok := n.xattr[attr]; ok {
-		if len(dest) < len(v) {
-			return uint32(len(v)), syscall.ERANGE
-		}
-		return uint32(copy(dest, v)), 0
-	}
-	return 0, syscall.ENODATA
-}
-
-var _ = (fusefs.NodeListxattrer)((*node)(nil))
-
-func (n *node) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errno) {
-	var attrs []byte
-	for k := range n.xattr {
-		attrs = append(attrs, []byte(k+"\x00")...)
-	}
-	if len(dest) < len(attrs) {
-		return uint32(len(attrs)), syscall.ERANGE
-	}
-	return uint32(copy(dest, attrs)), 0
-}
-
-var _ = (fusefs.NodeStatfser)((*node)(nil))
-
-func (n *node) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno {
-
-	// http://man7.org/linux/man-pages/man2/statfs.2.html
-	out.Blocks = 0 // dummy
-	out.Bfree = 0
-	out.Bavail = 0
-	out.Files = 0 // dummy
-	out.Ffree = 0
-	out.Bsize = 0 // dummy
-	out.NameLen = 1<<32 - 1
-	out.Frsize = 0 // dummy
-	out.Padding = 0
-	out.Spare = [6]uint32{}
-
-	return 0
-}
-
-func (n *node) InitNodes() error {
-	ctx := context.Background()
-
-	pw, err := positionwatcher.NewPositionWatcher(n.tr)
-	if err != nil {
-		return errors.Wrap(err, "Failed to make position watcher")
-	}
-	tr := tar.NewReader(pw)
-
-	// Walk functions for nodes
-	getormake := func(n *fusefs.Inode, base string) (c *fusefs.Inode, err error) {
-		if c = n.GetChild(base); c == nil {
-			// Make temporary dummy node (directory).
-			if ok := n.AddChild(base, n.NewPersistentInode(ctx, &node{}, fusefs.StableAttr{
-				Mode: syscall.S_IFDIR,
-			}), true); !ok {
-				return nil, fmt.Errorf("failed to add dummy child %q", base)
-			}
-			if c = n.GetChild(base); c == nil {
-				return nil, fmt.Errorf("dummy child %q hasn't been registered", base)
-			}
-		}
-		return
-	}
-	getnode := func(n *fusefs.Inode, base string) (c *fusefs.Inode, err error) {
-		if c = n.GetChild(base); c == nil {
-			// the original node isn't found. We do not allow it.
-			return nil, fmt.Errorf("Node %q not found", base)
-		}
-		return
-	}
-
-	var whiteouts []string
-
-	// Walk through all nodes.
-	for {
-		// Fetch and parse next header.
-		h, err := tr.Next()
-		if err != nil {
-			if err != io.EOF {
-				return errors.Wrap(err, "failed to parse tar file")
-			}
-			break
-		}
-		var (
-			fullname     = filepath.Clean(h.Name)
-			dir, base    = filepath.Split(fullname)
-			parentDir    *fusefs.Inode
-			existingNode *fusefs.Inode
-			xattrs       = make(map[string][]byte)
-		)
-		if parentDir, err = n.walkDown(dir, getormake); err != nil {
-			return errors.Wrap(err, "failed to make dummy nodes")
-		}
-		existingNode = parentDir.GetChild(base)
-		if h.PAXRecords != nil {
-			for k, v := range h.PAXRecords {
-				if strings.HasPrefix(k, xattrPAXRecordsPrefix) {
-					xattrs[k[len(xattrPAXRecordsPrefix):]] = []byte(v)
-				}
-			}
-		}
-		switch {
-		case existingNode != nil:
-			if !existingNode.IsDir() {
-				return fmt.Errorf("node %q is placeholder but not a directory", fullname)
-			}
-			// This is "placeholder node" which has been registered in previous loop as
-			// an intermediate directory. Now we update it with real one.
-			ph, ok := existingNode.Operations().(*node)
-			if !ok {
-				return fmt.Errorf("invalid placeholder node type for %q", fullname)
-			}
-			ph.fullname = fullname
-			ph.attr, _ = headerToAttr(h)
-			ph.link = h.Linkname
-			ph.xattr = override(ph.xattr, xattrs) // preserve previously assigned xattr
-			ph.tr = n.tr
-			ph.monitor = n.monitor
-		case strings.HasPrefix(base, whiteoutPrefix):
-			if base == whiteoutOpaqueDir {
-				// This node is opaque directory indicator so we append opaque xattr to
-				// the parent directory.
-				pn, ok := parentDir.Operations().(*node)
-				if !ok {
-					return fmt.Errorf("parent node %q isn't valid node", dir)
-				}
-				if pn.xattr == nil {
-					pn.xattr = make(map[string][]byte)
-				}
-				pn.xattr["trusted.overlay.opaque"] = []byte("y")
-			} else {
-				// This node is a whiteout, so we don't want to show it.
-				// We record it now and then hide the target node later.
-				whiteouts = append(whiteouts, fullname)
-			}
-		case h.Typeflag == tar.TypeLink:
-			if h.Linkname == "" {
-				return fmt.Errorf("Linkname of hardlink %q is not found", fullname)
-			}
-			// This node is a hardlink. Same as major tar tools(GNU tar etc.),
-			// we pretend that the target node of this hard link has already been appeared.
-			target, err := n.walkDown(filepath.Clean(h.Linkname), getnode)
-			if err != nil {
-				return errors.Wrapf(err, "hardlink(%q ==> %q) is not found",
-					fullname, h.Linkname)
-			}
-			n, ok := target.Operations().(*node)
-			if !ok {
-				return fmt.Errorf("original node %q isn't valid node", h.Linkname)
-			}
-			n.attr.Nlink++
-			// We register the target node as name "base". When we query this hardlink node,
-			// we can easily get the target name by seeing target's fullname field.
-			if ok := parentDir.AddChild(base, target, true); !ok {
-				return fmt.Errorf("failed to add child %q", base)
-			}
-		default:
-			// Normal node so simply create it.
-			attr, sAttr := headerToAttr(h)
-			if ok := parentDir.AddChild(base, n.NewPersistentInode(ctx, &node{
-				fullname: fullname,
-				attr:     attr,
-				r:        io.NewSectionReader(n.tr, pw.CurrentPos(), h.Size),
-				link:     h.Linkname,
-				xattr:    xattrs,
-				tr:       n.tr,
-				monitor:  n.monitor,
-			}, sAttr), true); !ok {
-				return fmt.Errorf("failed to add child %q", base)
-			}
-		}
-
-		continue
-	}
-
-	// Add whiteout nodes if necessary. If an entry exists as both of a
-	// whiteout file and a normal entry, we simply prioritize the normal entry.
-	for _, w := range whiteouts {
-		dir, base := filepath.Split(w)
-		if _, err := n.walkDown(filepath.Join(dir, base[len(whiteoutPrefix):]), getnode); err != nil {
-			p, err := n.walkDown(dir, getnode)
-			if err != nil {
-				return errors.Wrapf(err, "parent node of whiteout %q is not found", w)
-			}
-			if ok := p.AddChild(base[len(whiteoutPrefix):], n.NewPersistentInode(ctx, &whiteout{}, fusefs.StableAttr{Mode: syscall.S_IFCHR}), true); !ok {
-				return fmt.Errorf("failed to add child %q", base)
-			}
-		}
-	}
-
-	return nil
-}
-
-type walkFunc func(n *fusefs.Inode, base string) (*fusefs.Inode, error)
-
-func (n *node) walkDown(path string, walkFn walkFunc) (ino *fusefs.Inode, err error) {
-	ino = n.Root()
-	for _, comp := range strings.Split(path, "/") {
-		if len(comp) == 0 {
-			continue
-		}
-		if ino == nil {
-			return nil, fmt.Errorf("corresponding node of %q is not found", comp)
-		}
-		if ino, err = walkFn(ino, comp); err != nil {
-			return nil, err
-		}
-	}
-	return
-}
-
-type whiteout struct {
-	fusefs.Inode
-}
-
-var _ = (fusefs.NodeGetattrer)((*whiteout)(nil))
-
-func (w *whiteout) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno {
-	out.Ino = 0 // TODO
-	out.Size = 0
-	out.Blksize = uint32(DefaultBlockSize)
-	out.Blocks = 0
-	out.Mode = syscall.S_IFCHR
-	out.Owner = fuse.Owner{Uid: 0, Gid: 0}
-	out.Rdev = uint32(unix.Mkdev(0, 0))
-	out.Nlink = 1
-	out.Padding = 0 // TODO
-	return 0
-}
-
-// monitor
-type Monitor interface {
-	OnLookup(name string)
-	OnReadlink(name, linkname string)
-	OnOpen(name string)
-	OnRead(name string, off, size int64)
-	OnGetAttr(name string)
-	DumpLog() []string
-}
-
-func NewOpenReadMonitor() Monitor {
-	return &OpenReadMonitor{}
-}
-
-type OpenReadMonitor struct {
-	log   []string
-	logMu sync.Mutex
-}
-
-func (m *OpenReadMonitor) OnOpen(name string) {
-	m.logMu.Lock()
-	m.log = append(m.log, name)
-	m.logMu.Unlock()
-}
-
-func (m *OpenReadMonitor) OnRead(name string, off, size int64) {
-	m.logMu.Lock()
-	m.log = append(m.log, name)
-	m.logMu.Unlock()
-}
-
-func (m *OpenReadMonitor) DumpLog() []string {
-	m.logMu.Lock()
-	defer m.logMu.Unlock()
-	return m.log
-}
-
-func (m *OpenReadMonitor) OnLookup(name string)             {}
-func (m *OpenReadMonitor) OnReadlink(name, linkname string) {}
-func (m *OpenReadMonitor) OnGetAttr(name string)            {}
-
-// Utilities
-func override(a, b map[string][]byte) map[string][]byte {
-	if b == nil {
-		return a
-	}
-	if a == nil {
-		return b
-	}
-	for k, v := range b {
-		a[k] = v
-	}
-	return a
-}
-
-func headerToAttr(h *tar.Header) (fuse.Attr, fusefs.StableAttr) {
-	out := fuse.Attr{}
-
-	out.Size = uint64(h.Size)
-	out.Blocks = uint64(h.Size/DefaultBlockSize + 1)
-	out.Atime = uint64(h.AccessTime.Unix())
-	out.Mtime = uint64(h.ModTime.Unix())
-	out.Ctime = uint64(h.ChangeTime.Unix())
-	out.Atimensec = uint32(h.AccessTime.UnixNano())
-	out.Mtimensec = uint32(h.ModTime.UnixNano())
-	out.Ctimensec = uint32(h.ChangeTime.UnixNano())
-	out.Mode = fileModeToSystemMode(h.FileInfo().Mode())
-	out.Blksize = uint32(DefaultBlockSize)
-	out.Owner = fuse.Owner{Uid: uint32(h.Uid), Gid: uint32(h.Gid)}
-	out.Rdev = uint32(unix.Mkdev(uint32(h.Devmajor), uint32(h.Devminor)))
-
-	// ad-hoc
-	out.Ino = 0
-	out.Nlink = 1
-	out.Padding = 0
-
-	return out, fusefs.StableAttr{
-		Mode: out.Mode,
-		// We don't care about inode and generation and let it be
-		// managed by go-fuse.
-	}
-}
-
-func copyAttr(dest *fuse.Attr, src fuse.Attr) {
-	dest.Ino = src.Ino
-	dest.Size = src.Size
-	dest.Blocks = src.Blocks
-	dest.Atime = src.Atime
-	dest.Mtime = src.Mtime
-	dest.Ctime = src.Ctime
-	dest.Atimensec = src.Atimensec
-	dest.Mtimensec = src.Mtimensec
-	dest.Ctimensec = src.Ctimensec
-	dest.Mode = src.Mode
-	dest.Nlink = src.Nlink
-	dest.Owner = src.Owner
-	dest.Rdev = src.Rdev
-	dest.Blksize = src.Blksize
-	dest.Padding = src.Padding
-}
-
-func fileModeToSystemMode(m os.FileMode) uint32 {
-
-	// Convert os.FileMode to system's native bitmap.
-	sm := uint32(m & 0777)
-	switch m & os.ModeType {
-	case os.ModeDevice:
-		sm |= syscall.S_IFBLK
-	case os.ModeDevice | os.ModeCharDevice:
-		sm |= syscall.S_IFCHR
-	case os.ModeDir:
-		sm |= syscall.S_IFDIR
-	case os.ModeNamedPipe:
-		sm |= syscall.S_IFIFO
-	case os.ModeSymlink:
-		sm |= syscall.S_IFLNK
-	case os.ModeSocket:
-		sm |= syscall.S_IFSOCK
-	default: // regular file.
-		sm |= syscall.S_IFREG
-	}
-	if m&os.ModeSetgid != 0 {
-		sm |= syscall.S_ISGID
-	}
-	if m&os.ModeSetuid != 0 {
-		sm |= syscall.S_ISUID
-	}
-	if m&os.ModeSticky != 0 {
-		sm |= syscall.S_ISVTX
-	}
-
-	return sm
-}

converter/optimizer/logger/logger_test.go

@@ -1,535 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-/*
-   Copyright 2019 The Go Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style
-   license that can be found in the NOTICE.md file.
-*/
-
-package logger
-
-import (
-	"archive/tar"
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"syscall"
-	"testing"
-	"time"
-
-	"github.com/containerd/containerd/pkg/testutil"
-	fusefs "github.com/hanwen/go-fuse/v2/fs"
-	"github.com/hanwen/go-fuse/v2/fuse"
-	"golang.org/x/sys/unix"
-)
-
-const (
-	opaqueXattr      = "trusted.overlay.opaque"
-	opaqueXattrValue = "y"
-)
-
-func TestExistence(t *testing.T) {
-	tests := []struct {
-		name string
-		in   []tarent
-		want []check
-	}{
-		{
-			name: "1_whiteout_with_sibling",
-			in: []tarent{
-				directory("foo/"),
-				regfile("foo/bar.txt", ""),
-				regfile("foo/.wh.foo.txt", ""),
-			},
-			want: []check{
-				hasValidWhiteout("foo/foo.txt"),
-				fileNotExist("foo/.wh.foo.txt"),
-			},
-		},
-		{
-			name: "1_whiteout_with_duplicated_name",
-			in: []tarent{
-				directory("foo/"),
-				regfile("foo/bar.txt", "test"),
-				regfile("foo/.wh.bar.txt", ""),
-			},
-			want: []check{
-				hasFileContents("foo/bar.txt", "test"),
-				fileNotExist("foo/.wh.bar.txt"),
-			},
-		},
-		{
-			name: "1_opaque",
-			in: []tarent{
-				directory("foo/"),
-				regfile("foo/.wh..wh..opq", ""),
-			},
-			want: []check{
-				hasNodeXattrs("foo/", opaqueXattr, opaqueXattrValue),
-				fileNotExist("foo/.wh..wh..opq"),
-			},
-		},
-		{
-			name: "1_opaque_with_sibling",
-			in: []tarent{
-				directory("foo/"),
-				regfile("foo/.wh..wh..opq", ""),
-				regfile("foo/bar.txt", "test"),
-			},
-			want: []check{
-				hasNodeXattrs("foo/", opaqueXattr, opaqueXattrValue),
-				hasFileContents("foo/bar.txt", "test"),
-				fileNotExist("foo/.wh..wh..opq"),
-			},
-		},
-		{
-			name: "1_opaque_with_xattr",
-			in: []tarent{
-				directory("foo/", xAttr{"foo": "bar"}),
-				regfile("foo/.wh..wh..opq", ""),
-			},
-			want: []check{
-				hasNodeXattrs("foo/", opaqueXattr, opaqueXattrValue),
-				hasNodeXattrs("foo/", "foo", "bar"),
-				fileNotExist("foo/.wh..wh..opq"),
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			inTar, cancelIn := buildTar(t, tt.in)
-			defer cancelIn()
-			inTarData, err := ioutil.ReadAll(inTar)
-			if err != nil {
-				t.Fatalf("failed to read input tar: %q", err)
-			}
-			root := newRoot(bytes.NewReader(inTarData), NewOpenReadMonitor())
-			timeSec := time.Second
-			fusefs.NewNodeFS(root, &fusefs.Options{
-				AttrTimeout:     &timeSec,
-				EntryTimeout:    &timeSec,
-				NullPermissions: true,
-			})
-			if err := root.InitNodes(); err != nil {
-				t.Fatalf("failed to init nodes: %v", err)
-			}
-			for _, want := range tt.want {
-				want(t, root)
-			}
-		})
-	}
-}
-
-type check func(*testing.T, *node)
-
-func fileNotExist(file string) check {
-	return func(t *testing.T, root *node) {
-		_, err := getNode(t, root, file)
-		if err == nil {
-			t.Errorf("Node %q exists", file)
-		}
-	}
-}
-
-func hasFileContents(file string, want string) check {
-	return func(t *testing.T, root *node) {
-		inode, err := getNode(t, root, file)
-		if err != nil {
-			t.Fatalf("failed to get node %q: %v", file, err)
-		}
-		n, ok := inode.Operations().(*node)
-		if !ok {
-			t.Fatalf("entry %q isn't a normal node", file)
-		}
-		if n.r == nil {
-			t.Fatalf("reader not found for file %q", file)
-		}
-		data := make([]byte, n.attr.Size)
-		gotSize, err := n.r.ReadAt(data, 0)
-		if uint64(gotSize) != n.attr.Size || (err != nil && err != io.EOF) {
-			t.Errorf("failed to read %q: %v", file, err)
-		}
-		if string(data) != want {
-			t.Errorf("Contents(%q) = %q, want %q", file, data, want)
-		}
-	}
-}
-
-func hasValidWhiteout(name string) check {
-	return func(t *testing.T, root *node) {
-		inode, err := getNode(t, root, name)
-		if err != nil {
-			t.Fatalf("failed to get node %q: %v", name, err)
-		}
-		n, ok := inode.Operations().(*whiteout)
-		if !ok {
-			t.Fatalf("entry %q isn't a whiteout node", name)
-		}
-		var ao fuse.AttrOut
-		if errno := n.Operations().(fusefs.NodeGetattrer).Getattr(context.Background(), nil, &ao); errno != 0 {
-			t.Fatalf("failed to get attributes of file %q: %v", name, errno)
-		}
-
-		// validate the node
-		a := ao.Attr
-		if a.Mode&syscall.S_IFCHR != syscall.S_IFCHR {
-			t.Errorf("whiteout node %q isn't a char device %q but %q",
-				name, strconv.FormatUint(uint64(syscall.S_IFCHR), 2), strconv.FormatUint(uint64(a.Mode), 2))
-			return
-		}
-		if a.Rdev != uint32(unix.Mkdev(0, 0)) {
-			t.Errorf("whiteout %q has invalid device numbers (%d, %d); want (0, 0)",
-				name, unix.Major(uint64(a.Rdev)), unix.Minor(uint64(a.Rdev)))
-			return
-		}
-	}
-}
-
-func hasNodeXattrs(entry, name, value string) check {
-	return func(t *testing.T, root *node) {
-		inode, err := getNode(t, root, entry)
-		if err != nil {
-			t.Fatalf("failed to get node %q: %v", entry, err)
-		}
-		n, ok := inode.Operations().(*node)
-		if !ok {
-			t.Fatalf("entry %q isn't a normal node", entry)
-		}
-
-		// check xattr exists in the xattrs list.
-		buf := make([]byte, 1000)
-		nb, errno := n.Operations().(fusefs.NodeListxattrer).Listxattr(context.Background(), buf)
-		if errno != 0 {
-			t.Fatalf("failed to get xattrs list of node %q: %v", entry, err)
-		}
-
-		attrs := strings.Split(string(buf[:nb]), "\x00")
-		var found bool
-		for _, x := range attrs {
-			if x == name {
-				found = true
-			}
-		}
-		if !found {
-			t.Errorf("node %q doesn't have an opaque xattr %q", entry, value)
-			return
-		}
-
-		// check the xattr has valid value.
-		v := make([]byte, len(value))
-		nv, errno := n.Operations().(fusefs.NodeGetxattrer).Getxattr(context.Background(), name, v)
-		if errno != 0 {
-			t.Fatalf("failed to get xattr %q of node %q: %v", name, entry, err)
-		}
-		if int(nv) != len(value) {
-			t.Fatalf("invalid xattr size for file %q, value %q got %d; want %d",
-				name, value, nv, len(value))
-		}
-		if string(v) != value {
-			t.Errorf("node %q has an invalid xattr %q; want %q", entry, v, value)
-			return
-		}
-	}
-}
-
-func getNode(t *testing.T, root *node, path string) (n *fusefs.Inode, err error) {
-	dir, base := filepath.Split(filepath.Clean(path))
-
-	// get the target's parent directory.
-	var eo fuse.EntryOut
-	d := root
-	for _, name := range strings.Split(dir, "/") {
-		if len(name) == 0 {
-			continue
-		}
-		di, errno := d.Lookup(context.Background(), name, &eo)
-		if errno != 0 {
-			err = fmt.Errorf("failed to lookup directory %q: %v", name, errno)
-			return
-		}
-		var ok bool
-		if d, ok = di.Operations().(*node); !ok {
-			err = fmt.Errorf("directory %q isn't a normal node", name)
-			return
-		}
-
-	}
-
-	// get the target's node.
-	node, errno := d.Lookup(context.Background(), base, &eo)
-	if errno != 0 {
-		return nil, fmt.Errorf("failed to lookup node %q: %v", path, errno)
-	}
-
-	return node, nil
-}
-
-func TestOpenRead(t *testing.T) {
-	testutil.RequiresRoot(t)
-	tests := []struct {
-		name string
-		in   []tarent
-		do   accessFunc
-		want []string
-	}{
-		{
-			name: "noopt",
-			in: []tarent{
-				regfile("foo.txt", "foo"),
-				directory("bar/"),
-				regfile("bar/baz.txt", "baz"),
-				regfile("bar/bar.txt", "bar"),
-				regfile("bar/baa.txt", "baa"),
-			},
-			do:   doAccess(),
-			want: []string{},
-		},
-		{
-			name: "open_and_read",
-			in: []tarent{
-				regfile("foo.txt", "foo"),
-				directory("bar/"),
-				regfile("bar/baz.txt", "baz"),
-				regfile("bar/bar.txt", "bar"),
-				regfile("bar/baa.txt", "baa"),
-			},
-			do: doAccess(
-				openFile("bar/baa.txt"),
-				readFile("bar/baz.txt", make([]byte, 3)),
-			),
-			want: []string{
-				"bar/baa.txt", // open
-				"bar/baz.txt", // open for read
-				"bar/baz.txt", // read
-			},
-		},
-		{
-			name: "hardlink",
-			in: []tarent{
-				regfile("foo.txt", "foo"),
-				regfile("baz.txt", "baz"),
-				hardlink("bar.txt", "baz.txt"),
-				regfile("baa.txt", "baa"),
-			},
-			do: doAccess(
-				readFile("bar.txt", make([]byte, 3)),
-			),
-			want: []string{
-				"baz.txt", // open for read; must be original file
-				"baz.txt", // read; must be original file
-			},
-		},
-		{
-			name: "symlink",
-			in: []tarent{
-				regfile("foo.txt", "foo"),
-				regfile("baz.txt", "baz"),
-				symlink("bar.txt", "baz.txt"),
-				regfile("baa.txt", "baa"),
-			},
-			do: doAccess(
-				readFile("bar.txt", make([]byte, 3)),
-			),
-			want: []string{
-				"baz.txt", // open for read; must be original file
-				"baz.txt", // read; must be original file
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			// Prepare input tar file
-			inTar, cancelIn := buildTar(t, tt.in)
-			defer cancelIn()
-			inTarData, err := ioutil.ReadAll(inTar)
-			if err != nil {
-				t.Fatalf("failed to read input tar: %q", err)
-			}
-
-			dir, err := ioutil.TempDir("", "loggertest")
-			if err != nil {
-				t.Fatalf("failed to prepare temp directory")
-			}
-			defer os.RemoveAll(dir)
-
-			monitor := NewOpenReadMonitor()
-			cleanup, err := Mount(dir, bytes.NewReader(inTarData), monitor)
-			if err != nil {
-				t.Fatalf("failed to mount at %q: %q", dir, err)
-			}
-			defer cleanup()
-
-			if err := tt.do(dir); err != nil {
-				t.Fatalf("failed to do specified operations: %q", err)
-			}
-
-			if err := cleanup(); err != nil {
-				t.Logf("failed to unmount: %v", err)
-			}
-
-			log := monitor.DumpLog()
-			for i, l := range log {
-				t.Logf("  [%d]: %s", i, l)
-			}
-			if len(log) != len(tt.want) {
-				t.Errorf("num of log: got %d; want %d", len(log), len(tt.want))
-				return
-			}
-			for i, l := range log {
-				if l != tt.want[i] {
-					t.Errorf("log: got %q; want %q", l, tt.want[i])
-					return
-				}
-			}
-		})
-	}
-}
-
-func buildTar(t *testing.T, ents []tarent) (r io.Reader, cancel func()) {
-	pr, pw := io.Pipe()
-	go func() {
-		tw := tar.NewWriter(pw)
-		for _, ent := range ents {
-			if err := tw.WriteHeader(ent.header); err != nil {
-				t.Errorf("writing header to the input tar: %v", err)
-				pw.Close()
-				return
-			}
-			if _, err := tw.Write(ent.contents); err != nil {
-				t.Errorf("writing contents to the input tar: %v", err)
-				pw.Close()
-				return
-			}
-		}
-		if err := tw.Close(); err != nil {
-			t.Errorf("closing write of input tar: %v", err)
-		}
-		pw.Close()
-	}()
-	return pr, func() { go pr.Close(); go pw.Close() }
-}
-
-type tarent struct {
-	header   *tar.Header
-	contents []byte
-}
-
-func regfile(name string, contents string) tarent {
-	if strings.HasSuffix(name, "/") {
-		panic(fmt.Sprintf("file %q has suffix /", name))
-	}
-	return tarent{
-		header: &tar.Header{
-			Typeflag: tar.TypeReg,
-			Name:     name,
-			Mode:     0644,
-			Size:     int64(len(contents)),
-		},
-		contents: []byte(contents),
-	}
-}
-
-type xAttr map[string]string
-
-func directory(name string, opts ...interface{}) tarent {
-	if !strings.HasSuffix(name, "/") {
-		panic(fmt.Sprintf("dir %q hasn't suffix /", name))
-	}
-	var xattrs xAttr
-	for _, opt := range opts {
-		if v, ok := opt.(xAttr); ok {
-			xattrs = v
-		}
-	}
-	return tarent{
-		header: &tar.Header{
-			Typeflag: tar.TypeDir,
-			Name:     name,
-			Mode:     0755,
-			Xattrs:   xattrs,
-		},
-	}
-}
-
-func hardlink(name string, linkname string) tarent {
-	return tarent{
-		header: &tar.Header{
-			Typeflag: tar.TypeLink,
-			Name:     name,
-			Mode:     0644,
-			Linkname: linkname,
-		},
-	}
-}
-
-func symlink(name string, linkname string) tarent {
-	return tarent{
-		header: &tar.Header{
-			Typeflag: tar.TypeSymlink,
-			Name:     name,
-			Mode:     0644,
-			Linkname: linkname,
-		},
-	}
-}
-
-type accessFunc func(basepath string) error
-
-func doAccess(ac ...accessFunc) accessFunc {
-	return func(basepath string) error {
-		for _, a := range ac {
-			if err := a(basepath); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-}
-
-func openFile(filename string) accessFunc {
-	return func(basepath string) error {
-		f, err := os.Open(filepath.Join(basepath, filename))
-		if err != nil {
-			return err
-		}
-		f.Close()
-		return nil
-	}
-}
-
-func readFile(filename string, b []byte) accessFunc {
-	return func(basepath string) error {
-		f, err := os.Open(filepath.Join(basepath, filename))
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-		if _, err := f.Read(b); err != nil {
-			if err != io.EOF {
-				return err
-			}
-		}
-		return nil
-	}
-}

converter/optimizer/optimizer.go

@@ -1,274 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-/*
-   Copyright 2019 The Go Authors. All rights reserved.
-   Use of this source code is governed by a BSD-style
-   license that can be found in the NOTICE.md file.
-*/
-
-package optimizer
-
-import (
-	"compress/gzip"
-	gocontext "context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/layerconverter"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/logger"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/recorder"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/sampler"
-	"github.com/containerd/stargz-snapshotter/converter/optimizer/util"
-	"github.com/containerd/stargz-snapshotter/estargz"
-	"github.com/containerd/stargz-snapshotter/util/tempfiles"
-	regpkg "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/mutate"
-	ocidigest "github.com/opencontainers/go-digest"
-	spec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-	"golang.org/x/sync/errgroup"
-)
-
-type Opts struct {
-	Reuse  bool
-	Period time.Duration
-}
-
-func Optimize(ctx gocontext.Context, opts *Opts, srcImg regpkg.Image, tf *tempfiles.TempFiles, rec *recorder.Recorder, samplerOpts ...sampler.Option) ([]mutate.Addendum, error) {
-	// Get image's basic information
-	manifest, err := srcImg.Manifest()
-	if err != nil {
-		return nil, err
-	}
-	configData, err := srcImg.RawConfigFile()
-	if err != nil {
-		return nil, err
-	}
-	var config spec.Image
-	if err := json.Unmarshal(configData, &config); err != nil {
-		return nil, errors.Wrap(err, "failed to parse image config file")
-	}
-	// The order is base layer first, top layer last.
-	in, err := srcImg.Layers()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get image layers")
-	}
-
-	// Setup temporary workspace
-	tmpRoot, err := ioutil.TempDir("", "optimize-work")
-	if err != nil {
-		return nil, err
-	}
-	defer os.RemoveAll(tmpRoot)
-	log.G(ctx).Debugf("workspace directory: %q", tmpRoot)
-	mktemp := func(name string) (path string, err error) {
-		if path, err = ioutil.TempDir(tmpRoot, "optimize-"+name+"-"); err != nil {
-			return "", err
-		}
-		if err = os.Chmod(path, 0755); err != nil {
-			return "", err
-		}
-		return path, nil
-	}
-
-	// mount layer loggers on temp directories.
-	var (
-		eg           errgroup.Group
-		lowerdirs    []string
-		convertLayer = make([](func() (mutate.Addendum, error)), len(in))
-		monitors     = make([]logger.Monitor, len(in))
-	)
-	for i := range in {
-		i := i
-		dgst, err := in[i].Digest()
-		if err != nil {
-			return nil, err
-		}
-		ctx := log.WithLogger(ctx, log.G(ctx).WithField("digest", dgst))
-		mp, err := mktemp(fmt.Sprintf("lower%d", i))
-		if err != nil {
-			return nil, err
-		}
-		defer syscall.Unmount(mp, syscall.MNT_FORCE)
-		lowerdirs = append([]string{mp}, lowerdirs...) // top layer first, base layer last (for overlayfs).
-		eg.Go(func() error {
-			// TODO: These files should be deduplicated.
-			compressedFile, err := tf.TempFile("", "compresseddata")
-			if err != nil {
-				return err
-			}
-			decompressedFile, err := tf.TempFile("", "decompresseddata")
-			if err != nil {
-				return err
-			}
-
-			// Mount the layer
-			r, err := in[i].Compressed()
-			if err != nil {
-				return err
-			}
-			defer r.Close()
-			zr, err := gzip.NewReader(io.TeeReader(r, compressedFile))
-			if err != nil {
-				return err
-			}
-			defer zr.Close()
-			if _, err := io.Copy(decompressedFile, zr); err != nil {
-				return err
-			}
-			mon := logger.NewOpenReadMonitor()
-			monitors[i] = mon
-			if _, err := logger.Mount(mp, decompressedFile, mon); err != nil {
-				return errors.Wrapf(err, "failed to mount on %q", mp)
-			}
-
-			// Prepare converters according to the layer type
-			var cvts []func() (mutate.Addendum, error)
-			if tocdgst, ok := getTOCDigest(manifest, dgst); ok && opts.Reuse {
-				// If this layer is a valid eStargz, try to reuse this layer.
-				// If no access occur to this layer during the specified workload,
-				// this layer will be reused without conversion.
-				compressedLayer, err := util.FileSectionReader(compressedFile)
-				if err != nil {
-					return err
-				}
-				f, err := layerconverter.FromEStargz(ctx, tocdgst, in[i], compressedLayer, mon)
-				if err == nil {
-					// TODO: remotely mount it instead of downloading the layer.
-					cvts = append(cvts, f)
-				}
-			}
-			decompressedLayer, err := util.FileSectionReader(decompressedFile)
-			if err != nil {
-				return err
-			}
-			convertLayer[i] = layerconverter.Compose(
-				append(cvts, layerconverter.FromTar(ctx, decompressedLayer, mon, tf))...)
-			log.G(ctx).Infof("unpacked")
-			return nil
-		})
-	}
-	if err := eg.Wait(); err != nil {
-		return nil, err
-	}
-
-	// prepare FileSystem Bundle
-	var (
-		bundle   string
-		upperdir string
-		workdir  string
-	)
-	if bundle, err = mktemp("bundle"); err != nil {
-		return nil, err
-	}
-	if upperdir, err = mktemp("upperdir"); err != nil {
-		return nil, err
-	}
-	if workdir, err = mktemp("workdir"); err != nil {
-		return nil, err
-	}
-	var (
-		rootfs = sampler.GetRootfsPathUnder(bundle)
-		option = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s",
-			strings.Join(lowerdirs, ":"), upperdir, workdir)
-	)
-	if err = os.Mkdir(rootfs, 0777); err != nil {
-		return nil, err
-	}
-	if err = syscall.Mount("overlay", rootfs, "overlay", 0, option); err != nil {
-		return nil, errors.Wrapf(err, "mount overlayfs on %q with data %q", rootfs, option)
-	}
-	defer syscall.Unmount(rootfs, syscall.MNT_FORCE)
-
-	// run the workload with timeout
-	runCtx, cancel := gocontext.WithTimeout(ctx, opts.Period)
-	defer cancel()
-	if err = sampler.Run(runCtx, bundle, config, samplerOpts...); err != nil {
-		return nil, errors.Wrap(err, "failed to run the sampler")
-	}
-
-	// get converted layers
-	var (
-		adds   = make([]mutate.Addendum, len(convertLayer))
-		addsMu sync.Mutex
-	)
-	for i, f := range convertLayer {
-		i, f := i, f
-		eg.Go(func() error {
-			addendum, err := f()
-			if err != nil {
-				return errors.Wrap(err, "failed to get converted layer")
-			}
-			addsMu.Lock()
-			adds[i] = addendum
-			addsMu.Unlock()
-
-			return nil
-		})
-	}
-	if err := eg.Wait(); err != nil {
-		return nil, err
-	}
-
-	if rec != nil {
-		manifestDigest, err := srcImg.Digest()
-		if err != nil {
-			return nil, err
-		}
-		manifestDigestStr := manifestDigest.String()
-		for i, mon := range monitors {
-			i := i
-			for _, f := range mon.DumpLog() {
-				e := &recorder.Entry{
-					Path:           f,
-					ManifestDigest: manifestDigestStr,
-					LayerIndex:     &i,
-				}
-				if err := rec.Record(e); err != nil {
-					return nil, err
-				}
-			}
-		}
-	}
-
-	return adds, nil
-}
-
-func getTOCDigest(manifest *regpkg.Manifest, dgst regpkg.Hash) (ocidigest.Digest, bool) {
-	if manifest == nil {
-		return "", false
-	}
-	for _, desc := range manifest.Layers {
-		if desc.Digest.Algorithm == dgst.Algorithm && desc.Digest.Hex == dgst.Hex {
-			dgstStr, ok := desc.Annotations[estargz.TOCJSONDigestAnnotation]
-			if ok {
-				if tocdgst, err := ocidigest.Parse(dgstStr); err == nil {
-					return tocdgst, true
-				}
-			}
-		}
-	}
-	return "", false
-}

converter/optimizer/sampler/options.go

@@ -1,127 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package sampler
-
-type Option func(*options)
-
-type options struct {
-	envs             []string
-	args             []string
-	entrypoint       []string
-	user             string
-	workingDir       string
-	terminal         bool
-	waitOnSignal     bool
-	mounts           []string
-	dnsNameservers   []string
-	dnsSearchDomains []string
-	dnsOptions       []string
-	extraHosts       []string
-	cni              bool
-	cniPluginConfDir string
-	cniPluginDir     string
-}
-
-func WithEnvs(envs []string) Option {
-	return func(opts *options) {
-		opts.envs = envs
-	}
-}
-
-func WithMounts(mounts []string) Option {
-	return func(opts *options) {
-		opts.mounts = mounts
-	}
-}
-
-func WithArgs(args []string) Option {
-	return func(opts *options) {
-		opts.args = args
-	}
-}
-
-func WithEntrypoint(entrypoint []string) Option {
-	return func(opts *options) {
-		opts.entrypoint = entrypoint
-	}
-}
-
-func WithUser(user string) Option {
-	return func(opts *options) {
-		opts.user = user
-	}
-}
-
-func WithWorkingDir(workingDir string) Option {
-	return func(opts *options) {
-		opts.workingDir = workingDir
-	}
-}
-
-func WithTerminal() Option {
-	return func(opts *options) {
-		opts.terminal = true
-	}
-}
-
-func WithWaitOnSignal() Option {
-	return func(opts *options) {
-		opts.waitOnSignal = true
-	}
-}
-
-func WithDNSNameservers(dnsNameservers []string) Option {
-	return func(opts *options) {
-		opts.dnsNameservers = dnsNameservers
-	}
-}
-
-func WithDNSSearchDomains(dnsSearchDomains []string) Option {
-	return func(opts *options) {
-		opts.dnsSearchDomains = dnsSearchDomains
-	}
-}
-
-func WithDNSOptions(dnsOptions []string) Option {
-	return func(opts *options) {
-		opts.dnsOptions = dnsOptions
-	}
-}
-
-func WithExtraHosts(extraHosts []string) Option {
-	return func(opts *options) {
-		opts.extraHosts = extraHosts
-	}
-}
-
-func WithCNI() Option {
-	return func(opts *options) {
-		opts.cni = true
-	}
-}
-
-func WithCNIPluginConfDir(cniPluginConfDir string) Option {
-	return func(opts *options) {
-		opts.cniPluginConfDir = cniPluginConfDir
-	}
-}
-
-func WithCNIPluginDir(cniPluginDir string) Option {
-	return func(opts *options) {
-		opts.cniPluginDir = cniPluginDir
-	}
-}

converter/optimizer/sampler/sampler.go

@@ -1,419 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package sampler
-
-import (
-	"bytes"
-	"context"
-	"encoding/csv"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/pkg/netns"
-	gocni "github.com/containerd/go-cni"
-	runc "github.com/containerd/go-runc"
-	"github.com/docker/docker/oci"
-	"github.com/hashicorp/go-multierror"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/opencontainers/runc/libcontainer/user"
-	specs "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/pkg/errors"
-	"github.com/rs/xid"
-)
-
-func Run(ctx context.Context, bundle string, config v1.Image, opts ...Option) error {
-	opt := options{}
-	for _, o := range opts {
-		o(&opt)
-	}
-
-	spec, done, err := conf2spec(config.Config, GetRootfsPathUnder(bundle), opt)
-	if err != nil {
-		return errors.Wrap(err, "failed to convert config to spec")
-	}
-	defer func() {
-		if err := done(); err != nil {
-			log.G(ctx).WithError(err).Warnf("failed to cleanup")
-			return
-		}
-		log.G(ctx).Infof("cleaned up successfully")
-	}()
-	sf, err := os.Create(filepath.Join(bundle, "config.json"))
-	if err != nil {
-		return errors.Wrap(err, "failed to create config.json in the bundle")
-	}
-	defer sf.Close()
-	if err = json.NewEncoder(sf).Encode(spec); err != nil {
-		return errors.Wrap(err, "failed to parse user")
-	}
-
-	// run the container
-	if err := runContainer(ctx, bundle, opt.waitOnSignal); err != nil {
-		return errors.Wrap(err, "failed to run containers")
-	}
-
-	return nil
-}
-
-func GetRootfsPathUnder(bundle string) string {
-	return filepath.Join(bundle, "rootfs")
-}
-
-func conf2spec(config v1.ImageConfig, rootfs string, opt options) (spec specs.Spec, done func() error, rErr error) {
-	var cleanups []func() error
-	done = func() (allErr error) {
-		for i := len(cleanups) - 1; i >= 0; i-- {
-			if err := cleanups[i](); err != nil {
-				allErr = multierror.Append(allErr, err)
-			}
-		}
-		return
-	}
-	defer func() {
-		if rErr != nil {
-			if err := done(); err != nil {
-				rErr = errors.Wrap(rErr, "failed to cleanup")
-			}
-		}
-	}()
-
-	s := oci.DefaultSpec()
-	s.Root = &specs.Root{
-		Path: rootfs,
-	}
-
-	// Terminal
-	if opt.terminal {
-		s.Process.Terminal = true
-	}
-
-	// User
-	username := config.User
-	if opt.user != "" {
-		username = opt.user
-	}
-	if username != "" {
-		// Username is specified, we need to resolve the uid and gid
-		passwdPath, err := user.GetPasswdPath()
-		if err != nil {
-			rErr = errors.Wrap(err, "failed to get passwd file path")
-			return
-		}
-		groupPath, err := user.GetGroupPath()
-		if err != nil {
-			rErr = errors.Wrap(err, "failed to get group file path")
-			return
-		}
-		execUser, err := user.GetExecUserPath(username, nil,
-			filepath.Join(rootfs, passwdPath), filepath.Join(rootfs, groupPath))
-		if err != nil {
-			rErr = errors.Wrapf(err, "failed to resolve username %q", username)
-			return
-		}
-		s.Process.User.UID = uint32(execUser.Uid)
-		s.Process.User.GID = uint32(execUser.Gid)
-		for _, g := range execUser.Sgids {
-			s.Process.User.AdditionalGids = append(s.Process.User.AdditionalGids, uint32(g))
-		}
-	}
-
-	// Env
-	s.Process.Env = append(config.Env, opt.envs...)
-
-	// WorkingDir
-	s.Process.Cwd = config.WorkingDir
-	if opt.workingDir != "" {
-		s.Process.Cwd = opt.workingDir
-	}
-	if s.Process.Cwd == "" {
-		s.Process.Cwd = "/"
-	}
-
-	// Entrypoint, Cmd
-	entrypoint := config.Entrypoint
-	if len(opt.entrypoint) != 0 {
-		entrypoint = opt.entrypoint
-	}
-	args := config.Cmd
-	if len(opt.args) != 0 {
-		args = opt.args
-	}
-	s.Process.Args = append(entrypoint, args...)
-
-	// Generate /etc/hosts and /etc/resolv.conf
-	resolvDir, err := ioutil.TempDir("", "tmpetc")
-	if err != nil {
-		rErr = errors.Wrap(err, "failed to prepare tmp dir")
-		return
-	}
-	cleanups = append(cleanups, func() error { return os.RemoveAll(resolvDir) })
-	var (
-		etcHostsPath      = filepath.Join(resolvDir, "hosts")
-		etcResolvConfPath = filepath.Join(resolvDir, "resolv.conf")
-		buf               = new(bytes.Buffer)
-	)
-	for _, n := range opt.dnsNameservers {
-		if _, err := fmt.Fprintf(buf, "nameserver %s\n", n); err != nil {
-			rErr = errors.Wrap(err, "failed to prepare nameserver of /etc/resolv.conf")
-			return
-		}
-	}
-	if len(opt.dnsSearchDomains) > 0 {
-		_, err := fmt.Fprintf(buf, "search %s\n", strings.Join(opt.dnsSearchDomains, " "))
-		if err != nil {
-			rErr = errors.Wrap(err, "failed to prepare search contents of /etc/resolv.conf")
-			return
-		}
-	}
-	if len(opt.dnsOptions) > 0 {
-		_, err := fmt.Fprintf(buf, "options %s\n", strings.Join(opt.dnsOptions, " "))
-		if err != nil {
-			rErr = errors.Wrap(err, "failed to prepare options contents of /etc/resolv.conf")
-			return
-		}
-	}
-	if err := ioutil.WriteFile(etcResolvConfPath, buf.Bytes(), 0644); err != nil {
-		rErr = errors.Wrap(err, "failed to write contents to /etc/resolv.conf")
-		return
-	}
-	buf.Reset() // Reusing for /etc/hosts
-	for _, h := range []struct {
-		host string
-		ip   string
-	}{
-		// Configuration compatible to docker's config
-		// https://github.com/moby/libnetwork/blob/535ef365dc1dd82a5135803a58bc6198a3b9aa27/etchosts/etchosts.go#L28-L36
-		{"localhost", "127.0.0.1"},
-		{"localhost ip6-localhost ip6-loopback", "::1"},
-		{"ip6-localnet", "fe00::0"},
-		{"ip6-mcastprefix", "ff00::0"},
-		{"ip6-allnodes", "ff02::1"},
-		{"ip6-allrouters", "ff02::2"},
-	} {
-		if _, err := fmt.Fprintf(buf, "%s\t%s\n", h.ip, h.host); err != nil {
-			rErr = errors.Wrap(err, "failed to write default hosts to /etc/hosts")
-			return
-		}
-	}
-	for _, h := range opt.extraHosts {
-		parts := strings.SplitN(h, ":", 2) // host:IP
-		if len(parts) != 2 {
-			rErr = fmt.Errorf("cannot parse %q as extra host; must be \"host:IP\"", h)
-			return
-		}
-		// TODO: Validate them
-		if _, err := fmt.Fprintf(buf, "%s\t%s\n", parts[1], parts[0]); err != nil {
-			rErr = errors.Wrap(err, "failed to write extra hosts to /etc/hosts")
-			return
-		}
-	}
-	if err := ioutil.WriteFile(etcHostsPath, buf.Bytes(), 0644); err != nil {
-		rErr = errors.Wrap(err, "failed to write contents to /etc/hosts")
-		return
-	}
-	s.Mounts = append(
-		[]specs.Mount{
-			{
-				Destination: "/etc/resolv.conf",
-				Source:      etcResolvConfPath,
-				Options:     []string{"bind"},
-			},
-			{
-				Destination: "/etc/hosts",
-				Source:      etcHostsPath,
-				Options:     []string{"bind"},
-			},
-		}, s.Mounts...)
-
-	// Mounts (syntax is compatible to ctr command)
-	// e.g.) "type=foo,source=/path,destination=/target,options=rbind:rw"
-	for _, m := range opt.mounts {
-		r := csv.NewReader(strings.NewReader(m))
-		fields, err := r.Read()
-		if err != nil {
-			rErr = errors.Wrap(err, "failed to parse mounts config")
-			return
-		}
-		mc := specs.Mount{}
-		for _, field := range fields {
-			v := strings.Split(field, "=")
-			if len(v) != 2 {
-				rErr = fmt.Errorf("invalid (non key=val) mount spec")
-				return
-			}
-			key, val := v[0], v[1]
-			switch key {
-			case "type":
-				mc.Type = val
-			case "source", "src":
-				mc.Source = val
-			case "destination", "dst":
-				mc.Destination = val
-			case "options":
-				mc.Options = strings.Split(val, ":")
-			default:
-				rErr = fmt.Errorf("mount option %q not supported", key)
-				return
-			}
-		}
-		s.Mounts = append(s.Mounts, mc)
-	}
-
-	// CNI-based networking (if enabled).
-	if opt.cni {
-		// Create a new network namespace for configuring it with CNI plugins
-		ns, err := netns.NewNetNS()
-		if err != nil {
-			rErr = errors.Wrapf(err, "failed to prepare netns")
-			return
-		}
-		cleanups = append(cleanups, ns.Remove)
-
-		// Configure the namespace with CNI plugins
-		var cniopts []gocni.CNIOpt
-		if cdir := opt.cniPluginConfDir; cdir != "" {
-			cniopts = append(cniopts, gocni.WithPluginConfDir(cdir))
-		}
-		if pdir := opt.cniPluginDir; pdir != "" {
-			cniopts = append(cniopts, gocni.WithPluginDir([]string{pdir}))
-		}
-		// The first-found configration file will be effective
-		// TODO: Should we make the number of reading files configurable?
-		cniopts = append(cniopts, gocni.WithDefaultConf)
-		network, err := gocni.New(cniopts...)
-		if err != nil {
-			rErr = errors.Wrap(err, "failed to prepare CNI plugins")
-			return
-		}
-		id := xid.New().String()
-		ctx := context.Background()
-		if _, err := network.Setup(ctx, id, ns.GetPath()); err != nil {
-			rErr = errors.Wrap(err, "failed to setup netns with CNI plugins")
-			return
-		}
-		cleanups = append(cleanups, func() error {
-			return network.Remove(ctx, id, ns.GetPath())
-		})
-
-		// Make the container use this network namespace
-		for i, e := range s.Linux.Namespaces {
-			if e.Type == specs.NetworkNamespace {
-				before := s.Linux.Namespaces[:i]
-				after := s.Linux.Namespaces[i+1:]
-				s.Linux.Namespaces = append(append(before, specs.LinuxNamespace{
-					Type: specs.NetworkNamespace,
-					Path: ns.GetPath(),
-				}), after...)
-				break
-			}
-		}
-	}
-
-	return s, done, nil
-}
-
-func runContainer(ctx context.Context, bundle string, ignoreCtxCancel bool) error {
-	runtime := &runc.Runc{
-		Log:          filepath.Join(bundle, "runc-log.json"),
-		LogFormat:    runc.JSON,
-		PdeathSignal: syscall.SIGKILL,
-		// Setpgid:      true,         // TODO: do we need this?
-	}
-
-	// Run the container
-	id := xid.New().String()
-	stdio, err := runc.NewSTDIO()
-	if err != nil {
-		return err
-	}
-	runCtx, cancelRun := context.WithCancel(context.Background())
-	defer cancelRun()
-	done := make(chan struct{})
-	go func() {
-		defer close(done)
-		log.G(ctx).Infof("running container %q", id)
-		runtime.Run(runCtx, id, bundle, &runc.CreateOpts{
-			IO: stdio,
-		})
-		log.G(ctx).Infof("container %q stopped", id)
-	}()
-
-	// Wait until context is canceled or signal is detected
-	sc := make(chan os.Signal, 1)
-	signal.Notify(sc,
-		syscall.SIGHUP,
-		syscall.SIGINT,
-		syscall.SIGTERM,
-		syscall.SIGQUIT)
-	defer signal.Stop(sc)
-	if ignoreCtxCancel {
-		log.G(ctx).Infof("press Ctrl+C to terminate the container %q", id)
-		select {
-		case <-done:
-			return nil
-		case <-sc:
-			log.G(ctx).Info("signal detected")
-		}
-	} else {
-		log.G(ctx).Debugf("waiting for the termination of container %q", id)
-		select {
-		case <-done:
-			return nil
-		case <-ctx.Done():
-			log.G(ctx).Info("context canceled")
-		case <-sc:
-			log.G(ctx).Info("signal detected")
-		}
-	}
-
-	// Kill the container
-	for {
-		select {
-		case <-done:
-			// container terminated
-			return nil
-		default:
-			log.G(ctx).Debugf("trying to kill container %q", id)
-			killCtx, timeout := context.WithTimeout(context.Background(), 7*time.Second)
-			if err := runtime.Kill(killCtx, id, int(syscall.SIGKILL), nil); err != nil {
-				log.G(ctx).WithError(err).Warnf("failed to kill container %q", id)
-				select {
-				case <-killCtx.Done():
-					// runc kill seems to hang. we shouldn't retry this anymore.
-					timeout()
-					return err
-				default:
-				}
-			}
-			timeout()
-			select {
-			case <-done:
-				return nil
-			case <-time.After(50 * time.Millisecond):
-				// retry runc kill
-			}
-		}
-	}
-}

docs/INSTALL.md

@@ -0,0 +1,184 @@
+# Install Stargz Snapshotter and Stargz Store
+
+## What's Stargz Snapshotter and Stargz Store?
+
+*Stargz Snapshotter* is a plugin for containerd, which enables it to perform lazy pulling of eStargz.
+This is an implementation of *remote snapshotter* plugin and provides remotely-mounted eStargz layers to containerd.
+Communication between containerd and Stargz Snapshotter is done with gRPC over unix socket.
+For more details about Stargz Snapshotter and the relationship with containerd, [please refer to the doc](./overview.md).
+
+If you are using CRI-O/Podman, you can't use Stargz Snapshotter for enabling lazy pulling of eStargz.
+Instead, use *Stargz Store* plugin.
+This is an implementation of *additional layer store* plugin of CRI-O/Podman.
+Stargz Store provides remotely-mounted eStargz layers to CRI-O/Podman.
+
+Stargz Store exposes mounted filesystem structured like the following.
+CRI-O/Podman access to this filesystem to acquire eStargz layers.
+
+```
+<mountpoint>/base64(imageref)/<layerdigest>/
+- diff  : exposes the extracted eStargz layer
+- info  : contains JSON-formatted metadata of this layer
+- use   : files to notify the use of this layer (used for GC)
+```
+
+## Install Stargz Snapshotter for containerd with Systemd
+
+To enable lazy pulling of eStargz on containerd, you need to install *Stargz Snapshotter* plugin.
+This section shows the step to install Stargz Snapshotter with systemd.
+We assume that you are using containerd (> v1.4.2) as a CRI runtime.
+
+- Download release tarball from [the release page](https://github.com/containerd/stargz-snapshotter/releases).
+
+- Add the following configuration to containerd's configuration file (typically: /etc/containerd/config.toml). Please see also [an example configuration file](../script/config/etc/containerd/config.toml).
+  ```toml
+  version = 2
+
+  # Enable stargz snapshotter for CRI
+  [plugins."io.containerd.grpc.v1.cri".containerd]
+    snapshotter = "stargz"
+    disable_snapshot_annotations = false
+
+  # Plug stargz snapshotter into containerd
+  [proxy_plugins]
+    [proxy_plugins.stargz]
+      type = "snapshot"
+      address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+    [proxy_plugins.stargz.exports]
+      root = "/var/lib/containerd-stargz-grpc/"
+
+  ```
+
+- Install fuse
+
+  ###### centos
+  ```
+  # centos 7
+  yum install fuse
+  # centos 8
+  dnf install fuse
+
+  modprobe fuse
+  ```
+
+  ###### ubuntu
+
+  ```
+  apt-get install fuse
+  modprobe fuse
+  ```
+
+- Start stargz-snapshotter and restart containerd
+  ```
+  tar -C /usr/local/bin -xvf stargz-snapshotter-${version}-linux-${arch}.tar.gz containerd-stargz-grpc ctr-remote
+  wget -O /etc/systemd/system/stargz-snapshotter.service https://raw.githubusercontent.com/containerd/stargz-snapshotter/main/script/config/etc/systemd/system/stargz-snapshotter.service
+  systemctl enable --now stargz-snapshotter
+  systemctl restart containerd
+  ```
+
+## Install Stargz Store for CRI-O/Podman with Systemd
+
+To enable lazy pulling of eStargz on CRI-O/Podman, you need to install *Stargz Store* plugin.
+This section shows the step to install Stargz Store with systemd.
+We assume that you are using CRI-O newer than https://github.com/cri-o/cri-o/pull/4850 or Podman newer than https://github.com/containers/podman/pull/10214 .
+
+- Download release tarball from [the release page](https://github.com/containerd/stargz-snapshotter/releases).
+
+- Add the following configuration to the storage configuration file of CRI-O/Podman (typically: /etc/containers/storage.conf). Please see also [an example configuration file](../script/config-cri-o/etc/containers/storage.conf).
+  ```toml
+  [storage]
+  driver = "overlay"
+  graphroot = "/var/lib/containers/storage"
+  runroot = "/run/containers/storage"
+
+  [storage.options]
+  additionallayerstores = ["/var/lib/stargz-store/store:ref"]
+  ```
+
+- Install fuse
+
+  ###### centos
+  ```
+  # centos 7
+  yum install fuse
+  # centos 8
+  dnf install fuse
+
+  modprobe fuse
+  ```
+
+  ###### ubuntu
+
+  ```
+  apt-get install fuse
+  modprobe fuse
+  ```
+
+- Start stargz-store (CRI-O also needs to be restarted if you are using)
+  ```
+  tar -C /usr/local/bin -xvf stargz-snapshotter-${version}-linux-${arch}.tar.gz stargz-store
+  wget -O /etc/systemd/system/stargz-store.service https://raw.githubusercontent.com/containerd/stargz-snapshotter/main/script/config-cri-o/etc/systemd/system/stargz-store.service
+  systemctl enable --now stargz-store
+  systemctl restart cri-o # if you are using CRI-O
+  ```
+
+## Install Stargz Snapshotter for Docker(Moby) with Systemd
+
+- Docker(Moby) newer than [`5c1d6c957b97321c8577e10ddbffe6e01981617a`](https://github.com/moby/moby/commit/5c1d6c957b97321c8577e10ddbffe6e01981617a) is needed on your host. The commit is expected to be included in Docker v24.
+
+- Download stargz-snapshotter release tarball from [the release page](https://github.com/containerd/stargz-snapshotter/releases).
+
+- Enable `containerd-snapshotter` feature and `stargz` snapshotter in Docker. Add the following to docker's configuration file (typically: /etc/docker/daemon.json).
+  ```json
+  {
+    "features": {
+      "containerd-snapshotter": true
+    },
+    "storage-driver": "stargz"
+  }
+  ```
+
+- Enable stargz snapshotter in containerd. Add the following configuration to containerd's configuration file (typically: /etc/containerd/config.toml).
+  ```toml
+  version = 2
+
+  # Plug stargz snapshotter into containerd
+  [proxy_plugins]
+    [proxy_plugins.stargz]
+      type = "snapshot"
+      address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+    [proxy_plugins.stargz.exports]
+      root = "/var/lib/containerd-stargz-grpc/"
+  ```
+
+- Install fuse
+
+  ###### centos
+  ```
+  # centos 7
+  yum install fuse
+  # centos 8
+  dnf install fuse
+
+  modprobe fuse
+  ```
+
+  ###### ubuntu
+
+  ```
+  apt-get install fuse
+  modprobe fuse
+  ```
+
+- Start stargz-snapshotter and restart containerd and docker
+  ```
+  tar -C /usr/local/bin -xvf stargz-snapshotter-${version}-linux-${arch}.tar.gz containerd-stargz-grpc ctr-remote
+  wget -O /etc/systemd/system/stargz-snapshotter.service https://raw.githubusercontent.com/containerd/stargz-snapshotter/${version}/script/config/etc/systemd/system/stargz-snapshotter.service
+  systemctl enable --now stargz-snapshotter
+  systemctl restart containerd
+  systemctl restart docker
+  ```
+
+## Using stargz-snapshotter on Lima
+
+See [`./lima.md`](./lima.md)

docs/ctr-remote.md

@@ -3,7 +3,14 @@
 This doc describes example usages of `ctr-remote image optimize` command for converting images into eStargz.
 
 `ctr-remote images optimize` command (call `ctr-remote` in this doc) enables users to convert an image into eStargz.
-The converted image can be lazily pulled by Stargz Snapshotter which can speed up the container startup.
+This command works on containerd so containerd needs to run on your environment.
+So this converts an image stored in containerd and stores the resulting image to containerd.
+
+Because the resulting image is stored to containerd, you can use `ctr-remote image pull` and `ctr-remote image push` commands for pulling/pushing images from/to regstries.
+[nerdctl](https://github.com/containerd/nerdctl), Docker-compatible CLI for containerd, allows you to pull/push images using `~/.docker/config.json`.
+Various other containerd-based commands like `ctr-remote content get`, `ctr-remote images export` and other `ctr-remote` and `nerdctl` commands can also be used for debugging and inspecting the resulting eStargz image.
+
+The converted eStargz image can be lazily pulled by Stargz Snapshotter which can speed up the container startup.
 Because this image is backward compatible to OCI/Docker image, this can be also run by other runtimes that don't support lazy pull (e.g. Docker).
 
 Though lazy pull speeds up the container's startup, it's possible, especially with slow network, that the runtime performance becomes lower because reading files can induce remotely downloading file contents.
@@ -13,15 +20,16 @@ This optimization is done by baking the information about files that are likely
 On runtime, Stargz Snapshotter prefetches these prioritized files before mounting the layer for making sure these files are locally accessible.
 This can avoid downloading chunks on every file read and mitigate the runtime performance drawbacks.
 
-For more details about eStargz and its optimization, refer also to [eStargz: Standard-Compatible Extensions to Tar.gz Layers for Lazy Pulling Container Images](/docs/stargz-estargz.md).
+:information_source: For more details about eStargz and its optimization, refer also to [eStargz: Standard-Compatible Extensions to Tar.gz Layers for Lazy Pulling Container Images](/docs/stargz-estargz.md).
+
+:information_source: Please see also [Creating smaller eStargz images](/docs/smaller-estargz.md) if you're interested in creating a smaller size of eStargz images.
 
 ## Requirements
 
-- fusermount: Installable through `fuse` (or `fuse3`) using package manager. e.g. `apt install fuse` on ubuntu.
-- runc: Release binaries are available on https://github.com/opencontainers/runc/. Or you should already have this if you are using container runtimes (containerd, Docker, etc.) on your host.
+- containerd: Release binaries are available on https://github.com/containerd/containerd/releases.
 - CNI plugins (if network connection is needed during optimization): Release binaries are available on https://github.com/containernetworking/plugins.
 
-`ctr-remote` requires the root privilege, so run this command by root or with `sudo`.
+`ctr-remote` requires CAP_SYS_ADMIN.
 Rootless execution of this command is still WIP.
 
 For trying the examples described in this doc, you can also use the docker-compose-based demo environment.
@@ -41,40 +49,66 @@ $ docker exec -it containerd_demo /bin/bash
 The following command optimizes an (non-eStargz) image `ghcr.io/stargz-containers/golang:1.15.3-buster-org` (this is a copy of `golang:1.15.3-buster`) and pushes the result eStargz image into `registry2:5000/golang:1.15.3-esgz`.
 This doesn't append workload-related configuration options (e.g. `--entrypoint`) so this optimizes the image against the default configurations baked to the image e.g. through Dockefile instructions (`ENTRYPOINT`, etc) when building the original image.
 
-Here, `registry2:5000` in the demo environment serves images with HTTP (not HTTPS) so we need to tell it to the `ctr-remote` command using `--plain-http` option and `http://` prefix.
-
 ```
-ctr-remote image optimize --plain-http \
-           ghcr.io/stargz-containers/golang:1.15.3-buster-org \
-           http://registry2:5000/golang:1.15.3-esgz
+ctr-remote image pull ghcr.io/stargz-containers/golang:1.15.3-buster-org
+ctr-remote image optimize --oci ghcr.io/stargz-containers/golang:1.15.3-buster-org registry2:5000/golang:1.15.3-esgz
+ctr-remote image push --plain-http registry2:5000/golang:1.15.3-esgz
 ```
 
-When you run this command, `ctr-remote` runs the source image (`ghcr.io/stargz-containers/golang:1.15.3-buster-org`) as a container and profiles all file accesses during the execution.
+When you run `ctr-remote image optimize`, this runs the source image (`ghcr.io/stargz-containers/golang:1.15.3-buster-org`) as a container and profiles all file accesses during the execution.
 Then these accessed files are marked as "prioritized" files and will be prefetched on runtime.
 
-You can lazy-pull this image with Stargz Snapshotter.
-The following example lazily pulls this image to containerd, using `ctr-remote image rpull` command (`http://` prefix isn't needed here).
+You can specify the GZIP compression level the converter should use using the `--estargz-compression-level` flag. The values range from 1-9. If the flag isn't provided, the compression level will default to 9.
+
+A value of 9 indicates the archive will be gzipped with max compression. This will reduce the bytes transferred over the network but increase the CPU cycles required to decompress the payload. Whereas gzip compression value 1 indicates archive will be gzipped with least compression. This will increase the bytes transferred over the network but decreases the CPU cycles required to decompress the payload. This value should be chosen based on the workload and host characteristics.
+
+The following example optimizes an image with a compression level of 1.
+
+```console
+# ctr-remote image optimize --oci --estargz-compression-level 1 ghcr.io/stargz-containers/golang:1.15.3-buster-org registry2:5000/golang:1.15.3-esgz
+```
+
+You can enable host networking for the container using the `net-host` flag.
+
+```console
+# ctr-remote i optimize -t -i --oci --entrypoint='[ "/bin/bash", "-c" ]' --net-host --args='[ "ip a && curl example.com" ]' ghcr.io/stargz-containers/centos:8-test registry2:5000/centos:8-test-esgz
+```
+
+You can optimize GPU-based images using the `gpu` flag. The flag expects a comma separated list of integers or 'all'.
+
+```console
+# ctr-remote i optimize --oci --gpus "0" <src> <target>
+# ctr-remote i optimize --oci --gpus "all" <src> <target>
+```
+
+`--oci` option is highly recommended to add when you create eStargz image.
+If the source image is [Docker image](https://github.com/moby/moby/blob/master/image/spec/v1.2.md) that doesn't allow us [content verification of eStargz](/docs/verification.md), `ctr-remote` converts this image into the [OCI starndard compliant image](https://github.com/opencontainers/image-spec/).
+OCI image also can run on most of modern container runtimes.
+
+You can lazy-pull this image into other hosts with Stargz Snapshotter.
+The following example lazily pulls this image to containerd, using `ctr-remote image rpull` command.
 
 ```console
 # ctr-remote image rpull --plain-http registry2:5000/golang:1.15.3-esgz
-egistry2:5000/golang:1.15.3-esgz
-fetching sha256:bd2e5983... application/vnd.oci.image.index.v1+json
-fetching sha256:55aef317... application/vnd.docker.distribution.manifest.v2+json
-fetching sha256:d1123476... application/vnd.docker.container.image.v1+json
+fetching sha256:9f9b5a43... application/vnd.oci.image.index.v1+json
+fetching sha256:16debc17... application/vnd.oci.image.manifest.v1+json
+fetching sha256:a610ec55... application/vnd.oci.image.config.v1+json
 # ctr-remote run --rm -t --snapshotter=stargz registry2:5000/golang:1.15.3-esgz test echo hello
 hello
 ```
 
+In the following examples, we omit `ctr-remote image pull` and `ctr-remote image push` from the example.
+
 ## Optimizing an image with custom configuration
 
 You can also specify the custom workload configuration that the image is optimized against.
 The following example optimizes the image against the workload running `go version` on `/bin/bash`.
 
 ```
-ctr-remote image optimize --plain-http \
+ctr-remote image optimize --oci \
            --entrypoint='[ "/bin/bash", "-c" ]' --args='[ "go version" ]' \
            ghcr.io/stargz-containers/golang:1.15.3-buster-org \
-           http://registry2:5000/golang:1.15.3-esgz-go-version
+           registry2:5000/golang:1.15.3-esgz-go-version
 ```
 
 Other options are also available for configuring the workload.
@@ -87,7 +121,8 @@ Other options are also available for configuring the workload.
 |`--user`|User name to run the process in the container|
 |`--cwd`|Working directory|
 |`--period`|The time seconds during profiling the file accesses|
-|`-t`or`--terminal`|Enable to interactively control the container|
+|`-t`or`--terminal`|Attach terminal to the container. This flag must be specified with `-i`|
+|`-i`|Attach stdin to the container|
 
 ## Mounting files from the host
 
@@ -112,11 +147,11 @@ func main() {
 Then you can build it inside the container by bind-mounting the file `/tmp/hello.go` on the host to this container.
 
 ```
-ctr-remote image optimize --plain-http \
+ctr-remote image optimize --oci \
            --mount=type=bind,source=/tmp/hello.go,destination=/hello.go,options=bind:ro \
            --entrypoint='[ "/bin/bash", "-c" ]' --args='[ "go build -o /hello /hello.go && /hello" ]' \
            ghcr.io/stargz-containers/golang:1.15.3-buster-org \
-           http://registry2:5000/golang:1.15.3-esgz-hello-world
+           registry2:5000/golang:1.15.3-esgz-hello-world
 ```
 
 The syntax of the `--mount` option is compatible to containerd's `ctr` tool and [corresponds to the OCI Runtime Spec](https://github.com/opencontainers/runtime-spec/blob/v1.0.2/config.md#mounts).
@@ -136,12 +171,12 @@ Once you configure CNI plugins on the host, CNI-based networking can be enabled
 
 The following example accesses to https://example.com from the container, using `bridge` CNI plugin installed in the demo environment.
 ```
-ctr-remote image optimize --plain-http \
+ctr-remote image optimize --oci \
            --cni \
            --entrypoint='[ "/bin/bash", "-c" ]' \
            --args='[ "curl example.com" ]' \
            ghcr.io/stargz-containers/golang:1.15.3-buster-org \
-           http://registry2:5000/golang:1.15.3-esgz-curl
+           registry2:5000/golang:1.15.3-esgz-curl
 ```
 
 If CNI plugins and configurations are installed to locations other than well-known paths (/opt/cni/bin and /etc/cni/net.d), you can tell it to `ctr-remote` using the following options.
@@ -179,33 +214,6 @@ If you want to customize the configuration, the following options are useful.
 |`--dns-search-domains`|Comma-separated `search` configs added to the container's `/etc/resolv.conf`|
 |`--dns-options`|Comma-separated `options` configs added to the container's `/etc/resolv.conf`|
 
-## Image input and output
-
-You can pull the source image and push the result image from/to the following locations.
-
-- Registry
-- Local directory
-
-By default, `ctr-remote` recognizes the specified references as pointing to the registry and `~/.docker/config.json` is used for the authentication.
-As shown in the above examples, you can also use HTTP registries by using `--plain-http` option and adding `http://` prefix to the reference.
-
-```
-ctr-remote image optimize --plain-http \
-           ghcr.io/stargz-containers/golang:1.15.3-buster-org \
-           http://registry2:5000/golang:1.15.3-esgz
-```
-
-You can also pull/push images against local directories.
-This is useful especially for debugging including inspecting each layer and metadata blobs directly.
-You can specify local paths using `local://` prefix.
-The following example stores the result image to the local directory `/tmp/output` in the format of [OCI Image Layout](https://github.com/opencontainers/image-spec/blob/master/image-layout.md).
-
-```
-ctr-remote image optimize \
-           ghcr.io/stargz-containers/golang:1.15.3-buster-org \
-           local:///tmp/output/
-```
-
 ## Other useful features
 
 ### Reusing already-converted layers
@@ -217,23 +225,22 @@ For enabling this feature, add `--reuse` option to `ctr-remote`.
 The following example re-converts already converted eStargz image (`ghcr.io/stargz-containers/golang:1.15.3-buster-esgz`) with `--reuse` feature.
 
 ```
-ctr-remote image optimize --plain-http \
+ctr-remote image optimize --oci \
            --reuse \
            ghcr.io/stargz-containers/golang:1.15.3-buster-esgz \
-           http://registry2:5000/golang:1.15.3-esgz
+           registry2:5000/golang:1.15.3-esgz
 ```
 
 You will see `ctr-remote` skips converting some layers with printing `copying without conversion` log messages as the following.
 
 ```
 (... omit ...)
-INFO[0036] no access occur; copying without conversion   digest="sha256:6aedf0c74720e30b9093dc0d2b39c2dd88f35ead14e2087bb49c1608bb151e61"
-INFO[0036] no access occur; copying without conversion   digest="sha256:4416ecf7e2787af750fe3b1988f36a2c47edc2d3162739c6eedd739d6d5a14d1"
-INFO[0036] no access occur; copying without conversion   digest="sha256:976cc0da952505fede3abe08c0ff0c5277416828c4dff8bd01b306c5b4e5c6f5"
-INFO[0036] no access occur; copying without conversion   digest="sha256:79d28aed10b15d548b63eea4cc59e518c4939f9c8fb8498100ec658fc7e0baca"
-INFO[0073] converted                                     digest="sha256:62d6dbb94750b1be9f60535bb7d0b1b02c6e9e37d1c55a761c1fc7883ca7e8ad"
-INFO[0074] converted                                     digest="sha256:59cf7266511a915072804370a3083a1007c4fb757d800ceef848032ac4a5b605"
-INFO[0078] converted                                     digest="sha256:ca2a1da2dee341a3b87a14d56603e9c29c66721056a47bec156f9b04ee0b1e5e"
+WARN[0036] reusing "sha256:4416ecf7e2787af750fe3b1988f36a2c47edc2d3162739c6eedd739d6d5a14d1" without conversion
+WARN[0036] reusing "sha256:976cc0da952505fede3abe08c0ff0c5277416828c4dff8bd01b306c5b4e5c6f5" without conversion
+WARN[0036] reusing "sha256:59cf7266511a915072804370a3083a1007c4fb757d800ceef848032ac4a5b605" without conversion
+WARN[0036] reusing "sha256:ca2a1da2dee341a3b87a14d56603e9c29c66721056a47bec156f9b04ee0b1e5e" without conversion
+WARN[0036] reusing "sha256:79d28aed10b15d548b63eea4cc59e518c4939f9c8fb8498100ec658fc7e0baca" without conversion
+WARN[0036] reusing "sha256:6aedf0c74720e30b9093dc0d2b39c2dd88f35ead14e2087bb49c1608bb151e61" without conversion
 (... omit ...)
 ```
 
@@ -253,12 +260,47 @@ The format of the `--platform` option is `<os>|<arch>|<os>/<arch>[/<variant>]`,
 The following example converts all images contained in `ghcr.io/stargz-containers/golang:1.15.3-buster-org`.
 
 ```
-ctr-remote image optimize --plain-http \
+ctr-remote image optimize --oci \
            --all-platforms \
            ghcr.io/stargz-containers/golang:1.15.3-buster-org \
-           http://registry2:5000/golang:1.15.3-esgz-fat
+           registry2:5000/golang:1.15.3-esgz-fat
 ```
 
 By default, when the source image is a multi-platform image, `ctr-remote` converts the image corresponding to the platform where `ctr-remote` runs.
 
 Note that though the images specified by `--all-platform` and `--platform` are converted to eStargz, images that don't correspond to the current platform aren't *optimized*. That is, these images are lazily pulled but without prefetch.
+
+### Dump log of accessed files during optimization (`--record-out`)
+
+You can dump the information of which files are accesssed during optimization, using `--record-out` flag.
+
+For example, the following dumps logs of files accessed during running `ls` in `ubuntu:24.04`.
+
+```
+ctr-remote image pull docker.io/library/ubuntu:24.04
+ctr-remote image optimize --record-out=/tmp/log.json \
+  --entrypoint='[ "/bin/bash", "-c" ]' --args='[ "ls" ]' \
+  docker.io/library/ubuntu:24.04 registry2:5000/ubuntu:24.04
+```
+
+The following is the contents of the log (`/tmp/log.json`):
+
+```
+{"path":"usr/bin/bash","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/bin/bash","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/ld.so.cache","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libtinfo.so.6.4","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libc.so.6","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/nsswitch.conf","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/nsswitch.conf","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/passwd","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/bin/ls","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"etc/ld.so.cache","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libselinux.so.1","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libc.so.6","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+{"path":"usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.11.2","manifestDigest":"sha256:5d070ad5f7fe63623cbb99b4fc0fd997f5591303d4b03ccce50f403957d0ddc4","layerIndex":0}
+```
+
+For creating an optimized eStargz using this log, you can input this log into [`--estargz-record-in` or `--zstdchunked-record-in` of `nerdctl image convert`](https://github.com/containerd/nerdctl/blob/8b814ca7fe29cb505a02a3d85ba22860e63d15bf/docs/command-reference.md#nerd_face-nerdctl-image-convert) or the same flags for `ctr-remote image convert` .

docs/estargz.md

@@ -0,0 +1,422 @@
+# eStargz: Standard-Compatible Extension to Container Image Layers for Lazy Pulling
+
+This doc describes the extension to gzip layers of container images (`application/vnd.oci.image.layer.v1.tar+gzip` of [OCI Image Specification](https://github.com/opencontainers/image-spec/) and `application/vnd.docker.image.rootfs.diff.tar.gzip` of [Docker Image Specification](https://github.com/moby/moby/blob/master/image/spec/v1.2.md)) for *lazy pulling*.
+The extension is called *eStargz*.
+
+eStargz is a *backward-compatible extension* which means that images can be pushed to the extension-agnostic registry and can run on extension-agnostic runtimes.
+
+This extension is based on stargz (stands for *seekable tar.gz*) proposed by [Google CRFS](https://github.com/google/crfs) project (initially [discussed in Go community](https://github.com/golang/go/issues/30829)).
+eStargz extends stargz for chunk-level verification and runtime performance optimization.
+
+Notational convention follows [OCI Image Specification](https://github.com/opencontainers/image-spec/blob/v1.0.1/spec.md#notational-conventions).
+
+## Overview
+
+Lazy pulling is a technique of pulling container images aiming at the faster cold start.
+This allows a container to startup without waiting for the entire image layer contents to be locally available.
+Instead, necessary files (or chunks for large files) in the layer are fetched *on-demand* during running the container.
+
+For achieving this, runtimes need to fetch and extract each file in a layer independently.
+However, layer without eStargz extension doesn't allow this because of the following reasons,
+
+1. The entire layer blob needs to be extracted even for getting a single file entry.
+2. Digests aren't provided for each file so it cannot be verified independently.
+
+eStargz solves these issues and enables lazy pulling.
+
+Additionally, it supports prefetching of files.
+This can be used to mitigate runtime performance drawbacks caused by the on-demand fetching of each file.
+
+This extension is a backward-compatible so the eStargz-formatted image can be pushed to the registry and can run even on eStargz-agnostic runtimes.
+
+## The structure
+
+![The structure of eStargz](/docs/images/estargz-structure.png)
+
+eStargz is a gzip-compressed tar archive of files and a metadata component called *TOC* (described in the later section).
+In an eStargz-formatted blob, each non-empty regular file and each metadata component MUST be separately compressed as gzip.
+This structure is inherited from [stargz](https://github.com/google/crfs).
+
+Therefore, the gzip headers MUST locate at the following locations.
+
+- The top of the blob
+- The top of the payload of each non-empty regular file tar entry except *TOC*
+- The top of *TOC* tar header
+- The top of *footer* (described in the later section)
+
+Large regular files in an eStargz blob MAY be chunked into several smaller gzip members.
+Each chunked member is called *chunk* in this doc.
+
+Therefore, gzip headers MAY locate at the following locations.
+
+- Arbitrary location within the payload of non-empty regular file entry
+
+An eStargz-formatted blob is the concatenation of these gzip members, which is a still valid gzip blob.
+
+## TOC, TOCEntries and Footer
+
+### TOC and TOCEntries
+
+eStargz contains a regular file called *TOC* which records metadata (e.g. name, file type, owners, offset etc) of all file entries in eStargz, except TOC itself.
+Container runtimes MAY use TOC to mount the container's filesystem without downloading the entire layer contents.
+
+TOC MUST be a JSON file contained as the last tar entry and MUST be named `stargz.index.json`.
+
+The following fields contain the primary properties that constitute a TOC.
+
+- **`version`** *int*
+
+   This REQUIRED property contains the version of the TOC. This value MUST be `1`.
+
+- **`entries`** *array of objects*
+
+   This property MUST contain an array of *TOCEntry* of all tar entries and chunks in the blob, except `stargz.index.json`.
+
+*TOCEntry* consists of metadata of a file or chunk in eStargz.
+If metadata in a TOCEntry of a file differs from the corresponding tar entry, TOCEntry SHOULD be respected.
+
+The following fields contain the primary properties that constitute a TOCEntry.
+Properties other than `chunkDigest` are inherited from [stargz](https://github.com/google/crfs).
+
+- **`name`** *string*
+
+  This REQUIRED property contains the name of the tar entry.
+  This MUST be the complete path stored in the tar file.
+
+- **`type`** *string*
+
+  This REQUIRED property contains the type of tar entry.
+  This MUST be either of the following.
+  - `dir`: directory
+  - `reg`: regular file
+  - `symlink`: symbolic link
+  - `hardlink`: hard link
+  - `char`: character device
+  - `block`: block device
+  - `fifo`: fifo
+  - `chunk`: a chunk of regular file data
+  As described in the above section, a regular file can be divided into several chunks.
+  TOCEntry MUST be created for each chunk.
+  TOCEntry of the first chunk of that file MUST be typed as `reg`.
+  TOCEntry of each chunk after 2nd MUST be typed as `chunk`.
+  `chunk` TOCEntry MUST set *offset*, *chunkOffset* and *chunkSize* properties.
+
+- **`size`** *uint64*
+
+  This OPTIONAL property contains the uncompressed size of the regular file.
+  Non-empty `reg` file MUST set this property.
+
+- **`modtime`** *string*
+
+  This OPTIONAL property contains the modification time of the tar entry.
+  Empty means zero or unknown.
+  Otherwise, the value is in UTC RFC3339 format.
+
+- **`linkName`** *string*
+
+  This OPTIONAL property contains the link target.
+  `symlink` and `hardlink` MUST set this property.
+
+- **`mode`** *int64*
+
+  This REQUIRED property contains the permission and mode bits.
+
+- **`uid`** *uint*
+
+  This REQUIRED property contains the user ID of the owner of this file.
+
+- **`gid`** *uint*
+
+  This REQUIRED property contains the group ID of the owner of this file.
+
+- **`userName`** *string*
+
+  This OPTIONAL property contains the username of the owner.
+
+- **`groupName`** *string*
+
+  This OPTIONAL property contains the groupname of the owner.
+
+- **`devMajor`** *int*
+
+  This OPTIONAL property contains the major device number of device files.
+  `char` and `block` files MUST set this property.
+
+- **`devMinor`** *int*
+
+  This OPTIONAL property contains the minor device number of device files.
+  `char` and `block` files MUST set this property.
+
+- **`xattrs`** *string-bytes map*
+
+  This OPTIONAL property contains the extended attribute for the tar entry.
+
+- **`digest`** *string*
+
+  This OPTIONAL property contains the digest of the regular file contents.
+
+- **`offset`** *int64*
+
+  This OPTIONAL property contains the offset of the gzip header of the regular file or chunk in the blob.
+  TOCEntries of non-empty `reg` and `chunk` MUST set this property.
+
+- **`chunkOffset`** *int64*
+
+  This OPTIONAL property contains the offset of this chunk in the decompressed regular file payload.
+  TOCEntries of `chunk` type MUST set this property.
+
+- **`chunkSize`** *int64*
+
+  This OPTIONAL property contains the decompressed size of this chunk.
+  The last `chunk` in a `reg` file or `reg` file that isn't chunked MUST set this property to zero.
+  Other `reg` and `chunk` MUST set this property.
+
+- **`chunkDigest`** *string*
+
+  This OPTIONAL property contains a digest of this chunk.
+  TOCEntries of non-empty `reg` and `chunk` MUST set this property.
+  This MAY be used for verifying the data of the chunk.
+
+- **`innerOffset`** *int64*
+
+  This OPTIONAL property indicates the uncompressed offset of the "reg" or "chunk" entry payload in a stream starts from `offset` field.
+
+#### Details about `innerOffset`
+
+`innerOffset` enables to put multiple "reg" or "chunk" payloads in one gzip stream starts from `offset`.
+This field allows the following structure.
+
+![The structure of eStargz with innerOffset](/docs/images/estargz-inneroffset.png)
+
+Use case of this field is `--estargz-min-chunk-size` flag of `ctr-remote`.
+The value of this flag is the minimal number of bytes of data must be written in one gzip stream.
+If it's > 0, multiple files and chunks can be written into one gzip stream.
+Smaller number of gzip header and smaller size of the result blob can be expected.
+
+### Footer
+
+At the end of the blob, a *footer* MUST be appended.
+This MUST be an empty gzip member whose [Extra field](https://tools.ietf.org/html/rfc1952#section-2.3.1.1) contains the offset of TOC in the blob.
+The footer MUST be the following 51 bytes (1 byte = 8 bits in gzip).
+
+```
+- 10 bytes  gzip header
+- 2  bytes  XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ"))
+- 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
+- 2  bytes  Extra: LEN = 22 (16 hex digits + len("STARGZ"))
+- 22 bytes  Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC)
+- 5  bytes  flate header: BFINAL = 1(last block), BTYPE = 0(non-compressed block), LEN = 0
+- 8  bytes  gzip footer
+(End of eStargz)
+```
+
+Runtimes MAY first read and parse the footer to get the offset of TOC.
+
+Each file's metadata is recorded in the TOC so runtimes don't need to extract other parts of the archive as long as it only uses file metadata.
+If runtime needs to get a regular file's content, it can get the size and offset of that content from the TOC and extract that range without scanning the entire blob.
+By combining this with HTTP Range Request supported by [OCI Distribution Spec](https://github.com/opencontainers/distribution-spec/blob/ef28f81727c3b5e98ab941ae050098ea664c0960/detail.md#fetch-blob-part), runtimes can selectively download file entries from the registry.
+
+### Notes on compatibility with stargz
+
+eStargz is designed aiming to compatibility with gzip layers.
+For achieving this, eStargz's footer structure is incompatible with [stargz's one](https://github.com/google/crfs/blob/71d77da419c90be7b05d12e59945ac7a8c94a543/stargz/stargz.go#L36-L49).
+eStargz adds SI1, SI2 and LEN fields to the footer to make it compliant to [Extra field definition in RFC1952](https://tools.ietf.org/html/rfc1952#section-2.3.1.1).
+TOC, TOCEntry and the position of gzip headers are still compatible with stargz.
+
+## Prioritized Files and Landmark Files
+
+![Prioritized files and landmark files](/docs/images/estargz-landmark.png)
+
+Lazy pulling can cause runtime performance overhead by on-demand fetching of each file.
+eStargz mitigates this by supporting prefetching of important files called *prioritized files*.
+
+eStargz encodes the information about prioritized files to the *order* of file entries with some *landmark* file entries.
+
+File entries in eStargz are grouped into the following groups,
+
+- A. *prioritized files*
+- B. non *prioritized files*
+
+If no files are belonging to A, a landmark file *no-prefetch landmark* MUST be contained in the archive.
+
+If one or more files are belonging to A, eStargz MUST consist of two separated areas corresponding to these groups and a landmark file *prefetch landmark* MUST be contained at the boundary between these two areas.
+
+The Landmark file MUST be a regular file entry with 4 bits contents 0xf in eStargz.
+It MUST be recorded to TOC as a TOCEntry. Prefetch landmark MUST be named `.prefetch.landmark`. No-prefetch landmark MUST be named `.no.prefetch.landmark`.
+
+### Example use-case of prioritized files: workload-based image optimization in Stargz Snapshotter
+
+Stargz Snapshotter makes use of eStargz's prioritized files for *workload-based* optimization to mitigate the overhead of reading files.
+The *workload* of the image is the runtime configuration defined in the Dockerfile, including entrypoint command, environment variables and user.
+
+Stargz snapshotter provides an image converter command `ctr-remote images optimize` to create optimized eStargz images.
+When converting the image, this command runs the specified workload in a sandboxed environment and profiles all file accesses.
+This command treats all accessed files as prioritized files.
+Then it constructs eStargz by
+
+- putting prioritized files from the top of the archive, sorting them by the accessed order,
+- putting *prefetch landmark* file entry at the end of this range, and
+- putting all other files (non-prioritized files) after the prefetch landmark.
+
+Before running the container, stargz snapshotter prefetches and pre-caches the range where prioritized files are contained, by a single HTTP Range Request supported by the registry.
+This can increase the cache hit rate for the specified workload and can mitigate runtime overheads.
+
+## Content Verification in eStargz
+
+The goal of the content verification in eStargz is to ensure the downloaded metadata and contents of all files are the expected ones, based on the calculated digests.
+The verification of other components in the image including image manifests is out-of-scope of eStargz.
+On the verification step of an eStargz layer, we assume that the manifest that references this eStargz layer is already verified (using digest tag, etc).
+
+![the overview of the verification](/docs/images/estargz-verification.png)
+
+A non-eStargz layer can be verified by recalculating the digest and comparing it with the one written in the layer descriptor referencing that layer in the verified manifest.
+However, an eStargz layer is *lazily* pulled from the registry in file (or chunk if that file is large) granularity so each one needs to be independently verified every time fetched.
+
+The following describes how the verification of eStargz is done using the verified manifest.
+
+eStargz consists of the following components to be verified:
+
+- TOC (a set of metadata of all files contained in the layer)
+- chunks of contents of each regular file
+
+TOC contains metadata (name, type, mode, etc.) of all files and chunks in the blob.
+On mounting eStargz, filesystem fetches the TOC from the registry.
+For making the TOC verifiable using the verified manifest, we define an annotation `containerd.io/snapshot/stargz/toc.digest`.
+The value of this annotation is the digest of the TOC and this MUST be contained in the descriptor that references this eStargz layer.
+Using this annotation, filesystem can verify the TOC by recalculating the digest and comparing it to the annotation value.
+
+Each file's metadata is encoded to a TOCEntry in the TOC.
+TOCEntry is created also for each chunk of regular files.
+For making the contents of each file and chunk verifiable using the verified manifest, TOCEntry has a property *chunkDigest*.
+*chunkDigest* contains the digest of the content of the `reg` or `chunk` entry.
+As mentioned above, the TOC is verifiable using the special annotation.
+Using *chunkDigest* fields written in the verified TOC, each file and chunk can be independently verified by recalculating the digest and comparing it to the property.
+
+As the conclusion, eStargz MUST contain the following metadata:
+
+- `containerd.io/snapshot/stargz/toc.digest` annotation in the descriptor that references eStargz layer: The value is the digest of the TOC.
+- *chunkDigest* properties of non-empty `reg` or `chunk` TOCEntry: The value is the digest of the contents of the file or chunk.
+
+### Example usecase: Content verification in Stargz Snapshotter
+
+Stargz Snapshotter verifies eStargz layers leveraging the above metadata.
+As mentioned above, the verification of other image components including the manifests is out-of-scope of the snapshotter.
+When this snapshotter mounts an eStargz layer, the manifest that references this layer must be verified in advance and the TOC digest annotation written in the verified manifest must be passed down to this snapshotter.
+
+On mounting a layer, stargz snapshotter fetches the TOC from the registry.
+Then it verifies the TOC by recalculating the digest and comparing it with the one written in the manifest.
+After the TOC is verified, the snapshotter mounts this layer using the metadata recorded in the TOC.
+
+During runtime of the container, this snapshotter fetches chunks of regular file contents lazily.
+Before providing a chunk to the filesystem user, snapshotter recalculates the digest and checks it matches the one recorded in the corresponding TOCEntry.
+
+## eStargz image with an external TOC (OPTIONAL)
+
+This OPTIONAL feature allows separating TOC into another image called *TOC image*.
+This type of eStargz is the same as the normal eStargz but doesn't contain TOC JSON file (`stargz.index.json`) in the layer blob and has a special footer.
+This feature enables creating a smaller eStargz blob by avoiding including TOC JSON file in that blob.
+
+Footer has the following structure:
+
+```
+// The footer is an empty gzip stream with no compression and an Extra header.
+//
+// 46 comes from:
+//
+// 10 bytes  gzip header
+// 2  bytes  XLEN (length of Extra field) = 21 (4 bytes header + len("STARGZEXTERNALTOC"))
+// 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
+// 2  bytes  Extra: LEN = 17 (len("STARGZEXTERNALTOC"))
+// 17 bytes  Extra: subfield = "STARGZEXTERNALTOC"
+// 5  bytes  flate header
+// 8  bytes  gzip footer
+// (End of the eStargz blob)
+```
+
+TOC image is an OCI image containing TOC.
+Each layer contains a TOC JSON file (`stargz.index.json`) in the root directory.
+
+Layer descriptors in the manifest must contain an annotation `containerd.io/snapshot/stargz/layer.digest`.
+The value of this annotation is the digest of the eStargz layer blob corresponding to that TOC.
+
+The following is an example layer descriptor in the TOC image.
+This layer (`sha256:64dedefd539280a5578c8b94bae6f7b4ebdbd12cb7a7df0770c4887a53d9af70`) contains the TOC JSON file (`stargz.index.json`) in the root directory and can be used for eStargz layer blob that has the digest `sha256:5da5601c1f2024c07f580c11b2eccf490cd499473883a113c376d64b9b10558f`.
+
+```json
+{
+  "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+  "digest": "sha256:64dedefd539280a5578c8b94bae6f7b4ebdbd12cb7a7df0770c4887a53d9af70",
+  "size": 154425,
+  "annotations": {
+    "containerd.io/snapshot/stargz/layer.digest": "sha256:5da5601c1f2024c07f580c11b2eccf490cd499473883a113c376d64b9b10558f"
+  }
+}
+```
+
+### Example usecase: lazy pulling with Stargz Snapshotter
+
+Stargz Snapshotter supports eStargz with external TOC.
+If an eStargz blob's footer indicates that it requires the TOC image, stargz snapshotter also pulls it from the registry.
+
+Stargz snapshotter assumes the TOC image has the reference name same as the eStargz with `-esgztoc` suffix.
+For example, if an eStargz image is named `ghcr.io/stargz-containers/ubuntu:22.04-esgz`, stargz snapshotter acquires the TOC image from `ghcr.io/stargz-containers/ubuntu:22.04-esgz-esgztoc`.
+Note that future versions of stargz snapshotter will support more ways to search the TOC image (e.g. allowing custom suffix, using OCI Reference Type, etc.)
+
+Once stargz snapshotter acquires TOC image, it tries to find the TOC corresponding to the mounting eStargz blob, by looking `containerd.io/snapshot/stargz/layer.digest` annotations.
+As describe in the above, the acquired TOC JSON is validated using `containerd.io/snapshot/stargz/toc.digest` annotation.
+
+## Example of TOC
+
+Here is an example TOC JSON:
+
+```json
+{
+  "version": 1,
+  "entries": [
+    {
+      "name": "bin/",
+      "type": "dir",
+      "modtime": "2019-08-20T10:30:43Z",
+      "mode": 16877,
+      "NumLink": 0
+    },
+    {
+      "name": "bin/busybox",
+      "type": "reg",
+      "size": 833104,
+      "modtime": "2019-06-12T17:52:45Z",
+      "mode": 33261,
+      "offset": 126,
+      "NumLink": 0,
+      "digest": "sha256:8b7c559b8cccca0d30d01bc4b5dc944766208a53d18a03aa8afe97252207521f",
+      "chunkDigest": "sha256:8b7c559b8cccca0d30d01bc4b5dc944766208a53d18a03aa8afe97252207521f"
+    },
+    {
+      "name": "lib/",
+      "type": "dir",
+      "modtime": "2019-08-20T10:30:43Z",
+      "mode": 16877,
+      "NumLink": 0
+    },
+    {
+      "name": "lib/ld-musl-x86_64.so.1",
+      "type": "reg",
+      "size": 580144,
+      "modtime": "2019-08-07T07:15:30Z",
+      "mode": 33261,
+      "offset": 512427,
+      "NumLink": 0,
+      "digest": "sha256:45c6ee3bd1862697eab8058ec0e462f5a760927331c709d7d233da8ffee40e9e",
+      "chunkDigest": "sha256:45c6ee3bd1862697eab8058ec0e462f5a760927331c709d7d233da8ffee40e9e"
+    },
+    {
+      "name": ".prefetch.landmark",
+      "type": "reg",
+      "size": 1,
+      "offset": 886633,
+      "NumLink": 0,
+      "digest": "sha256:dc0e9c3658a1a3ed1ec94274d8b19925c93e1abb7ddba294923ad9bde30f8cb8",
+      "chunkDigest": "sha256:dc0e9c3658a1a3ed1ec94274d8b19925c93e1abb7ddba294923ad9bde30f8cb8"
+    },
+... (omit) ...
+```

docs/integration.md

@@ -0,0 +1,205 @@
+# Integration of eStargz with other tools
+
+This document lists links and information about integrations of stargz-snapshotter with tools in commuinty.
+
+You can refer to [issue #258 "Tracker issue for adoption status"](https://github.com/containerd/stargz-snapshotter/issues/258) for the list of the latest status of these integrations.
+
+## Kubernetes
+
+To use stargz snapshotter on Kubernetes nodes, you need to use containerd as the CRI runtime.
+You also need to run stargz snapshotter on the node.
+
+### Kind
+
+See [`/README.md#quick-start-with-kubernetes`](/README.md#quick-start-with-kubernetes).
+
+### k3s
+
+k3s >= v1.22 supports stagz-snapshotter as an experimental feature.
+`--snapshotter=stargz` for k3s server and agent enables this feature.
+
+```
+k3s server --snapshotter=stargz
+```
+
+Refer to [k3s docs](https://docs.k3s.io/advanced#enabling-lazy-pulling-of-estargz-experimental) for more details.
+
+The following is a quick demo using [k3d](https://github.com/k3d-io/k3d) (k3s in Docker).
+
+```console
+$ k3d cluster create mycluster --k3s-arg='--snapshotter=stargz@server:*;agent:*'
+$ cat <<'EOF' | kubectl --context=k3d-mycluster apply -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nodejs
+spec:
+  containers:
+  - name: nodejs-stargz
+    image: ghcr.io/stargz-containers/node:17.8.0-esgz
+    command: ["node"]
+    args:
+    - -e
+    - var http = require('http');
+      http.createServer(function(req, res) {
+        res.writeHead(200);
+        res.end('Hello World!\n');
+      }).listen(80);
+    ports:
+    - containerPort: 80
+EOF
+$ kubectl --context=k3d-mycluster get po nodejs -w
+$ kubectl --context=k3d-mycluster port-forward nodejs 8080:80 &
+$ curl 127.0.0.1:8080
+Hello World!
+$ k3d cluster delete mycluster
+```
+
+### Google Kubernetes Engine
+
+There is no node image includes stargz snapshotter by default as of now so you need to manually customize the nodes.
+
+A brief instrcution of enabling stargz snapshotter is the following:
+
+- Create a Kubernetes cluster using containerd-supported Linux node images like `ubuntu_containerd`. containerd must be >= v1.4.2.
+- SSH into each node and install stargz snapshotter following [`./INSTALL.md`](./INSTALL.md#install-stargz-snapshotter-for-containerd-with-systemd). You need this installation on all worker nodes.
+- Optionally apply configuration to allow stargz-snapshotter to access private registries following [`./overview.md`](./overview.md#authentication).
+
+### Amazon Elastic Kubernetes Service
+
+There is no AMI includes stargz snapshotter by default as of now so you need to manually customize the nodes.
+
+A brief instrcution of enabling stargz snapshotter is the following:
+
+- Create a Kubernetes cluster using containerd-supported Linux AMIs. containerd must be >= v1.4.2. e.g. Amazon EKS optimized Amazon Linux AMIs with [containerd runtime bootstrap flag](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html).
+- SSH into each node and install stargz snapshotter following [`./INSTALL.md`](./INSTALL.md#install-stargz-snapshotter-for-containerd-with-systemd). You need this installation on all worker nodes.
+- Optionally apply configuration to allow stargz-snapshotter to access private registries following [`./overview.md`](./overview.md#authentication).
+
+## CRI runtimes
+
+### containerd
+
+See [`./INSTALL.md`](./INSTALL.md#install-stargz-snapshotter-for-containerd-with-systemd)
+
+> :information_source: There is also a doc for [integration with firecracker-containerd](https://github.com/firecracker-microvm/firecracker-containerd/blob/24f1fcf99ebf6edcb94edd71a2affbcdae6b08e7/docs/remote-snapshotter-getting-started.md).
+
+### CRI-O
+
+See [`./INSTALL.md`](./INSTALL.md#install-stargz-store-for-cri-opodman-with-systemd).
+
+## High-level container engines
+
+### Docker
+
+#### Moby
+
+Moby supports lazy pulling of eStargz since [`5c1d6c957b97321c8577e10ddbffe6e01981617a`](https://github.com/moby/moby/commit/5c1d6c957b97321c8577e10ddbffe6e01981617a) .
+See [`./INSTALL.md`](./INSTALL.md#install-stargz-snapshotter-for-dockermoby-with-systemd) for details.
+
+#### Docker Desktop
+
+Docker Desktop 4.12.0 "Containerd Image Store (Beta)" uses stargz-snapshotter.
+Refer to [Docker documentation](https://docs.docker.com/desktop/containerd/).
+
+### nerdctl
+
+See the [docs in nerdctl](https://github.com/containerd/nerdctl/blob/main/docs/stargz.md).
+
+### Podman
+
+See [`./INSTALL.md`](./INSTALL.md#install-stargz-store-for-cri-opodman-with-systemd).
+
+## Image builders
+
+### BuildKit
+
+#### Building eStargz
+
+BuildKit >= v0.10 supports creating eStargz images.
+See [`README.md`](/README.md#building-estargz-images-using-buildkit) for details.
+
+#### Lazy pulling of eStargz
+
+BuildKit >= v0.8 supports stargz-snapshotter and can perform lazy pulling of eStargz-formatted base images during build.
+`--oci-worker-snapshotter=stargz` flag enables this feature.
+
+You can try this feature using Docker Buildx as the following.
+
+```
+$ docker buildx create --use --name lazy-builder --buildkitd-flags '--oci-worker-snapshotter=stargz'
+$ docker buildx inspect --bootstrap lazy-builder
+```
+
+The following is a sample Dockerfile that uses eStargz-formatted golang image (`ghcr.io/stargz-containers/golang:1.18-esgz`) as the base image.
+
+```Dockerfile
+FROM ghcr.io/stargz-containers/golang:1.18-esgz AS dev
+COPY ./hello.go /hello.go
+RUN go build -o /hello /hello.go
+
+FROM scratch
+COPY --from=dev /hello /
+ENTRYPOINT [ "/hello" ]
+```
+
+Put the following Go source code in the context directory with naming it `hello.go`.
+
+```golang
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Println("Hello, world!")
+}
+```
+
+The following build performs lazy pulling of the eStargz-formatted golang base image.
+
+```console
+$ docker buildx build --load -t hello /tmp/ctx/
+$ docker run --rm hello
+Hello, world!
+```
+
+### Kaniko
+
+#### Building eStargz
+
+Kaniko >= v1.5.0 creates eStargz images when `GGCR_EXPERIMENT_ESTARGZ=1` is specified.
+See [`README.md`](/README.md#building-estargz-images-using-kaniko) for details.
+
+### ko
+
+ko >= v0.7.0 creates eStargz images when `GGCR_EXPERIMENT_ESTARGZ=1` is specified.
+Please see also [the docs in ko](https://github.com/ko-build/ko/blob/f70e3cad38c3bbd232f51604d922b8baff31144e/docs/advanced/faq.md#can-i-optimize-images-for-estargz-support).
+
+## P2P image distribution
+
+### IPFS
+
+See [`./ipfs.md`](./ipfs.md)
+
+### Dragonfly
+
+Change the `/etc/containerd-stargz-grpc/config.toml` configuration to make dragonfly as registry mirror.
+`127.0.0.1:65001` is the proxy address of dragonfly peer,
+and the `X-Dragonfly-Registry` header is the address of origin registry,
+which is provided for dragonfly to download the images.
+
+```toml
+[[resolver.host."docker.io".mirrors]]
+  host = "127.0.0.1:65001"
+  insecure = true
+  [resolver.host."docker.io".mirrors.header]
+    X-Dragonfly-Registry = ["https://index.docker.io"]
+```
+
+For more details about dragonfly as registry mirror,
+refer to [How to use Dragonfly With eStargz](https://d7y.io/docs/setup/integration/stargz/).
+
+## Registry-side conversion of eStargz
+
+### Harbor
+
+See the docs in Harbor: https://github.com/goharbor/acceleration-service

docs/ipfs.md

@@ -0,0 +1,177 @@
+# Running containers on IPFS (experimental)
+
+:information_source: This document isn't for Kubernetes environemnt. For information about node-to-node image sharing on Kubernetes, please refer to [the docs in nerdctl project](https://github.com/containerd/nerdctl/tree/main/examples/nerdctl-ipfs-registry-kubernetes).
+
+You can run OCI-compatible container images on IPFS with lazy pulling.
+
+To enable this feature, add the following configuration to `config.toml` of Stargz Snapsohtter (typically located at `/etc/containerd-stargz-grpc/config.toml`).
+
+```toml
+ipfs = true
+```
+
+> NOTE: containerd-stargz-grpc tries to connect to IPFS API written in `~/.ipfs/api` (or the file under `$IPFS_PATH` if configured) via HTTP (not HTTPS).
+
+## IPFS-enabled OCI Image
+
+For obtaining IPFS-enabled OCI Image, each descriptor in an OCI image must contain the following [IPFS URL](https://docs.ipfs.io/how-to/address-ipfs-on-web/#native-urls) in `urls` field.
+
+```
+ipfs://<CID>
+```
+
+`<CID>` is the Base32 case-insensitive CIDv1 of the blob that the descriptor points to.
+
+An image is represented as a CID pointing to the OCI descriptor of the top-level blob of the image (i.e. image index).
+
+The following is an example OCI descriptor pointing to the image index of an IPFS-enabled image:
+
+```console
+# ipfs cat bafkreie7754qk7fl56ebauawdgfuqqa3kdd7sotvuhsm6wbz3qin6ssw3a | jq
+{
+  "mediaType": "application/vnd.oci.image.index.v1+json",
+  "digest": "sha256:80d6aec48c0a74635a5f3dc106328c1673afaa21ed6e1270a9a44de66e8ffa55",
+  "size": 314,
+  "urls": [
+    "ipfs://bafkreiea22xmjdakorrvuxz5yeddfdawoox2uipnnyjhbknejxtg5d72ku"
+  ]
+}
+```
+
+## Lazy pulling with Stargz Snapshotter
+
+If layer descriptors of an image contain the URLs described above and these blobs are formatted as eStargz, Stargz Snapshotter mounts them from IPFS to the container's rootfs using FUSE with lazy pulling support.
+Thus container can startup without waiting for the entire image contents being locally available.
+Necessary chunks of contents (e.g. each file in the rootfs) are fetched from IPFS on-demand.
+
+If the container image isn't eStargz or the snapshotter isn't Stargz Snapshotter (e.g. overlayfs snapshotter), containerd fetches the entire image contents from IPFS and unpacks it to the local directory before starting the container.
+Thus possibly you'll see slow container cold-start.
+
+## Examples
+
+This section describes some examples of storing images to IPFS and running them as containers.
+Make sure IPFS daemon runs on your node.
+For example, you can run an IPFS daemon using the following command.
+
+```
+ipfs daemon
+```
+
+:information_source: If you don't want IPFS to communicate with nodes on the internet, you can run IPFS daemon in offline mode using `--offline` flag or you can create a private IPFS network as described in Appendix 1.
+
+### Running a container with lazy pulling
+
+`ctr-remote image ipfs-push` command converts an image to IPFS-enabled eStargz and stores it to IPFS.
+
+```console
+# ctr-remote i pull ghcr.io/stargz-containers/python:3.9-org
+# ctr-remote i ipfs-push ghcr.io/stargz-containers/python:3.9-org
+bafkreie7754qk7fl56ebauawdgfuqqa3kdd7sotvuhsm6wbz3qin6ssw3a
+```
+
+The printed IPFS CID (`bafkreie7754qk7fl56ebauawdgfuqqa3kdd7sotvuhsm6wbz3qin6ssw3a`) points to an OCI descriptor which points to the image index of the added image.
+
+```console
+# ipfs cat bafkreie7754qk7fl56ebauawdgfuqqa3kdd7sotvuhsm6wbz3qin6ssw3a | jq
+{
+  "mediaType": "application/vnd.oci.image.index.v1+json",
+  "digest": "sha256:80d6aec48c0a74635a5f3dc106328c1673afaa21ed6e1270a9a44de66e8ffa55",
+  "size": 314,
+  "urls": [
+    "ipfs://bafkreiea22xmjdakorrvuxz5yeddfdawoox2uipnnyjhbknejxtg5d72ku"
+  ]
+}
+```
+
+You can run this image from IPFS using that CID as an image reference for `ctr-remote image rpull`.
+`--ipfs` option is needed for enabling this.
+
+Note that `ctr-remote` accepts an IPFS CID as the image reference but doesn't support `/ipfs`-prefixed path as of now.
+We're working on eliminating this limitation.
+
+```console
+# time ( ctr-remote i rpull --ipfs bafkreie7754qk7fl56ebauawdgfuqqa3kdd7sotvuhsm6wbz3qin6ssw3a && \
+         ctr-remote run --snapshotter=stargz --rm -t bafkreie7754qk7fl56ebauawdgfuqqa3kdd7sotvuhsm6wbz3qin6ssw3a foo python -c 'print("Hello, World!")' )
+fetching sha256:80d6aec4... application/vnd.oci.image.index.v1+json
+fetching sha256:16d36f86... application/vnd.oci.image.manifest.v1+json
+fetching sha256:236b4bd7... application/vnd.oci.image.config.v1+json
+Hello, World!
+
+real	0m1.099s
+user	0m0.047s
+sys	0m0.037s
+```
+
+### Running a container without lazy pulling
+
+Though eStargz-based lazy pulling is highly recommended for speeding up the container startup time, you can store and run non-eStargz images with IPFS as well.
+In this case, containerd fetches the entire image contents from IPFS and unpacks it to the local directory before starting the container.
+
+You can add a non-eStargz image to IPFS using `--estargz=false` option.
+
+```console
+# ctr-remote i pull ghcr.io/stargz-containers/python:3.9-org
+# ctr-remote i ipfs-push --estargz=false ghcr.io/stargz-containers/python:3.9-org
+bafkreienbir4knaofs3o5f57kqw2the2v7zdhdlzpkq346mipuopwvqhty
+```
+
+You don't need FUSE nor stargz snapshotter for running this image but will see slow container cold-start.
+This example uses overlayfs snapshotter of containerd.
+
+```console
+# time ( ctr-remote i rpull --snapshotter=overlayfs --ipfs bafkreienbir4knaofs3o5f57kqw2the2v7zdhdlzpkq346mipuopwvqhty && \
+         ctr-remote run --snapshotter=overlayfs --rm -t bafkreienbir4knaofs3o5f57kqw2the2v7zdhdlzpkq346mipuopwvqhty foo python -c 'print("Hello, World!")' )
+fetching sha256:7240ac9f... application/vnd.oci.image.index.v1+json
+fetching sha256:17dc54f4... application/vnd.oci.image.manifest.v1+json
+fetching sha256:6f1289b1... application/vnd.oci.image.config.v1+json
+fetching sha256:9476e460... application/vnd.oci.image.layer.v1.tar+gzip
+fetching sha256:64c0f10e... application/vnd.oci.image.layer.v1.tar+gzip
+fetching sha256:4c25b309... application/vnd.oci.image.layer.v1.tar+gzip
+fetching sha256:942374d5... application/vnd.oci.image.layer.v1.tar+gzip
+fetching sha256:3fff52a3... application/vnd.oci.image.layer.v1.tar+gzip
+fetching sha256:5cf06daf... application/vnd.oci.image.layer.v1.tar+gzip
+fetching sha256:419e258e... application/vnd.oci.image.layer.v1.tar+gzip
+fetching sha256:1acf5650... application/vnd.oci.image.layer.v1.tar+gzip
+fetching sha256:b95c0dd0... application/vnd.oci.image.layer.v1.tar+gzip
+Hello, World!
+
+real	0m11.320s
+user	0m0.556s
+sys	0m0.280s
+```
+
+## Appendix 1: Creating IPFS private network
+
+You can create a private IPFS network as described in the official docs.
+
+- https://github.com/ipfs/go-ipfs/blob/v0.10.0/docs/experimental-features.md#private-networks
+
+The following is the summary.
+
+First, generate a key and save it to `~/.ipfs/swarm.key` (or under `$IPFS_PATH` if configured) of nodes you want to have in the network.
+IPFS only connects to peers having this key.
+
+```
+go install github.com/Kubuxu/go-ipfs-swarm-key-gen/ipfs-swarm-key-gen@latest
+~/go/bin/ipfs-swarm-key-gen > ~/.ipfs/swarm.key
+```
+
+Select nodes as a bootstrap nodes.
+IPFS daemons learn about the peers on the private network from them.
+Configure all non-bootstrap nodes to recognize only our bootstrap nodes instead of public ones like the following example.
+
+```
+ipfs bootstrap rm --all
+ipfs bootstrap add /ip4/<ip address of bootstrap node>/tcp/4001/ipfs/<Peer ID of the bootstrap node>
+```
+
+:information_source: You can get Peer ID of a node by `ipfs config show | grep "PeerID"`.
+
+Finally, start all nodes in the private network.
+
+```
+export LIBP2P_FORCE_PNET=1
+ipfs daemon
+```
+
+`LIBP2P_FORCE_PNET=1` makes sure that the daemon uses the private network and fails if the private network isn't configured.

docs/lima.md

@@ -0,0 +1,52 @@
+# Getting started with Stargz Snapshotter on Lima
+
+[Lima](https://github.com/lima-vm/lima) is a tool to manage Linux virtual machines on various hosts, including MacOS and Linux.
+Lima can be used as an easy way to get started with Stargz Snapshotter as Lima provides a default VM image bundling [containerd](https://github.com/containerd/containerd), [nerdctl](https://github.com/containerd/nerdctl)(Docker-compatible CLI of containerd) and Stargz Snapshotter.
+
+This document describes how to get started with Stargz Snapshotter on Lima.
+
+## Enable Stargz Snapshotter using `--snapshotter=stargz` flag
+
+nerdctl's `--snapshotter=stargz` flag enables stargz-snapshotter.
+
+```
+$ nerdctl.lima --snapshotter=stargz system info | grep stargz
+ Storage Driver: stargz
+```
+
+Using this flag, you can perform lazy pulling of a python eStargz image and run it.
+
+```
+$ nerdctl.lima --snapshotter=stargz run --rm -it --name python ghcr.io/stargz-containers/python:3.13-esgz
+Python 3.13.2 (main, Feb  6 2025, 22:37:13) [GCC 12.2.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>>
+```
+
+## Use Stargz Snapshotter as the default snapshotter
+
+nerdctl recognizes an environment variable `CONTAINERD_SNAPSHOTTER` for the snapshotter to use.
+You can add this environment variable to the VM by configuring Lima config as shown in the following:
+
+```
+$ cat <<EOF >> ~/.lima/_config/override.yaml
+env:
+  CONTAINERD_SNAPSHOTTER: stargz
+EOF
+$ limactl stop
+$ limactl start
+$ nerdctl.lima system info | grep Storage
+Storage Driver: stargz
+```
+
+> NOTE: `override.yaml` applies to all the instances of Lima
+
+
+You can perform lazy pulling of eStargz using nerdctl, without any extra flags.
+
+```
+$ nerdctl.lima run --rm -it --name python ghcr.io/stargz-containers/python:3.13-esgz
+Python 3.13.2 (main, Feb  6 2025, 22:37:13) [GCC 12.2.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>>
+```

docs/overview.md

@@ -1,22 +1,22 @@
 # Containerd Stargz Snapshotter Plugin Overview
 
-__Before get through this overview document, we recommend you to read [README](README.md).__
+__Before reading this overview document, we recommend you read [README](../README.md).__
 
-Pulling image is one of the time-consuming steps in the container startup process.
-In containerd community, we have had a lot of discussions to address this issue as the following,
+Pulling images is one of the most time-consuming steps in the container startup process.
+In the containerd community, we have had a lot of discussions to address this issue at the following:
 
 - [#3731 Support remote snapshotter to speed up image pulling](https://github.com/containerd/containerd/issues/3731)
 - [#2968 Support `Prepare` for existing snapshots in Snapshotter interface](https://github.com/containerd/containerd/issues/2968)
 - [#2943 remote filesystem snapshotter](https://github.com/containerd/containerd/issues/2943)
 
-The solution for the fast image distribution is called *Remote Snapshotter* plugin.
-This prepares container's rootfs layers by directly mounting from remote stores instead of downloading and unpacking the entire image contents.
-The actual image contents can be fetched *lazily* so runtimes can startup containers before the entire image contents to be locally available.
-We call these remotely mounted layers as *remote snapshots*.
+The solution for fast image distribution is called *Remote Snapshotter* plugin.
+This prepares the container's rootfs layers by directly mounting from remote stores instead of downloading and unpacking the entire image contents.
+The actual image contents can be fetched *lazily* so runtimes can start containers before the entire image contents are locally available.
+We call these remotely mounted layers *remote snapshots*.
 
 *Stargz Snapshotter* is a remote snapshotter plugin implementation which supports standard compatible remote snapshots functionality.
 This snapshotter leverages [eStargz](/docs/stargz-estargz.md) image, which is lazily-pullable and still standard-compatible.
-Because of this compatibility, eStargz image can be pushed to and lazily pulled from [OCI](https://github.com/opencontainers/distribution-spec)/[Docker](https://docs.docker.com/registry/spec/api/) registries (e.g. ghcr.io).
+Because of this compatibility, eStargz images can be pushed to and lazily pulled from [OCI](https://github.com/opencontainers/distribution-spec)/[Docker](https://docs.docker.com/registry/spec/api/) registries (e.g. ghcr.io).
 Furthermore, images can run even on eStargz-agnostic runtimes (e.g. Docker).
 When you run a container image and it is formatted by eStargz, stargz snapshotter prepares container's rootfs layers as remote snapshots by mounting layers from the registry to the node, instead of pulling the entire image contents.
 
@@ -27,10 +27,10 @@ This document gives you a high-level overview of stargz snapshotter.
 ## Stargz Snapshotter proxy plugin
 
 Stargz snapshotter is implemented as a [proxy plugin](https://github.com/containerd/containerd/blob/04985039cede6aafbb7dfb3206c9c4d04e2f924d/PLUGINS.md#proxy-plugins) daemon (`containerd-stargz-grpc`) for containerd.
-When containerd starts a container, it queries the rootfs snapshots to stargz snapshotter daemon through an unix socket.
+When containerd starts a container, it queries the rootfs snapshots to stargz snapshotter daemon through a unix socket.
 This snapshotter remotely mounts queried eStargz layers from registries to the node and provides these mount points as remote snapshots to containerd.
 
-Containerd recognizes this plugin through an unix socket specified in the configuration file (e.g. `/etc/containerd/config.toml`).
+Containerd recognizes this plugin through a unix socket specified in the configuration file (e.g. `/etc/containerd/config.toml`).
 Stargz snapshotter can also be used through Kubernetes CRI by specifying the snapshotter name in the CRI plugin configuration.
 We assume that you are using containerd (> v1.4.2).
 
@@ -44,6 +44,8 @@ version = 2
   [proxy_plugins.stargz]
     type = "snapshot"
     address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
 
 # Use stargz snapshotter through CRI
 [plugins."io.containerd.grpc.v1.cri".containerd]
@@ -51,24 +53,26 @@ version = 2
   disable_snapshot_annotations = false
 ```
 
+> NOTE: `root` field of `proxy_plugins` is needed for the CRI plugin to recognize stargz snapshotter's root directory.
+
 This repo contains [a Dockerfile as a KinD node image](/Dockerfile) which includes the above configuration.
 
 ## State directory
 
 Stargz snapshotter mounts eStargz layers from registries to the node using FUSE.
-The all files metadata in the image are preserved on the filesystem and files contents are fetched from registries on demand.
+Metadata for all files in the image are preserved on the container filesystem and the file contents are fetched from registries on demand.
 
-At the root of the filesystem, there is a *state directory* (`/.stargz-snapshotter`) for status monitoring for the filesystem.
+At the root of the container filesystem, there is a *state directory* (`/.stargz-snapshotter`) for status monitoring for the filesystem.
 This directory is hidden from `getdents(2)` so you can't see this with `ls -a /`.
 Instead, you can directly access the directory by specifying the path (`/.stargz-snapshotter`).
 
-State directory contains JSON-formatted metadata files for each layer.
+The state directory contains JSON-formatted metadata files for each layer.
 In the following example, metadata JSON files for overlayed 7 layers are visible.
-In each metadata JSON file, the following fields are contained,
+In each metadata JSON file, the following fields are contained:
 
 - `digest` contains the layer digest. This is the same value as that in the image's manifest.
 - `size` is the size bytes of the layer.
-- `fetchedSize` and `fetchedPercent` indicate how many bytes have been fetched for this layer. Stargz snapshotter aggressively downloads this layer in the background - unless configured otherwise - so these values gradually increase. When `fetchedPercent` reaches to `100` percents, this layer has been fully downloaded on the node and no further access will occur for reading files.
+- `fetchedSize` and `fetchedPercent` indicate how many bytes have been fetched for this layer. Stargz snapshotter aggressively downloads this layer in the background - unless configured otherwise - so these values gradually increase. When `fetchedPercent` reaches `100` percent, this layer has been fully downloaded on the node and no further access will occur for reading files.
 
 Note that the state directory layout and the metadata JSON structure are subject to change.
 
@@ -95,6 +99,59 @@ root@1d43741b8d29:/go# cat /.stargz-snapshotter/*
 {"digest":"sha256:f077511be7d385c17ba88980379c5cd0aab7068844dffa7a1cefbf68cc3daea3","size":580,"fetchedSize":580,"fetchedPercent":100}
 ```
 
+## Fuse Manager
+
+The fuse manager is designed to maintain the availability of running containers by managing the lifecycle of FUSE mountpoints independently from the stargz snapshotter.
+
+### Fuse Manager Overview
+Remote snapshots are mounted using FUSE, and its filesystem processes are attached to the stargz snapshotter. If the stargz snapshotter restarts (due to configuration changes or crashes), all filesystem processes will be killed and restarted, which causes the remount of FUSE mountpoints, making running containers unavailable.
+
+To avoid this, we use a fuse daemon called the fuse manager to handle filesystem processes. The fuse manager is responsible for mounting and unmounting remote snapshotters. Its process is detached from the stargz snapshotter main process to an independent one in a shim-like way during the snapshotter's startup. This design ensures that the restart of the snapshotter won't affect the filesystem processes it manages, keeping mountpoints and running containers available during the restart. However, it is important to note that the restart of the fuse manager itself triggers a remount, so it is recommended to keep the fuse manager running in a good state.
+
+You can enable the fuse manager by adding the following configuration.
+
+```toml
+[fusem_anager]
+enable = true
+```
+
+## Killing and restarting Stargz Snapshotter
+
+Stargz Snapshotter works as a FUSE server for the snapshots.
+When you stop Stargz Sanpshotter on the node, it takes the following behaviour depending on the configuration.
+
+### FUSE manager mode is disabled
+
+killing containerd-stargz-grpc will result in unmounting all snapshot mounts managed by Stargz Snapshotter.
+When containerd-stargz-grpc is restarted, all those snapshots are mounted again by lazy pulling all layers.
+If the snapshotter fails to mount one of the snapshots (e.g. because of lazy pulling failure) during this step, the behaviour differs depending on `allow_invalid_mounts_on_restart` flag in the config TOML.
+
+- `allow_invalid_mounts_on_restart = true`: containerd-stargz-grpc leaves the failed snapshots as empty directories. The user needs to manually remove those snapshot via containerd (e.g. using `ctr snapshot rm` command). The name of those snapshots can be seen in the log with `failed to restore remote snapshot` message.
+
+- `allow_invalid_mounts_on_restart = false`: containerd-stargz-grpc doesn't start. The user needs to manually recover this (e.g. by wiping snapshotter and containerd state).
+
+### FUSE manager mode is enabled
+
+Killing containerd-stargz-grpc using non-SIGINT signal (e.g. using SIGTERM) doesn't affect the snapshot mounts because the FUSE manager process detached from containerd-stargz-grpc keeps on serving FUSE mounts to the kernel.
+This is useful when you reload the updated config TOML to Stargz Snapshotter without unmounting existing snapshots.
+
+FUSE manager serves FUSE mounts of the snapshots so if you kill this process, all snapshot mounts will be unavailable.
+When stopping FUSE manager for upgrading the binary or restarting the node, you can use SIGINT signal to trigger the graceful exit as shown in the following steps.
+
+1. Stop containers that use Stargz Snapshotter. Stopping FUSE manager makes all snapshot mounts unavailable so containers can't keep working.
+2. Stop containerd-stargz-grpc process using SIGINT. This signal triggers unmounting of all snapshots and cleaning up of the associated resources.
+3. Kill the FUSE manager process (`stargz-fuse-manager`)
+4. Restart the containerd-stargz-grpc process. This restores all snapshot mounts by lazy pulling them. `allow_invalid_mounts_on_restart` (described in the above) can still be used for controlling the behaviour of the error cases.
+5. Restart the containers.
+
+### Unexpected restart handling
+
+When Stargz Snapshotter is killed unexpectedly (e.g., by OOM killer or system crash), the process doesn't get a chance to perform graceful cleanup. In such cases, the snapshotter can successfully restart and restore remote snapshots, but this may lead to fscache duplicating cached data.
+
+**Recommended handling:**
+
+Since this scenario is caused by abnormal exit, users are expected to manually clean up the cache directory after an unexpected restart to avoid cache duplication issues. The cache cleanup should be performed before restarting the snapshotter service.
+
 ## Registry-related configuration
 
 You can configure stargz snapshotter for accessing registries with custom configurations.
@@ -106,10 +163,13 @@ Stargz snapshotter doesn't share private registries creds with containerd.
 Instead, this supports authentication in the following methods,
 
 - Using `$DOCKER_CONFIG` or `~/.docker/config.json`
+- Proxying and scanning CRI Image Service API
 - Using Kubernetes secrets (type = `kubernetes.io/dockerconfigjson`)
 
+#### dockerconfig-based authentication
+
 By default, This snapshotter tries to get creds from `$DOCKER_CONFIG` or `~/.docker/config.json`.
-Following example enables stargz snapshotter to access to private registries using `docker login` command.
+Following example enables stargz snapshotter to access to private registries using `docker login` command. [`nerdctl login`](https://github.com/containerd/nerdctl) can also be used for this.
 Stargz snapshotter doesn't share credentials with containerd so credentials specified by `ctr-remote`'s `--user` option in the example is just for containerd.
 
 ```console
@@ -118,7 +178,36 @@ Stargz snapshotter doesn't share credentials with containerd so credentials spec
 # ctr-remote image rpull --user <username>:<password> docker.io/<your-repository>/ubuntu:18.04
 ```
 
-Following configuration enables stargz snapshotter to access to private registries using kubernetes secrets (type = `kubernetes.io/dockerconfigjson`) in the cluster using kubeconfig files.
+#### CRI-based authentication
+
+Following configuration (typically located at `/etc/containerd-stargz-grpc/config.toml`) enables stargz snapshotter to pull private images on Kubernetes.
+The snapshotter works as a proxy of CRI Image Service and exposes CRI Image Service API on the snapshotter's unix socket (i.e. `/run/containerd-stargz-grpc/containerd-stargz-grpc.sock`).
+The snapshotter acquires registry creds by scanning requests.
+
+You must specify `--image-service-endpoint=unix:///run/containerd-stargz-grpc/containerd-stargz-grpc.sock` option to kubelet.
+
+You can specify the backing image service's socket using `image_service_path`.
+The default is the containerd's socket (`/run/containerd/containerd.sock`).
+
+```toml
+# Stargz Snapshotter proxies CRI Image Service into containerd socket.
+[cri_keychain]
+enable_keychain = true
+image_service_path = "/run/containerd/containerd.sock"
+```
+
+The default path where containerd-stargz-grpc serves the CRI Image Service API is `unix:///run/containerd-stargz-grpc/containerd-stargz-grpc.sock`.
+You can also change this path using `listen_path` field.
+
+> Note that if you enabled the FUSE manager and CRI-based authentication together, `listen_path` is a mandatory field with some caveats:
+> - This path must be different from the FUSE manager's socket path (`/run/containerd-stargz-grpc/fuse-manager.sock`) because they have different lifecycle. Specifically, the CRI socket is recreted on each reload of the configuration to the FUSE manager.
+> - containerd-stargz-grpc's socket path (`/run/containerd-stargz-grpc/containerd-stargz-grpc.sock`) can't be used as `listen_path` because the CRI socket is served by the FUSE manager process (not containerd-stargz-grpc process).
+
+#### kubeconfig-based authentication
+
+This is another way to enable lazy pulling of private images on Kubernetes.
+
+Following configuration (typically located at `/etc/containerd-stargz-grpc/config.toml`) enables stargz snapshotter to access to private registries using kubernetes secrets (type = `kubernetes.io/dockerconfigjson`) in the cluster using kubeconfig files.
 You can specify the path of kubeconfig file using `kubeconfig_path` option.
 It's no problem that the specified file doesn't exist when this snapshotter starts.
 In this case, snapsohtter polls the file until actually provided.
@@ -132,7 +221,8 @@ enable_keychain = true
 kubeconfig_path = "/etc/kubernetes/snapshotter/config.conf"
 ```
 
-The config file can be passed to stargz snapshotter using `containerd-stargz-grpc`'s `--config` option.
+Please note that kubeconfig-based authentication requires additional privilege (i.e. kubeconfig to list/watch secrets) to the node.
+And this doesn't work if kubelet retrieve creds from somewhere not API server (e.g. [credential provider](https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider/)).
 
 ### Registry mirrors and insecure connection
 
@@ -153,6 +243,17 @@ host = "exampleregistry.io"
 insecure = true
 ```
 
+`header` field allows to set headers to send to the server.
+
+```toml
+[[resolver.host."registry2:5000".mirrors]]
+  host = "registry2:5000"
+  [resolver.host."registry2:5000".mirrors.header]
+    x-custom-2 = ["value3", "value4"]
+```
+
+> NOTE: Headers aren't passed to the redirected location.
+
 The config file can be passed to stargz snapshotter using `containerd-stargz-grpc`'s `--config` option.
 
 ## Make your remote snapshotter

docs/passthrough.md

@@ -0,0 +1,60 @@
+# Introduction
+
+FUSE Passthrough has been introduced in the Linux kernel version 6.9 ([Linux Kernel Commit](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=6ce8b2ce0d7e3a621cdc9eb66d74436ca7d0e66e)). This feature has shown significant performance improvements, as detailed in the following articles:
+
+[Phoronix Article on FUSE Passthrough](https://www.phoronix.com/news/FUSE-Passthrough-In-6.9-Next)<br>
+
+FUSE Passthrough allows performing read and write (also via memory maps) on a backing file without incurring the overhead of roundtrips to userspace.
+
+![passhthrough feature](/docs/images/passthrough01.png)
+
+Additionally, the `go-fuse` package, which Stargz-Snapshotter depends on, has also added support for this passthrough feature:
+
+[go-fuse Commit 1](https://github.com/hanwen/go-fuse/commit/e0641a46c6cca7e5370fc135f78caf7cb7fc3aa8#diff-f830ac3db25844bf71102b09e4e02f7213e9cdb577b32745979d61d775462bd3R157)<br>
+[go-fuse Commit 2](https://github.com/hanwen/go-fuse/commit/e0a0b09ae8287249c38033a27fd69a3593c7e235#diff-1521152f1fc3600273bda897c669523dc1e9fc9cbe24046838f043a8040f0d67R749)<br>
+[go-fuse Commit 3](https://github.com/hanwen/go-fuse/commit/1a7d98b0360f945fca50ac79905332b7106c049f)
+
+When a user-defined file implements the `FilePassthroughFder` interface, `go-fuse` will attempt to register the file `fd` from the file with the kernel.
+
+# Configuration
+
+## Basic Configuration
+
+To enable FUSE passthrough mode, first verify that your host's kernel supports this feature. You can check this by running the following command:
+
+```bash
+$ cat /boot/config-$(uname -r) | grep "CONFIG_FUSE_PASSTHROUGH=y"
+CONFIG_FUSE_PASSTHROUGH=y
+```
+
+Once you have confirmed kernel support, you need to enable passthrough mode in your `config.toml` file with the following configuration:
+
+```toml
+[fuse]
+passthrough = true
+```
+
+After updating the configuration, specify the `config.toml` file when starting `containerd-stargz-grpc` and restart the service:
+
+```bash
+$ containerd-stargz-grpc -config config.toml
+```
+
+## Advanced Configuration
+
+In passthrough mode, the initial pull of an image requires merging chunks into a file. This process can be time-consuming, especially for large files.
+
+To optimize the time taken for the initial image pull, you can use the `merge_buffer_size` and `merge_worker_count` configuration options. The `merge_buffer_size` specifies the size of the buffer used for reading the image, with a default value of 400MB. The `merge_worker_count` determines the level of concurrency for reading the image, with a default value of 10.
+
+By concurrently reading chunks and caching them for batch writing, you can significantly enhance the performance of the initial image pull in passthrough mode.
+
+# Important Considerations
+
+When passthrough mode is enabled, the following configuration is applied by default, even if it is set to false in the configuration file:
+
+```toml
+[directory_cache]
+direct = true
+```
+
+This is because, in passthrough mode, read operations after opening a file are handled directly by the kernel.

docs/pre-converted-images.md

@@ -1,16 +1,19 @@
 # Trying pre-converted images
 
-We have several pre-converted stargz images on Github Container Registry, mainly for benchmarking purpose. This doc lists these images in a table format. You can try them on your machine with our snapshotter. Please refer to README for the procedure.
+We have several pre-converted stargz images on Github Container Registry (`ghcr.io/stargz-containers`), mainly for benchmarking purpose.
+This document lists them.
 
-Please do not use them in production. You always can build your eStargz images optimized for your workload, using [`ctr-remote` command](/docs/ctr-remote.md).
+:information_source: You can build eStargz from Dockerfile using BuildKit, [using Docker Buildx](../README.md#building-estargz-images-using-buildkit) or [Kaniko](../README.md#building-estargz-images-using-kaniko).
+
+:information_source: You can convert arbitrary images into eStargz optimized for your workload, using [`ctr-remote` command](/docs/ctr-remote.md).
+
+:information_source: You can convert arbitrary images into eStargz on the registry-side, using [`estargz.kontain.me`](https://estargz.kontain.me).
 
 ## Pre-converted images
 
-This section contains a table of pre-converted images which can be used for benchmarking, testing, etc.
+:information_source: You can request new pre-converted images from our CI repository ([`github.com/stargz-containers/image-ci`](https://github.com/stargz-containers/image-ci)).
 
-We have pre-converted images on GitHub Container Registry. Images are stored under the repository `ghcr.io/stargz-containers`.
-
-Additionally, image names listed in `Image Name` contain the following suffixes based on the type of the image.
+In the following table, image names listed in `Image Name` contain the following suffixes based on the type of the image.
 
 - `org`: Legacy image copied from `docker.io/library` without optimization. Layers are normal tarballs.
 - `esgz`: eStargz-formatted version of the `org` images. `ctr-remote images optimize` command is used for the optimization.
@@ -19,35 +22,61 @@ Additionally, image names listed in `Image Name` contain the following suffixes
 
 |Image Name|Optimized Workload|
 ---|---
-|`ghcr.io/stargz-containers/alpine:3.10.2-org`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/alpine:3.10.2-esgz`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/drupal:8.7.6-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
-|`ghcr.io/stargz-containers/drupal:8.7.6-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
-|`ghcr.io/stargz-containers/fedora:30-org`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/fedora:30-esgz`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/gcc:9.2.0-org`|Compiling and executing a program which prints `hello`|
-|`ghcr.io/stargz-containers/gcc:9.2.0-esgz`|Compiling and executing a program which prints `hello`|
-|`ghcr.io/stargz-containers/glassfish:4.1-jdk8-org`|Code execution until up and ready message (`Running GlassFish`) is printed|
-|`ghcr.io/stargz-containers/glassfish:4.1-jdk8-esgz`|Code execution until up and ready message (`Running GlassFish`) is printed|
-|`ghcr.io/stargz-containers/golang:1.12.9-org`|Compiling and executing a program which prints `hello`|
-|`ghcr.io/stargz-containers/golang:1.12.9-esgz`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/alpine:3.15.3-org`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/alpine:3.15.3-esgz`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/drupal:9.3.9-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/drupal:9.3.9-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/fedora:35-org`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/fedora:35-esgz`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/gcc:11.2.0-org`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/gcc:11.2.0-esgz`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/golang:1.18-org`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/golang:1.18-esgz`|Compiling and executing a program which prints `hello`|
 |`ghcr.io/stargz-containers/jenkins:2.60.3-org`|Code execution until up and ready message (`Jenkins is fully up and running`) is printed|
 |`ghcr.io/stargz-containers/jenkins:2.60.3-esgz`|Code execution until up and ready message (`Jenkins is fully up and running`) is printed|
-|`ghcr.io/stargz-containers/jruby:9.2.8.0-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/jruby:9.2.8.0-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/node:13.13.0-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/node:13.13.0-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/perl:5.30-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/perl:5.30-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/php:7.3.8-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/php:7.3.8-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/pypy:3.5-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/pypy:3.5-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/python:3.7-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/python:3.7-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/r-base:3.6.1-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/r-base:3.6.1-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/redis:5.0.5-org`|Code execution until up and ready message (`Ready to accept connections`) is printed|
-|`ghcr.io/stargz-containers/redis:5.0.5-esgz`|Code execution until up and ready message (`Ready to accept connections`) is printed|
-|`ghcr.io/stargz-containers/rethinkdb:2.3.6-org`|Code execution until up and ready message (`Server ready`) is printed|
-|`ghcr.io/stargz-containers/rethinkdb:2.3.6-esgz`|Code execution until up and ready message (`Server ready`) is printed|
+|`ghcr.io/stargz-containers/jruby:9.3.4-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/jruby:9.3.4-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/node:17.8.0-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/node:17.8.0-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/perl:5.34.1-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/perl:5.34.1-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/php:8.1.4-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/php:8.1.4-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/pypy:3.9-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/pypy:3.9-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/python:3.10-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/python:3.10-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/r-base:4.1.3-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/r-base:4.1.3-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/redis:6.2.6-org`|Code execution until up and ready message (`Ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/redis:6.2.6-esgz`|Code execution until up and ready message (`Ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/rethinkdb:2.4.1-org`|Code execution until up and ready message (`Server ready`) is printed|
+|`ghcr.io/stargz-containers/rethinkdb:2.4.1-esgz`|Code execution until up and ready message (`Server ready`) is printed|
+|`ghcr.io/stargz-containers/tomcat:10.1.0-jdk17-openjdk-bullseye-org`|Code execution until up and ready message (`Server startup`) is printed|
+|`ghcr.io/stargz-containers/tomcat:10.1.0-jdk17-openjdk-bullseye-esgz`|Code execution until up and ready message (`Server startup`) is printed|
+|`ghcr.io/stargz-containers/postgres:14.2-org`|Code execution until up and ready message (`database system is ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/postgres:14.2-esgz`|Code execution until up and ready message (`database system is ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/wordpress:5.9.2-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/wordpress:5.9.2-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/mariadb:10.7.3-org`|Code execution until up and ready message (`mysqld: ready for connections`) is printed|
+|`ghcr.io/stargz-containers/mariadb:10.7.3-esgz`|Code execution until up and ready message (`mysqld: ready for connections`) is printed|
+|`ghcr.io/stargz-containers/php:8.1.4-apache-bullseye-org`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/php:8.1.4-apache-bullseye-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
+|`ghcr.io/stargz-containers/rabbitmq:3.9.14-org`|Code execution until up and ready message (`Server startup complete`) is printed|
+|`ghcr.io/stargz-containers/rabbitmq:3.9.14-esgz`|Code execution until up and ready message (`Server startup complete`) is printed|
+|`ghcr.io/stargz-containers/elasticsearch:8.1.1-org`|Code execution until up and ready message (`started`) is printed|
+|`ghcr.io/stargz-containers/elasticsearch:8.1.1-esgz`|Code execution until up and ready message (`started`) is printed|
+|`ghcr.io/stargz-containers/nixos/nix:2.3.12-org`|Executing `echo hello` on the shell|
+|`ghcr.io/stargz-containers/nixos/nix:2.3.12-esgz`|Executing `echo hello` on the shell|
+
+## lazy-pulling-enabled KinD node image
+
+You can enable lazy pulling of eStargz on [KinD](https://github.com/kubernetes-sigs/kind) using our prebuilt node image [`ghcr.io/containerd/stargz-snapshotter:${VERSION}-kind`](https://github.com/orgs/containerd/packages/container/package/stargz-snapshotter) namespace.
+
+Example:
+
+```console
+$ kind create cluster --name stargz-demo --image ghcr.io/containerd/stargz-snapshotter:0.12.1-kind
+```
+
+Please refer to README for more details.

docs/rootless.md

@@ -0,0 +1,56 @@
+# Rootless execution of stargz snapshotter
+
+This document lists links and information about how to run Stargz Snapshotter and Stargz Store from the non-root user.
+
+## nerdctl (Stargz Snapshotter)
+
+Rootless Stargz Snapshotter for nerdctl can be installed via `containerd-rootless-setuptool.sh install-stargz` command.
+Please see [the doc in nerdctl repo](https://github.com/containerd/nerdctl/blob/v1.1.0/docs/rootless.md#stargz-snapshotter) for details.
+
+## Podman (Stargz Store)
+
+> NOTE: This is an experimental configuration leveraging [`podman unshare`](https://docs.podman.io/en/latest/markdown/podman-unshare.1.html). Limitation: `--uidmap` of `podman run` doesn't work.
+
+First, allow podman using Stargz Store by adding the following store configuration.
+Put the configuration file to [`/etc/containers/storage.conf` or `$HOME/.config/containers/storage.conf`](https://github.com/containers/podman/blob/v4.3.1/docs/tutorials/rootless_tutorial.md#storageconf).
+
+> NOTE: Replace `/path/to/home` to the actual home directory.
+
+```
+[storage]
+driver = "overlay"
+
+[storage.options]
+additionallayerstores = ["/path/to/homedir/.local/share/stargz-store/store:ref"]
+```
+
+Start Stargz Store in the namespace managed by podman via [`podman unshare`](https://docs.podman.io/en/latest/markdown/podman-unshare.1.html) command.
+
+```
+$ podman unshare stargz-store --root $HOME/.local/share/stargz-store/data $HOME/.local/share/stargz-store/store &
+```
+
+Podman performs lazy pulling when it pulls eStargz images.
+
+```
+$ podman pull ghcr.io/stargz-containers/python:3.9-esgz
+```
+
+<details>
+<summary>Creating systemd unit file for Stargz Store</summary>
+
+It's possible to create systemd unit file of Stargz Store for easily managing it.
+An example systemd unit file can be found [here](../script/podman/config/podman-rootless-stargz-store.service)
+
+After installing that file (e.g. to `$HOME/.config/systemd/user/`), start the service using `systemctl`.
+
+```
+$ systemctl --user start podman-rootless-stargz-store
+```
+
+</details>
+
+## BuildKit (Stargz Snapshotter)
+
+BuildKit supports running Stargz Snapshotter from the non-root user.
+Please see [the doc in BuildKit repo](https://github.com/moby/buildkit/blob/8b132188aa7af944c813d02da63c93308d83cf75/docs/stargz-estargz.md) (unmerged 2023/1/18) for details.

docs/smaller-estargz.md

@@ -0,0 +1,79 @@
+# Creating smaller eStargz images
+
+The following flags of `ctr-remote i convert` and `ctr-remote i optimize` allow users optionally creating smaller eStargz images.
+
+- `--estargz-external-toc`: Separate TOC JSON into another image (called "TOC image"). The result eStargz doesn't contain TOC so we can expect a smaller size than normal eStargz.
+
+- `--estargz-min-chunk-size`: The minimal number of bytes of data must be written in one gzip stream. If it's > 0, multiple files and chunks can be written into one gzip stream. Smaller number of gzip header and smaller size of the result blob can be expected. `--estargz-min-chunk-size=0` produces normal eStargz.
+
+## `--estargz-external-toc` usage
+
+convert:
+
+```console
+# ctr-remote i pull ghcr.io/stargz-containers/ubuntu:22.04
+# ctr-remote i convert --oci --estargz --estargz-external-toc ghcr.io/stargz-containers/ubuntu:22.04 registry2:5000/ubuntu:22.04-ex
+```
+
+Layers in eStargz (`registry2:5000/ubuntu:22.04-ex`) don't contain TOC JSON.
+TOC image (`registry2:5000/ubuntu:22.04-ex-esgztoc`) contains TOC of all layers of the eStargz image.
+Suffix `-esgztoc` is automatically added to the image name by `ctr-remote`.
+
+Then push eStargz(`registry2:5000/ubuntu:22.04-ex`) and TOC image(`registry2:5000/ubuntu:22.04-ex-esgztoc`) to the same registry:
+
+```console
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-ex
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-ex-esgztoc
+```
+
+Pull it lazily:
+
+```console
+# ctr-remote i rpull --plain-http registry2:5000/ubuntu:22.04-ex
+fetching sha256:14fb0ea2... application/vnd.oci.image.index.v1+json
+fetching sha256:24471b45... application/vnd.oci.image.manifest.v1+json
+fetching sha256:d2e4737e... application/vnd.oci.image.config.v1+json
+# mount | grep "stargz on"
+stargz on /var/lib/containerd-stargz-grpc/snapshotter/snapshots/1/fs type fuse.rawBridge (rw,nodev,relatime,user_id=0,group_id=0,allow_other)
+```
+
+Stargz Snapshotter automatically refers to the TOC image on the same registry.
+
+### optional `--estargz-keep-diff-id` flag for conversion without changing layer diffID
+
+`ctr-remote i convert` supports optional flag `--estargz-keep-diff-id` specified with `--estargz-external-toc`.
+This converts an image to eStargz without changing the diffID (uncompressed digest) so even eStargz-agnostic gzip decompressor (e.g. gunzip) can restore the original tar blob.
+
+```console
+# ctr-remote i pull ghcr.io/stargz-containers/ubuntu:22.04
+# ctr-remote i convert --oci --estargz --estargz-external-toc --estargz-keep-diff-id ghcr.io/stargz-containers/ubuntu:22.04 registry2:5000/ubuntu:22.04-ex-keepdiff
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-ex-keepdiff
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-ex-keepdiff-esgztoc
+# crane --insecure blob registry2:5000/ubuntu:22.04-ex-keepdiff@sha256:2dc39ba059dcd42ade30aae30147b5692777ba9ff0779a62ad93a74de02e3e1f | jq -r '.rootfs.diff_ids[]'
+sha256:7f5cbd8cc787c8d628630756bcc7240e6c96b876c2882e6fc980a8b60cdfa274
+# crane blob ghcr.io/stargz-containers/ubuntu:22.04@sha256:2dc39ba059dcd42ade30aae30147b5692777ba9ff0779a62ad93a74de02e3e1f | jq -r '.rootfs.diff_ids[]'
+sha256:7f5cbd8cc787c8d628630756bcc7240e6c96b876c2882e6fc980a8b60cdfa274
+```
+
+## `--estargz-min-chunk-size` usage
+
+conversion:
+
+```console
+# ctr-remote i pull ghcr.io/stargz-containers/ubuntu:22.04
+# ctr-remote i convert --oci --estargz --estargz-min-chunk-size=50000 ghcr.io/stargz-containers/ubuntu:22.04 registry2:5000/ubuntu:22.04-chunk50000
+# ctr-remote i push --plain-http registry2:5000/ubuntu:22.04-chunk50000
+```
+
+Pull it lazily:
+
+```console
+# ctr-remote i rpull --plain-http registry2:5000/ubuntu:22.04-chunk50000
+fetching sha256:5d1409a2... application/vnd.oci.image.index.v1+json
+fetching sha256:859e2b50... application/vnd.oci.image.manifest.v1+json
+fetching sha256:c07a44b9... application/vnd.oci.image.config.v1+json
+# mount | grep "stargz on"
+stargz on /var/lib/containerd-stargz-grpc/snapshotter/snapshots/1/fs type fuse.rawBridge (rw,nodev,relatime,user_id=0,group_id=0,allow_other)
+```
+
+> NOTE: This flag creates an eStargz image with newly-added `innerOffset` funtionality of eStargz. Stargz Snapshotter < v0.13.0 cannot perform lazy pulling for the images created with this flag.

docs/stargz-estargz.md

@@ -1,293 +1,3 @@
 # eStargz: Standard-Compatible Extensions to Tar.gz Layers for Lazy Pulling Container Images
 
-This doc describes the extension to image layers for enabling *lazy image pulling*.
-The extended layer format is called *eStargz* in this project.
-eStargz is backward-compatible to tar.gz layers used in the current [OCI](https://github.com/opencontainers/image-spec/)/[Docker](https://github.com/moby/moby/blob/master/image/spec/v1.2.md) Image Specs so eStargz-formatted images can be pushed to and lazily pulled from standard registries.
-Furthermore, they can run even on extension-agnostic runtimes (e.g. Docker).
-
-This extension is based on stargz (stands for *seekable tar.gz*) proposed by [Google CRFS](https://github.com/google/crfs) project (initially [discussed in Go community](https://github.com/golang/go/issues/30829)).
-eStargz is an extended-version of stargz and comes with additional features including chunk-level verification and runtime performance optimization.
-
-Notational convention follows [OCI Image Spec](https://github.com/opencontainers/image-spec/blob/v1.0.1/spec.md#notational-conventions).
-
-## Overview
-
-When lazily pulling an image from the registry, necessary chunks of its layers are fetched *on-demand* during running the container, instead of downloading the entire contents of that image at once.
-For achieving this, runtimes need to *selectively* fetch and extract files contents in the layer.
-However, current OCI/Docker Image Spec uses tar (optionally with compression) for archiving layers, which doesn't suit to this use-case because of the following reasons,
-
-1. The entire archive needs to be scanned even for finding and extracting a single file.
-2. If the archive is compressed by gzip, this is no longer seekable.
-3. File entries in the archive cannot be verified separately (In Docker/OCI specs, verification is done for *the entire contents of the layer*, not per entry).
-
-eStargz is a tar.gz-compatible archive format which solves these issues and enables lazy pulling.
-Each file (or chunk for large files) in eStargz can be extracted selectively and verified separately.
-Additionally, eStargz has a feature called *prioritized files* for mitigating runtime performance drawbacks caused by on-demand fetching of each file/chunk.
-This format is compatible to tar.gz so eStargz layers are storable to container registries, lazily-pullable from container registries and still runnable even on eStargz-agnostic runtimes.
-
-This doc defines the basic structure of eStargz layer that has the above features.
-For details about content verfication in eStargz, please refer to [Content Verification in eStargz](/docs/verification.md).
-
-## The structure
-
-![The structure of eStargz](/docs/images/estargz-structure.png)
-
-In eStargz archive, each non-empty regular file is separately compressed by gzip.
-This structure is inherited from [stargz](https://github.com/google/crfs).
-
-The gzip headers MUST locate at the following locations.
-
-- The top of the tar archive
-- The top of the payload of each non-empty regular file entry except *TOC*
-- The top of *TOC* tar header
-- The top of *footer* (described in the later section)
-
-The gzip headers MAY locate at the following locations.
-
-- The end of the payload of each non-empty regular file entry
-- Arbitrary location within the payload of non-empty regular file entry
-
-The gzip header locations described in the second item MAY be used for chunking large regular files into several gzip members.
-Each chunked member is called *chunk* in this doc.
-An eStargz archive is the concatenation of these gzip members, which is a still valid gzip.
-
-## TOC, TOCEntries and Footer
-
-### TOC and TOCEntries
-
-A regular file entry called *TOC* MUST be contained as the last tar entry in the archive.
-TOC MUST be a JSON file and MUST be named `stargz.index.json`.
-
-TOC records all file's metadata (e.g. name, file type, owners, offset etc) in the tar archive, except TOC itself.
-The TOC is defined as the following.
-
-- **`version`** *int*
-
-   This REQUIRED property contains the version of the TOC. This value MUST be `1`.
-
-- **`entries`** *array of objects*
-
-   Each item in the array MUST be a TOCEntry.
-   This property MUST contain TOCEntries that reflect all tar entries and chunks, except `stargz.index.json`.
-
-The TOCEntry is defined as the following.
-If the information written in TOCEntry differs from the corresponding tar entry, TOCEntry SHOULD be respected.
-TOCEntries fields other than `chunkDigest` are inherited from [stargz](https://github.com/google/crfs).
-
-- **`name`** *string*
-
-  This REQUIRED property contains the name of the tar entry.
-  This MUST be the complete path stored in the tar file.
-
-- **`type`** *string*
-
-  This REQUIRED property contains the type of the tar entry.
-  This MUST be either of the following.
-  - `dir`: directory
-  - `reg`: regular file
-  - `symlink`: symbolic link
-  - `hardlink`: hard link
-  - `char`: character device
-  - `block`: block device
-  - `fifo`: fifo
-  - `chunk`: a chunk of regular file data
-  As described in the above section, a regular file can be divided into several chunks.
-  Corresponding to the first chunk of that file, TOCEntry typed `reg` MUST be contained.
-  Corresponding to the chunks after 2nd, TOCEntries typed `chunk` MUST be contained.
-  `chunk`-typed TOCEntry must set offset, chunkOffset and chunkSize properties.
-
-- **`size`** *uint64*
-
-  This OPTIONAL property contains the uncompressed size of the regular file tar entry.
-
-- **`modtime`** *string*
-
-  This OPTIONAL property contains the modification time of the tar entry.
-  Empty means zero or unknown.
-  Otherwize, the value is in UTC RFC3339 format.
-
-- **`linkName`** *string*
-
-  This OPTIONAL property contains the link target of `symlink` and `hardlink`.
-
-- **`mode`** *int64*
-
-  This OPTIONAL property contains the permission and mode bits.
-
-- **`uid`** *uint*
-
-  This OPTIONAL property contains the user ID of the owner of this file.
-
-- **`gid`** *uint*
-
-  This OPTIONAL property contains the group ID of the owner of this file.
-
-- **`userName`** *string*
-
-  This OPTIONAL property contains the username of the owner.
-
-- **`groupName`** *string*
-
-  This OPTIONAL property contains the groupname of the owner.
-
-- **`offset`** *int64*
-
-  This OPTIONAL property contains the offset of the gzip header of the regular file or chunk in the archive.
-
-- **`devMajor`** *int*
-
-  This OPTIONAL property contains the major device number for character and block device files.
-
-- **`devMinor`** *int*
-
-  This OPTIONAL property contains the minor device number for character and block device files.
-
-- **`xattrs`** *string-bytes map*
-
-  This OPTIONAL property contains the extended attribute for the tar entry.
-
-- **`digest`** *string*
-
-  This OPTIONAL property contains the OCI [Digest](https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md#digests) of the regular file contents.
-  TOCEntries of non-empty `reg` file MUST set this property.
-
-- **`chunkOffset`** *int64*
-
-  This OPTIONAL property contains the offset of this chunk in the regular file payload.
-  Note that this is the offset of this chunk in the decompressed file content.
-  TOCEntries of `chunk` type MUST set this property.
-
-- **`chunkSize`** *int64*
-
-  This OPTIONAL property contains the decompressed size of this chunk.
-  The last `chunk` in a `reg` file or `reg` file that isn't chunked MUST set this property to zero.
-  Other `reg` and `chunk` MUST set this property.
-
-- **`chunkDigest`** *string*
-
-  This OPTIONAL property contains an OCI [Digest](https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md#digests) of this chunk.
-  TOCEntries of non-empty `reg` and `chunk` MUST set this property.
-  This MAY be used for verifying the data of this entry in the way described in [Content Verification in eStargz](/docs/verification.md).
-
-### Footer
-
-At the end of the archive, a *footer* MUST be appended.
-This MUST be an empty gzip member ([RFC1952](https://tools.ietf.org/html/rfc1952)) whose [Extra field](https://tools.ietf.org/html/rfc1952#section-2.3.1.1) contains the offset of TOC in the archive.
-The footer MUST be the following 51 bytes (1 byte = 8 bits in gzip).
-
-```
-- 10 bytes  gzip header
-- 2  bytes  XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ"))
-- 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
-- 2  bytes  Extra: LEN = 22 (16 hex digits + len("STARGZ"))
-- 22 bytes  Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC)
-- 5  bytes  flate header: BFINAL = 1(last block), BTYPE = 0(non-compressed block), LEN = 0
-- 8  bytes  gzip footer
-(End of eStargz)
-```
-
-Runtimes MAY first read and parse the footer of the archive to get the offset of TOC.
-Each file's metadata is recorded in the TOC so runtimes don't need to extract other parts of the archive as long as it only uses file metadata.
-If runtime needs to get a regular file's content, it MAY get size and offset information of that content from the TOC and MAY extract that range without scanning the whole archive.
-By combining this with HTTP Range Request supported by [OCI Distribution Spec](https://github.com/opencontainers/distribution-spec/blob/master/spec.md#fetch-blob-part) and [Docker Registry API](https://docs.docker.com/registry/spec/api/#fetch-blob-part), runtimes can selectively download file entries from registries
-
-### Notes on compatibility with stargz
-
-eStargz is designed aiming to the compatibility with tar.gz.
-For achieving this, eStargz's footer structure is incompatible to [stargz's one](https://github.com/google/crfs/blob/71d77da419c90be7b05d12e59945ac7a8c94a543/stargz/stargz.go#L36-L49).
-eStargz adds SI1, SI2 and LEN fields to the footer for making it compliant to [Extra field definition in RFC1952](https://tools.ietf.org/html/rfc1952#section-2.3.1.1).
-TOC, TOCEntry and the position of gzip headers are still compatible with stargz.
-
-## Prioritized Files and Landmark Files
-
-![Prioritized files and landmark files](/docs/images/estargz-landmark.png)
-
-Lazy pulling costs extra time for reading files which induces remotely fetching file contents.
-The eStargz archive mitigates this problem with the ability to indicate the likely accessed files called *prioritized files*.
-Runtimes can leverage this information (e.g. for prefetching prioritized files) for increasing cache hit ratio and mitigating the read overhead (example usage of this information in Stargz Snapshotter is described in the later section).
-
-eStargz indicates the information about prioritized files as the *order* of file entries, with some [*landmark* file entries](https://github.com/containerd/stargz-snapshotter/blob/28af649b55ac39efc547b2e7f14f81a33a8212e1/stargz/fs.go#L93-L99).
-
-File entries in eStargz are grouped into the following groups,
-
-- A. files *likely accessed* by containers during runtime (i.e. prioritized files), and
-- B. files not likely accessed
-
-If there are no files belong to A, a landmark file *no-prefetch landmark* MUST be contained in the archive.
-If there are files belong to A, an eStargz archive MUST be made with two separated areas corresponding to these groups and a landmark file *prefetch landmark* MUST be containerd at the border between these two areas.
-That is, entries stored in the range between the top and the prefetch landmark are likely accessed during runtime.
-
-Both of landmark files MUST be regular file entries with 4 bits contents 0xf.
-Prefetch landmark MUST be registered to TOC as a TOCEntry named `.prefetch.landmark` and no-prefetch landmark MUST be registered as a TOCEntry named `.no.prefetch.landmark`.
-
-On container startup, the runtime SHOULD prefetch the range where prioritized files are contained.
-When the runtime finds no-prefetch landmark, it SHOULD NOT prefetch anything.
-
-## Example use-case of prioritized files: workload-based image optimization in Stargz Snapshotter
-
-Stargz Snapshotter makes use of eStargz's prioritized files for *workload-based* optimization for mitigating overhead of reading files.
-
-Generally, container images are built with purpose and the workloads are determined at the build.
-In many cases, a workload is defined in the Dockerfile using some parameters including entrypoint command, environment variables and user.
-
-Stargz snapshotter provides an image converter command `ctr-remote images optimize`.
-This leverages eStargz archive format and mitigates reading performance for files that are *likely accessed* in the workload defined in the Dockerfile.
-
-When converting the image, this command runs the specified workload in a sandboxed environment and profiles all file accesses.
-This command regards all accessed files as likely accessed also in production (i.e. prioritized files).
-Then it constructs eStargz archive by
-
-- locating accessed files from top of the archive, with sorting them by the accessed order,
-- putting prefetch landmark file entry at the end of this range, and
-- locating all other files (not accessed files) after the prefetch landmark.
-
-Before running the container, stargz snapshotter prefetches and pre-caches the range where prioritized files are contained, by a single HTTP Range Request.
-This can increase the cache hit rate for the specified workload and can mitigate runtime overheads.
-
-## Example of TOC
-
-You can inspect TOC JSON generated by `ctr-remote` converter like the following:
-
-```
-ctr-remote i optimize ghcr.io/stargz-containers/alpine:3.10.2-org local:///tmp/alpine
-cat /tmp/alpine/blobs/sha256/eec0c3d59c45a5ed4a7343afe3e871ce1cc99fb5db2b29af49fe67b6ad23ee62 \
-  | tar xzOf -  stargz.index.json | jq
-```
-
-Then you will get the TOC JSON something like:
-
-```json
-{
-  "version": 1,
-  "entries": [
-    {
-      "name": "etc/",
-      "type": "dir",
-      "modtime": "2019-08-20T10:30:43Z",
-      "mode": 16877,
-      "NumLink": 0
-    },
-    {
-      "name": "etc/passwd",
-      "type": "reg",
-      "size": 1230,
-      "modtime": "2019-06-17T09:00:16Z",
-      "mode": 33188,
-      "offset": 123,
-      "NumLink": 0,
-      "digest": "sha256:832cd0f75b227d13aac82b1f70b7f90191a4186c151f9db50851d209c45ede11",
-      "chunkDigest": "sha256:832cd0f75b227d13aac82b1f70b7f90191a4186c151f9db50851d209c45ede11"
-    },
-    {
-      "name": "etc/group",
-      "type": "reg",
-      "size": 697,
-      "modtime": "2019-06-17T09:00:16Z",
-      "mode": 33188,
-      "offset": 675,
-      "NumLink": 0,
-      "digest": "sha256:132d13f9260edf201efd1400b830ec39d25769c1a5b5a5dc74acdfed126f9d0a",
-      "chunkDigest": "sha256:132d13f9260edf201efd1400b830ec39d25769c1a5b5a5dc74acdfed126f9d0a"
-    },
-... (omit) ...
-```
+Moved to [`/docs/estargz.md`](/docs/estargz.md).

docs/transfer.md

@@ -0,0 +1,99 @@
+# Enabling Stargz Snapshotter With Transfer Service
+
+Transfer Service is a containerd component which is used for image management in contianerd (e.g. pulling and pushing images).
+For details about Transfer Service, refer to [the official document in the containerd repo](https://github.com/containerd/containerd/blob/6af7c07905a317d4c343a49255e2392f4c8569f9/docs/transfer.md).
+
+To use Stargz Snapshotter on containerd with enabling Transfer Service, additional configurations is needed.
+
+## Availability of Transfer Service
+
+Transfer Service is available since v1.7.
+And this is enabled in different settings depending on the containerd version.
+
+|containerd version|`ctr`|CRI|
+---|---|---
+|containerd >= v1.7 and < v2.0|Disabled by default. Enabled by `--local=false`|Disabled|
+|containerd >= v2.0 and < v2.1|Enabled by default. Disabled by `--local`|Disabled|
+|containerd >= v2.1|Enabled by default. Disabled by `--local`|Enabled by default. Disabled when conditions described in [containerd's CRI document](https://github.com/containerd/containerd/blob/v2.1.0/docs/cri/config.md#image-pull-configuration-since-containerd-v21) are met|
+
+### Note about containerd v2.1
+
+Before containerd v2.1, `disable_snapshot_annotations = false` in containerd's config TOML was a mandatory field to enable Stargz Snapshotter in CRI.
+In containerd v2.1, `disable_snapshot_annotations = false` field can still be used to enable Stargz Snapshotter and containerd disables Transfer Service when this field is detected.
+If you want to enable Transfer Service, you need to remove `disable_snapshot_annotations = false` field and apply the configuration explaind in this document.
+
+## How to enable Stargz Snapshotter when Transfer Service is enabled?
+
+In containerd v2.1, Transfer Service added support for remote snapshotters like Stargz Snapshotter.
+
+### For ctr and other non-CRI clients
+
+To enable Stargz Snapshotter with Transfer Service, you need to start containerd-stargz-grpc on the node and add the following configuration to contianerd's config TOML file.
+Note that you need to add a field `enable_remote_snapshot_annotations = "true"` in `proxy_plugins.stargz.exports` so that containerd can correctly pass image-related information to Stargz Snapshotter.
+
+```toml
+version = 2
+
+# Enable Stargz Snapshotter in Transfer Service
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "stargz"
+
+# Plugin Stargz Snapshotter
+[proxy_plugins]
+  [proxy_plugins.stargz]
+    type = "snapshot"
+    address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
+    enable_remote_snapshot_annotations = "true"
+```
+
+#### Example client command
+
+When you enable Transfer Service with Stargz Snapshotter, you can perform lazy pulling using the normal `ctr` command. (of course, `ctr-remote` can still be used)
+
+```
+# ctr image pull --snapshotter=stargz ghcr.io/stargz-containers/ubuntu:24.04-esgz
+```
+
+Then `mount | grep stargz` prints stargz mounts on the node.
+
+### For CRI
+
+To enable Stargz Snapshotter with Transfer Service, you need to start containerd-stargz-grpc on the node and add the following configuration to contianerd's config TOML file.
+
+```toml
+version = 2
+
+# Basic CRI configuration with enabling Stargz Snapshotter
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  default_runtime_name = "runc"
+  snapshotter = "stargz"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+# Enable Stargz Snapshotter in Transfer Service
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+  platform = "linux"
+  snapshotter = "stargz"
+
+# Plugin Stargz Snapshotter
+[proxy_plugins]
+  [proxy_plugins.stargz]
+    type = "snapshot"
+    address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+  [proxy_plugins.stargz.exports]
+    root = "/var/lib/containerd-stargz-grpc/"
+    enable_remote_snapshot_annotations = "true"
+```
+
+#### Example client command
+
+You can quickly check the behaviour using `crictl` command.
+
+```
+# crictl image pull ghcr.io/stargz-containers/ubuntu:24.04-esgz
+```
+
+Then `mount | grep stargz` prints stargz mounts on the node.

docs/verification.md

@@ -1,61 +1,3 @@
 # Content Verification in eStargz
 
-The goal of the content verification in eStargz is to ensure the downloaded metadata and contents of all files are the expected ones, based on the calculated OCI [_digests_](https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md#digests).
-The verification of other components in the image including image manifests is out-of-scope.
-On the verification step of an eStargz layer, we assume that the [_manifest_](https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md) that references this eStargz layer is verified in containerd in advance (using digest tag or [`Docker-Content-Digest` header](https://docs.docker.com/registry/spec/api/#digest-header), etc).
-
-![the overview of the verification](/docs/images/verification.png)
-
-## Verifiable eStargz
-
-For a layer that isn't lazily pulled (i.e. traditional tar.gz layer), it can be verified by recalculating the digest and compare it with the one written in the layer [_descriptor_](https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md) referencing that layer in the verified manifest.
-However, an eStargz layer is **lazily** pulled from the registry in file (or chunk if that file is large) granularity.
-So it's not possible to recalculate and verify the digest of the entire layer on mount.
-Assuming that the manifest referencing the eStargz layer has already been verified, we verify that eStargz layer as the following.
-
-When stargz snapshotter lazily pulls an eStargz layer, the following components will be fetched from the registry.
-
-- TOC (a set of metadata of all files contained in the layer)
-- chunks of regular file contents
-
-As mentioned in [eStargz documentation](/docs/stargz-estargz.md), eStargz contains an index file called _TOC_.
-Not only offset information of file entries, it [contains metadata (name, type, mode, etc.) of all files contained in the layer blob](https://github.com/google/crfs/blob/71d77da419c90be7b05d12e59945ac7a8c94a543/stargz/stargz.go#L214-L218).
-On mount the layer, filesystem fetches the TOC from the registry.
-For making the TOC verifiable using the manifest, we define an [_annotation_](https://github.com/opencontainers/image-spec/blob/v1.0.1/descriptor.md#properties) `containerd.io/snapshot/stargz/toc.digest`.
-The value of this annotation is the digest of the TOC and this annotation must be contained in descriptors that references this eStargz layer in the manifest.
-Using this annotation, filesystem can verify the TOC by recalculating the digest and compare it to the one written in the verified manifest.
-
-Each file's metadata (name, type, mode, etc.) is formed as a [_TOCEntry_](https://github.com/google/crfs/blob/71d77da419c90be7b05d12e59945ac7a8c94a543/stargz/stargz.go#L109-L191) in the TOC.
-TOCEntry is also created for each chunk of regular file content.
-For making each chunk verifiable using the manifest, eStargz extends the TOCEntry definition with [an optional field `chunkDigest`](https://github.com/containerd/stargz-snapshotter/blob/b53e8fe8d37751753bc623b037729b6a6d9c1122/stargz/verify/verify.go#L56-L64).
-`chunkDigest` is a field to contain the digest of each chunk.
-As mentioned in the above, the TOC is verifiable using the manifest with the special annotation.
-So using `chunkDigest` fields, filesystem can verify each chunk by recalculating the digest and compare it to the one written in the verified TOC.
-
-As the conclusion, the following conditions must meet for eStargz.
-
-- the digest of the TOC is contained in the annotation(`containerd.io/snapshot/stargz/toc.digest`) of descriptors that references this layer, and
-- `chunkDigest` fields of all chunks in the TOC is filled with the digests of their contents.
-
-`ctr-remote images optimize` command in this project creates the verifiable eStargz image by default.
-
-## Example usecase: Content verification in Stargz Snapshotter
-
-Stargz Snapshotter verifies eStargz layers leveraging the above extensions.
-However, as mentioned in the above, the verification of other image component including the manifests is out-of-scope of the snapshotter.
-So when this snapshotter mounts an eStargz layer, the manifest that references this layer must be verified in the containerd in advance and the TOC's digest written in the manifest (as a layer annotation `containerd.io/snapshot/stargz/toc.digest`) must be passed down to this snapshotter.
-This annotation is valid only when it is specified in `.[]layers.annotations` of [an image manifest](https://github.com/opencontainers/image-spec/blob/v1.0.1/manifest.md#image-manifest-property-descriptions).
-
-If the layer doesn't contain `containerd.io/snapshot/stargz/toc.digest` annotation, verification can't be done for that layer so stargz snapshotter reports an error and doesn't mount it.
-You can bypass this check only if both of the following conditions meet.
-
-- `allow_no_verification = true` is specified in `config.toml` of stargz snapshotter, and
-- the content descriptor of this layer has an annotation `containerd.io/snapshot/remote/stargz.skipverify` (the value will be ignored).
-
-The other way is to disable verification completely by setting `disable_verification = true` in `config.toml` of stargz snapshotter.
-
-On mounting a layer, stargz snapshotter fetches this layer's TOC from the registry.
-Then it verifies the TOC by recaluculating the digest and comparing it with the one passed from containerd (written in the manifest).
-If the TOC is successfully verified, then the snapshotter mounts this layer using the metadata stored in the TOC.
-During runtime of the container, this snapshotter fetches chunks of regular files lazily.
-Before providing a chunk to the filesystem user, snapshotter recalculates the digest and checks it matches the one contained in the corresponding TOCEntry in the TOC.
+Moved to [`/docs/estargz.md`](/docs/estargz.md).

estargz/build.go

@@ -26,10 +26,10 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
-	"encoding/json"
+	"context"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"runtime"
@@ -37,8 +37,8 @@ import (
 	"sync"
 
 	"github.com/containerd/stargz-snapshotter/estargz/errorutil"
+	"github.com/klauspost/compress/zstd"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -47,6 +47,9 @@ type options struct {
 	compressionLevel       int
 	prioritizedFiles       []string
 	missedPrioritizedFiles *[]string
+	compression            Compression
+	ctx                    context.Context
+	minChunkSize           int
 }
 
 type Option func(o *options) error
@@ -61,6 +64,7 @@ func WithChunkSize(chunkSize int) Option {
 
 // WithCompressionLevel option specifies the gzip compression level.
 // The default is gzip.BestCompression.
+// This option will be ignored if WithCompression option is used.
 // See also: https://godoc.org/compress/gzip#pkg-constants
 func WithCompressionLevel(level int) Option {
 	return func(o *options) error {
@@ -94,6 +98,35 @@ func WithAllowPrioritizeNotFound(missedFiles *[]string) Option {
 	}
 }
 
+// WithCompression specifies compression algorithm to be used.
+// Default is gzip.
+func WithCompression(compression Compression) Option {
+	return func(o *options) error {
+		o.compression = compression
+		return nil
+	}
+}
+
+// WithContext specifies a context that can be used for clean canceleration.
+func WithContext(ctx context.Context) Option {
+	return func(o *options) error {
+		o.ctx = ctx
+		return nil
+	}
+}
+
+// WithMinChunkSize option specifies the minimal number of bytes of data
+// must be written in one gzip stream.
+// By increasing this number, one gzip stream can contain multiple files
+// and it hopefully leads to smaller result blob.
+// NOTE: This adds a TOC property that old reader doesn't understand.
+func WithMinChunkSize(minChunkSize int) Option {
+	return func(o *options) error {
+		o.minChunkSize = minChunkSize
+		return nil
+	}
+}
+
 // Blob is an eStargz blob.
 type Blob struct {
 	io.ReadCloser
@@ -112,11 +145,11 @@ func (b *Blob) TOCDigest() digest.Digest {
 	return b.tocDigest
 }
 
-// Build builds an eStargz blob which is an extended version of stargz, from tar blob passed
-// through the argument. If there are some prioritized files are listed in the option, these
-// files are grouped as "prioritized" and can be used for runtime optimization (e.g. prefetch).
-// This function builds a blob in parallel, with dividing that blob into several (at least the
-// number of runtime.GOMAXPROCS(0)) sub-blobs.
+// Build builds an eStargz blob which is an extended version of stargz, from a blob (gzip, zstd
+// or plain tar) passed through the argument. If there are some prioritized files are listed in
+// the option, these files are grouped as "prioritized" and can be used for runtime optimization
+// (e.g. prefetch). This function builds a blob in parallel, with dividing that blob into several
+// (at least the number of runtime.GOMAXPROCS(0)) sub-blobs.
 func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 	var opts options
 	opts.compressionLevel = gzip.BestCompression // BestCompression by default
@@ -125,19 +158,50 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 			return nil, err
 		}
 	}
+	if opts.compression == nil {
+		opts.compression = newGzipCompressionWithLevel(opts.compressionLevel)
+	}
 	layerFiles := newTempFiles()
+	ctx := opts.ctx
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-done:
+			// nop
+		case <-ctx.Done():
+			layerFiles.CleanupAll()
+		}
+	}()
 	defer func() {
 		if rErr != nil {
 			if err := layerFiles.CleanupAll(); err != nil {
-				rErr = errors.Wrapf(rErr, "failed to cleanup tmp files: %v", err)
+				rErr = fmt.Errorf("failed to cleanup tmp files: %v: %w", err, rErr)
 			}
 		}
+		if cErr := ctx.Err(); cErr != nil {
+			rErr = fmt.Errorf("error from context %q: %w", cErr, rErr)
+		}
 	}()
+	tarBlob, err := decompressBlob(tarBlob, layerFiles)
+	if err != nil {
+		return nil, err
+	}
 	entries, err := sortEntries(tarBlob, opts.prioritizedFiles, opts.missedPrioritizedFiles)
 	if err != nil {
 		return nil, err
 	}
-	tarParts := divideEntries(entries, runtime.GOMAXPROCS(0))
+	var tarParts [][]*entry
+	if opts.minChunkSize > 0 {
+		// Each entry needs to know the size of the current gzip stream so they
+		// cannot be processed in parallel.
+		tarParts = [][]*entry{entries}
+	} else {
+		tarParts = divideEntries(entries, runtime.GOMAXPROCS(0))
+	}
 	writers := make([]*Writer, len(tarParts))
 	payloads := make([]*os.File, len(tarParts))
 	var mu sync.Mutex
@@ -150,8 +214,15 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 			if err != nil {
 				return err
 			}
-			sw := NewWriterLevel(esgzFile, opts.compressionLevel)
+			sw := NewWriterWithCompressor(esgzFile, opts.compression)
 			sw.ChunkSize = opts.chunkSize
+			sw.MinChunkSize = opts.minChunkSize
+			if sw.needsOpenGzEntries == nil {
+				sw.needsOpenGzEntries = make(map[string]struct{})
+			}
+			for _, f := range []string{PrefetchLandmark, NoPrefetchLandmark} {
+				sw.needsOpenGzEntries[f] = struct{}{}
+			}
 			if err := sw.AppendTar(readerFromEntries(parts...)); err != nil {
 				return err
 			}
@@ -166,7 +237,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 		rErr = err
 		return nil, err
 	}
-	tocAndFooter, tocDgst, err := closeWithCombine(opts.compressionLevel, writers...)
+	tocAndFooter, tocDgst, err := closeWithCombine(writers...)
 	if err != nil {
 		rErr = err
 		return nil, err
@@ -182,11 +253,12 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 	diffID := digest.Canonical.Digester()
 	pr, pw := io.Pipe()
 	go func() {
-		r, err := gzip.NewReader(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw))
+		r, err := opts.compression.Reader(io.TeeReader(io.MultiReader(append(rs, tocAndFooter)...), pw))
 		if err != nil {
 			pw.CloseWithError(err)
 			return
 		}
+		defer r.Close()
 		if _, err := io.Copy(diffID.Hash(), r); err != nil {
 			pw.CloseWithError(err)
 			return
@@ -208,7 +280,7 @@ func Build(tarBlob *io.SectionReader, opt ...Option) (_ *Blob, rErr error) {
 // Writers doesn't write TOC and footer to the underlying writers so they can be
 // combined into a single eStargz and tocAndFooter returned by this function can
 // be appended at the tail of that combined blob.
-func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooter io.Reader, tocDgst digest.Digest, err error) {
+func closeWithCombine(ws ...*Writer) (tocAndFooterR io.Reader, tocDgst digest.Digest, err error) {
 	if len(ws) == 0 {
 		return nil, "", fmt.Errorf("at least one writer must be passed")
 	}
@@ -225,7 +297,7 @@ func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooter io.Read
 		}
 	}
 	var (
-		mtoc          = new(jtoc)
+		mtoc          = new(JTOC)
 		currentOffset int64
 	)
 	mtoc.Version = ws[0].toc.Version
@@ -243,40 +315,16 @@ func closeWithCombine(compressionLevel int, ws ...*Writer) (tocAndFooter io.Read
 		currentOffset += w.cw.n
 	}
 
-	tocJSON, err := json.MarshalIndent(mtoc, "", "\t")
+	return tocAndFooter(ws[0].compressor, mtoc, currentOffset)
+}
+
+func tocAndFooter(compressor Compressor, toc *JTOC, offset int64) (io.Reader, digest.Digest, error) {
+	buf := new(bytes.Buffer)
+	tocDigest, err := compressor.WriteTOCAndFooter(buf, offset, toc, nil)
 	if err != nil {
 		return nil, "", err
 	}
-	pr, pw := io.Pipe()
-	go func() {
-		zw, _ := gzip.NewWriterLevel(pw, compressionLevel)
-		tw := tar.NewWriter(zw)
-		if err := tw.WriteHeader(&tar.Header{
-			Typeflag: tar.TypeReg,
-			Name:     TOCTarName,
-			Size:     int64(len(tocJSON)),
-		}); err != nil {
-			pw.CloseWithError(err)
-			return
-		}
-		if _, err := tw.Write(tocJSON); err != nil {
-			pw.CloseWithError(err)
-			return
-		}
-		if err := tw.Close(); err != nil {
-			pw.CloseWithError(err)
-			return
-		}
-		if err := zw.Close(); err != nil {
-			pw.CloseWithError(err)
-			return
-		}
-		pw.Close()
-	}()
-	return io.MultiReader(
-		pr,
-		bytes.NewReader(footerBytes(currentOffset)),
-	), digest.FromBytes(tocJSON), nil
+	return buf, tocDigest, nil
 }
 
 // divideEntries divides passed entries to the parts at least the number specified by the
@@ -313,7 +361,7 @@ func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]stri
 	// Import tar file.
 	intar, err := importTar(in)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to sort")
+		return nil, fmt.Errorf("failed to sort: %w", err)
 	}
 
 	// Sort the tar file respecting to the prioritized files list.
@@ -324,7 +372,7 @@ func sortEntries(in io.ReaderAt, prioritized []string, missedPrioritized *[]stri
 				*missedPrioritized = append(*missedPrioritized, l)
 				continue // allow not found
 			}
-			return nil, errors.Wrap(err, "failed to sort tar entries")
+			return nil, fmt.Errorf("failed to sort tar entries: %w", err)
 		}
 	}
 	if len(prioritized) == 0 {
@@ -360,11 +408,11 @@ func readerFromEntries(entries ...*entry) io.Reader {
 		defer tw.Close()
 		for _, entry := range entries {
 			if err := tw.WriteHeader(entry.header); err != nil {
-				pw.CloseWithError(fmt.Errorf("Failed to write tar header: %v", err))
+				pw.CloseWithError(fmt.Errorf("failed to write tar header: %v", err))
 				return
 			}
 			if _, err := io.Copy(tw, entry.payload); err != nil {
-				pw.CloseWithError(fmt.Errorf("Failed to write tar payload: %v", err))
+				pw.CloseWithError(fmt.Errorf("failed to write tar payload: %v", err))
 				return
 			}
 		}
@@ -375,9 +423,9 @@ func readerFromEntries(entries ...*entry) io.Reader {
 
 func importTar(in io.ReaderAt) (*tarFile, error) {
 	tf := &tarFile{}
-	pw, err := newCountReader(in)
+	pw, err := newCountReadSeeker(in)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to make position watcher")
+		return nil, fmt.Errorf("failed to make position watcher: %w", err)
 	}
 	tr := tar.NewReader(pw)
 
@@ -388,9 +436,8 @@ func importTar(in io.ReaderAt) (*tarFile, error) {
 		if err != nil {
 			if err == io.EOF {
 				break
-			} else {
-				return nil, errors.Wrap(err, "failed to parse tar file")
 			}
+			return nil, fmt.Errorf("failed to parse tar file, %w", err)
 		}
 		switch cleanEntryName(h.Name) {
 		case PrefetchLandmark, NoPrefetchLandmark:
@@ -426,7 +473,7 @@ func moveRec(name string, in *tarFile, out *tarFile) error {
 	_, okIn := in.get(name)
 	_, okOut := out.get(name)
 	if !okIn && !okOut {
-		return errors.Wrapf(errNotFound, "file: %q", name)
+		return fmt.Errorf("file: %q: %w", name, errNotFound)
 	}
 
 	parent, _ := path.Split(strings.TrimSuffix(name, "/"))
@@ -512,12 +559,13 @@ func newTempFiles() *tempFiles {
 }
 
 type tempFiles struct {
-	files   []*os.File
-	filesMu sync.Mutex
+	files       []*os.File
+	filesMu     sync.Mutex
+	cleanupOnce sync.Once
 }
 
 func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) {
-	f, err := ioutil.TempFile(dir, pattern)
+	f, err := os.CreateTemp(dir, pattern)
 	if err != nil {
 		return nil, err
 	}
@@ -527,7 +575,14 @@ func (tf *tempFiles) TempFile(dir, pattern string) (*os.File, error) {
 	return f, nil
 }
 
-func (tf *tempFiles) CleanupAll() error {
+func (tf *tempFiles) CleanupAll() (err error) {
+	tf.cleanupOnce.Do(func() {
+		err = tf.cleanupAll()
+	})
+	return
+}
+
+func (tf *tempFiles) cleanupAll() error {
 	tf.filesMu.Lock()
 	defer tf.filesMu.Unlock()
 	var allErr []error
@@ -543,19 +598,19 @@ func (tf *tempFiles) CleanupAll() error {
 	return errorutil.Aggregate(allErr)
 }
 
-func newCountReader(r io.ReaderAt) (*countReader, error) {
+func newCountReadSeeker(r io.ReaderAt) (*countReadSeeker, error) {
 	pos := int64(0)
-	return &countReader{r: r, cPos: &pos}, nil
+	return &countReadSeeker{r: r, cPos: &pos}, nil
 }
 
-type countReader struct {
+type countReadSeeker struct {
 	r    io.ReaderAt
 	cPos *int64
 
 	mu sync.Mutex
 }
 
-func (cr *countReader) Read(p []byte) (int, error) {
+func (cr *countReadSeeker) Read(p []byte) (int, error) {
 	cr.mu.Lock()
 	defer cr.mu.Unlock()
 
@@ -566,18 +621,18 @@ func (cr *countReader) Read(p []byte) (int, error) {
 	return n, err
 }
 
-func (cr *countReader) Seek(offset int64, whence int) (int64, error) {
+func (cr *countReadSeeker) Seek(offset int64, whence int) (int64, error) {
 	cr.mu.Lock()
 	defer cr.mu.Unlock()
 
 	switch whence {
 	default:
-		return 0, fmt.Errorf("Unknown whence: %v", whence)
+		return 0, fmt.Errorf("unknown whence: %v", whence)
 	case io.SeekStart:
 	case io.SeekCurrent:
 		offset += *cr.cPos
 	case io.SeekEnd:
-		return 0, fmt.Errorf("Unsupported whence: %v", whence)
+		return 0, fmt.Errorf("unsupported whence: %v", whence)
 	}
 
 	if offset < 0 {
@@ -587,9 +642,48 @@ func (cr *countReader) Seek(offset int64, whence int) (int64, error) {
 	return offset, nil
 }
 
-func (cr *countReader) currentPos() int64 {
+func (cr *countReadSeeker) currentPos() int64 {
 	cr.mu.Lock()
 	defer cr.mu.Unlock()
 
 	return *cr.cPos
 }
+
+func decompressBlob(org *io.SectionReader, tmp *tempFiles) (*io.SectionReader, error) {
+	if org.Size() < 4 {
+		return org, nil
+	}
+	src := make([]byte, 4)
+	if _, err := org.Read(src); err != nil && err != io.EOF {
+		return nil, err
+	}
+	var dR io.Reader
+	if bytes.Equal([]byte{0x1F, 0x8B, 0x08}, src[:3]) {
+		// gzip
+		dgR, err := gzip.NewReader(io.NewSectionReader(org, 0, org.Size()))
+		if err != nil {
+			return nil, err
+		}
+		defer dgR.Close()
+		dR = io.Reader(dgR)
+	} else if bytes.Equal([]byte{0x28, 0xb5, 0x2f, 0xfd}, src[:4]) {
+		// zstd
+		dzR, err := zstd.NewReader(io.NewSectionReader(org, 0, org.Size()))
+		if err != nil {
+			return nil, err
+		}
+		defer dzR.Close()
+		dR = io.Reader(dzR)
+	} else {
+		// uncompressed
+		return io.NewSectionReader(org, 0, org.Size()), nil
+	}
+	b, err := tmp.TempFile("", "uncompresseddata")
+	if err != nil {
+		return nil, err
+	}
+	if _, err := io.Copy(b, dR); err != nil {
+		return nil, err
+	}
+	return fileSectionReader(b)
+}

estargz/errorutil/errors_test.go

@@ -17,9 +17,8 @@
 package errorutil
 
 import (
+	"errors"
 	"testing"
-
-	"github.com/pkg/errors"
 )
 
 func TestNoError(t *testing.T) {

estargz/estargz_test.go

@@ -22,758 +22,7 @@
 
 package estargz
 
-import (
-	"archive/tar"
-	"bytes"
-	"compress/gzip"
-	"crypto/sha256"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"reflect"
-	"sort"
-	"strings"
-	"testing"
-)
-
-var allowedPrefix = [4]string{"", "./", "/", "../"}
-
-var compressionLevels = [5]int{
-	gzip.NoCompression,
-	gzip.BestSpeed,
-	gzip.BestCompression,
-	gzip.DefaultCompression,
-	gzip.HuffmanOnly,
-}
-
-// Tests footer encoding, size, and parsing.
-func TestFooter(t *testing.T) {
-	for off := int64(0); off <= 200000; off += 1023 {
-		checkFooter(t, off)
-		checkLegacyFooter(t, off)
-	}
-}
-
-func checkFooter(t *testing.T, off int64) {
-	footer := footerBytes(off)
-	if len(footer) != FooterSize {
-		t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), FooterSize, footer)
-	}
-	got, size, err := parseFooter(footer)
-	if err != nil {
-		t.Fatalf("failed to parse footer for offset %d, footer: %x: err: %v",
-			off, footer, err)
-	}
-	if size != FooterSize {
-		t.Fatalf("invalid footer size %d; want %d", size, FooterSize)
-	}
-	if got != off {
-		t.Fatalf("ParseFooter(footerBytes(offset %d)) = %d; want %d", off, got, off)
-
-	}
-}
-
-func checkLegacyFooter(t *testing.T, off int64) {
-	footer := legacyFooterBytes(off)
-	if len(footer) != legacyFooterSize {
-		t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), legacyFooterSize, footer)
-	}
-	got, size, err := parseFooter(footer)
-	if err != nil {
-		t.Fatalf("failed to parse legacy footer for offset %d, footer: %x: err: %v",
-			off, footer, err)
-	}
-	if size != legacyFooterSize {
-		t.Fatalf("invalid legacy footer size %d; want %d", size, legacyFooterSize)
-	}
-	if got != off {
-		t.Fatalf("ParseFooter(legacyFooterBytes(offset %d)) = %d; want %d", off, got, off)
-
-	}
-}
-
-func legacyFooterBytes(tocOff int64) []byte {
-	buf := bytes.NewBuffer(make([]byte, 0, legacyFooterSize))
-	gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression)
-	gz.Header.Extra = []byte(fmt.Sprintf("%016xSTARGZ", tocOff))
-	gz.Close()
-	if buf.Len() != legacyFooterSize {
-		panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), legacyFooterSize))
-	}
-	return buf.Bytes()
-}
-
-func TestWriteAndOpen(t *testing.T) {
-	const content = "Some contents"
-	invalidUtf8 := "\xff\xfe\xfd"
-
-	xAttrFile := xAttr{"foo": "bar", "invalid-utf8": invalidUtf8}
-	sampleOwner := owner{uid: 50, gid: 100}
-
-	tests := []struct {
-		name      string
-		chunkSize int
-		in        []tarEntry
-		want      []stargzCheck
-		wantNumGz int // expected number of gzip streams
-	}{
-		{
-			name:      "empty",
-			in:        tarOf(),
-			wantNumGz: 2, // TOC + footer
-			want: checks(
-				numTOCEntries(0),
-			),
-		},
-		{
-			name: "1dir_1empty_file",
-			in: tarOf(
-				dir("foo/"),
-				file("foo/bar.txt", ""),
-			),
-			wantNumGz: 3, // dir, TOC, footer
-			want: checks(
-				numTOCEntries(2),
-				hasDir("foo/"),
-				hasFileLen("foo/bar.txt", 0),
-				entryHasChildren("foo", "bar.txt"),
-				hasFileDigest("foo/bar.txt", digestFor("")),
-			),
-		},
-		{
-			name: "1dir_1file",
-			in: tarOf(
-				dir("foo/"),
-				file("foo/bar.txt", content, xAttrFile),
-			),
-			wantNumGz: 4, // var dir, foo.txt alone, TOC, footer
-			want: checks(
-				numTOCEntries(2),
-				hasDir("foo/"),
-				hasFileLen("foo/bar.txt", len(content)),
-				hasFileDigest("foo/bar.txt", digestFor(content)),
-				hasFileContentsRange("foo/bar.txt", 0, content),
-				hasFileContentsRange("foo/bar.txt", 1, content[1:]),
-				entryHasChildren("", "foo"),
-				entryHasChildren("foo", "bar.txt"),
-				hasFileXattrs("foo/bar.txt", "foo", "bar"),
-				hasFileXattrs("foo/bar.txt", "invalid-utf8", invalidUtf8),
-			),
-		},
-		{
-			name: "2meta_2file",
-			in: tarOf(
-				dir("bar/", sampleOwner),
-				dir("foo/", sampleOwner),
-				file("foo/bar.txt", content, sampleOwner),
-			),
-			wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer
-			want: checks(
-				numTOCEntries(3),
-				hasDir("bar/"),
-				hasDir("foo/"),
-				hasFileLen("foo/bar.txt", len(content)),
-				entryHasChildren("", "bar", "foo"),
-				entryHasChildren("foo", "bar.txt"),
-				hasChunkEntries("foo/bar.txt", 1),
-				hasEntryOwner("bar/", sampleOwner),
-				hasEntryOwner("foo/", sampleOwner),
-				hasEntryOwner("foo/bar.txt", sampleOwner),
-			),
-		},
-		{
-			name: "3dir",
-			in: tarOf(
-				dir("bar/"),
-				dir("foo/"),
-				dir("foo/bar/"),
-			),
-			wantNumGz: 3, // 3 dirs, TOC, footer
-			want: checks(
-				hasDirLinkCount("bar/", 2),
-				hasDirLinkCount("foo/", 3),
-				hasDirLinkCount("foo/bar/", 2),
-			),
-		},
-		{
-			name: "symlink",
-			in: tarOf(
-				dir("foo/"),
-				symlink("foo/bar", "../../x"),
-			),
-			wantNumGz: 3, // metas + TOC + footer
-			want: checks(
-				numTOCEntries(2),
-				hasSymlink("foo/bar", "../../x"),
-				entryHasChildren("", "foo"),
-				entryHasChildren("foo", "bar"),
-			),
-		},
-		{
-			name:      "chunked_file",
-			chunkSize: 4,
-			in: tarOf(
-				dir("foo/"),
-				file("foo/big.txt", "This "+"is s"+"uch "+"a bi"+"g fi"+"le"),
-			),
-			wantNumGz: 9,
-			want: checks(
-				numTOCEntries(7), // 1 for foo dir, 6 for the foo/big.txt file
-				hasDir("foo/"),
-				hasFileLen("foo/big.txt", len("This is such a big file")),
-				hasFileDigest("foo/big.txt", digestFor("This is such a big file")),
-				hasFileContentsRange("foo/big.txt", 0, "This is such a big file"),
-				hasFileContentsRange("foo/big.txt", 1, "his is such a big file"),
-				hasFileContentsRange("foo/big.txt", 2, "is is such a big file"),
-				hasFileContentsRange("foo/big.txt", 3, "s is such a big file"),
-				hasFileContentsRange("foo/big.txt", 4, " is such a big file"),
-				hasFileContentsRange("foo/big.txt", 5, "is such a big file"),
-				hasFileContentsRange("foo/big.txt", 6, "s such a big file"),
-				hasFileContentsRange("foo/big.txt", 7, " such a big file"),
-				hasFileContentsRange("foo/big.txt", 8, "such a big file"),
-				hasFileContentsRange("foo/big.txt", 9, "uch a big file"),
-				hasFileContentsRange("foo/big.txt", 10, "ch a big file"),
-				hasFileContentsRange("foo/big.txt", 11, "h a big file"),
-				hasFileContentsRange("foo/big.txt", 12, " a big file"),
-				hasFileContentsRange("foo/big.txt", len("This is such a big file")-1, ""),
-				hasChunkEntries("foo/big.txt", 6),
-			),
-		},
-		{
-			name: "recursive",
-			in: tarOf(
-				dir("/", sampleOwner),
-				dir("bar/", sampleOwner),
-				dir("foo/", sampleOwner),
-				file("foo/bar.txt", content, sampleOwner),
-			),
-			wantNumGz: 4, // dirs, bar.txt alone, TOC, footer
-			want: checks(
-				maxDepth(2), // 0: root directory, 1: "foo/", 2: "bar.txt"
-			),
-		},
-		{
-			name: "block_char_fifo",
-			in: tarOf(
-				tarEntryFunc(func(w *tar.Writer, prefix string) error {
-					return w.WriteHeader(&tar.Header{
-						Name:     prefix + "b",
-						Typeflag: tar.TypeBlock,
-						Devmajor: 123,
-						Devminor: 456,
-					})
-				}),
-				tarEntryFunc(func(w *tar.Writer, prefix string) error {
-					return w.WriteHeader(&tar.Header{
-						Name:     prefix + "c",
-						Typeflag: tar.TypeChar,
-						Devmajor: 111,
-						Devminor: 222,
-					})
-				}),
-				tarEntryFunc(func(w *tar.Writer, prefix string) error {
-					return w.WriteHeader(&tar.Header{
-						Name:     prefix + "f",
-						Typeflag: tar.TypeFifo,
-					})
-				}),
-			),
-			wantNumGz: 3,
-			want: checks(
-				lookupMatch("b", &TOCEntry{Name: "b", Type: "block", DevMajor: 123, DevMinor: 456, NumLink: 1}),
-				lookupMatch("c", &TOCEntry{Name: "c", Type: "char", DevMajor: 111, DevMinor: 222, NumLink: 1}),
-				lookupMatch("f", &TOCEntry{Name: "f", Type: "fifo", NumLink: 1}),
-			),
-		},
-	}
-
-	for _, tt := range tests {
-		for _, cl := range compressionLevels {
-			cl := cl
-			for _, prefix := range allowedPrefix {
-				prefix := prefix
-				t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) {
-					tr, cancel := buildTar(t, tt.in, prefix)
-					defer cancel()
-					var stargzBuf bytes.Buffer
-					w := NewWriterLevel(&stargzBuf, cl)
-					w.ChunkSize = tt.chunkSize
-					if err := w.AppendTar(tr); err != nil {
-						t.Fatalf("Append: %v", err)
-					}
-					if _, err := w.Close(); err != nil {
-						t.Fatalf("Writer.Close: %v", err)
-					}
-					b := stargzBuf.Bytes()
-
-					diffID := w.DiffID()
-					wantDiffID := diffIDOfGz(t, b)
-					if diffID != wantDiffID {
-						t.Errorf("DiffID = %q; want %q", diffID, wantDiffID)
-					}
-
-					got := countGzStreams(t, b)
-					if got != tt.wantNumGz {
-						t.Errorf("number of gzip streams = %d; want %d", got, tt.wantNumGz)
-					}
-
-					r, err := Open(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))))
-					if err != nil {
-						t.Fatalf("stargz.Open: %v", err)
-					}
-					for _, want := range tt.want {
-						want.check(t, r)
-					}
-
-				})
-			}
-		}
-	}
-}
-
-func diffIDOfGz(t *testing.T, b []byte) string {
-	h := sha256.New()
-	zr, err := gzip.NewReader(bytes.NewReader(b))
-	if err != nil {
-		t.Fatalf("diffIDOfGz: %v", err)
-	}
-	if _, err := io.Copy(h, zr); err != nil {
-		t.Fatalf("diffIDOfGz.Copy: %v", err)
-	}
-	return fmt.Sprintf("sha256:%x", h.Sum(nil))
-}
-
-func countGzStreams(t *testing.T, b []byte) (numStreams int) {
-	len0 := len(b)
-	br := bytes.NewReader(b)
-	zr := new(gzip.Reader)
-	t.Logf("got gzip streams:")
-	for {
-		zoff := len0 - br.Len()
-		if err := zr.Reset(br); err != nil {
-			if err == io.EOF {
-				return
-			}
-			t.Fatalf("countGzStreams, Reset: %v", err)
-		}
-		zr.Multistream(false)
-		n, err := io.Copy(ioutil.Discard, zr)
-		if err != nil {
-			t.Fatalf("countGzStreams, Copy: %v", err)
-		}
-		var extra string
-		if len(zr.Header.Extra) > 0 {
-			extra = fmt.Sprintf("; extra=%q", zr.Header.Extra)
-		}
-		t.Logf("  [%d] at %d in stargz, uncompressed length %d%s", numStreams, zoff, n, extra)
-		numStreams++
-	}
-}
-
-func digestFor(content string) string {
-	sum := sha256.Sum256([]byte(content))
-	return fmt.Sprintf("sha256:%x", sum)
-}
-
-type numTOCEntries int
-
-func (n numTOCEntries) check(t *testing.T, r *Reader) {
-	if r.toc == nil {
-		t.Fatal("nil TOC")
-	}
-	if got, want := len(r.toc.Entries), int(n); got != want {
-		t.Errorf("got %d TOC entries; want %d", got, want)
-	}
-	t.Logf("got TOC entries:")
-	for i, ent := range r.toc.Entries {
-		entj, _ := json.Marshal(ent)
-		t.Logf("  [%d]: %s\n", i, entj)
-	}
-	if t.Failed() {
-		t.FailNow()
-	}
-}
-
-func tarOf(s ...tarEntry) []tarEntry { return s }
-
-func checks(s ...stargzCheck) []stargzCheck { return s }
-
-type stargzCheck interface {
-	check(t *testing.T, r *Reader)
-}
-
-type stargzCheckFn func(*testing.T, *Reader)
-
-func (f stargzCheckFn) check(t *testing.T, r *Reader) { f(t, r) }
-
-func maxDepth(max int) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		e, ok := r.Lookup("")
-		if !ok {
-			t.Fatal("root directory not found")
-		}
-		d, err := getMaxDepth(t, e, 0, 10*max)
-		if err != nil {
-			t.Errorf("failed to get max depth (wanted %d): %v", max, err)
-			return
-		}
-		if d != max {
-			t.Errorf("invalid depth %d; want %d", d, max)
-			return
-		}
-	})
-}
-
-func getMaxDepth(t *testing.T, e *TOCEntry, current, limit int) (max int, rErr error) {
-	if current > limit {
-		return -1, fmt.Errorf("walkMaxDepth: exceeds limit: current:%d > limit:%d",
-			current, limit)
-	}
-	max = current
-	e.ForeachChild(func(baseName string, ent *TOCEntry) bool {
-		t.Logf("%q(basename:%q) is child of %q\n", ent.Name, baseName, e.Name)
-		d, err := getMaxDepth(t, ent, current+1, limit)
-		if err != nil {
-			rErr = err
-			return false
-		}
-		if d > max {
-			max = d
-		}
-		return true
-	})
-	return
-}
-
-func hasFileLen(file string, wantLen int) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		for _, ent := range r.toc.Entries {
-			if ent.Name == file {
-				if ent.Type != "reg" {
-					t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type)
-				} else if ent.Size != int64(wantLen) {
-					t.Errorf("file size of %q = %d; want %d", file, ent.Size, wantLen)
-				}
-				return
-			}
-		}
-		t.Errorf("file %q not found", file)
-	})
-}
-
-func hasFileXattrs(file, name, value string) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		for _, ent := range r.toc.Entries {
-			if ent.Name == file {
-				if ent.Type != "reg" {
-					t.Errorf("file type of %q is %q; want \"reg\"", file, ent.Type)
-				}
-				if ent.Xattrs == nil {
-					t.Errorf("file %q has no xattrs", file)
-					return
-				}
-				valueFound, found := ent.Xattrs[name]
-				if !found {
-					t.Errorf("file %q has no xattr %q", file, name)
-					return
-				}
-				if string(valueFound) != value {
-					t.Errorf("file %q has xattr %q with value %q instead of %q", file, name, valueFound, value)
-				}
-
-				return
-			}
-		}
-		t.Errorf("file %q not found", file)
-	})
-}
-
-func hasFileDigest(file string, digest string) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		ent, ok := r.Lookup(file)
-		if !ok {
-			t.Fatalf("didn't find TOCEntry for file %q", file)
-		}
-		if ent.Digest != digest {
-			t.Fatalf("Digest(%q) = %q, want %q", file, ent.Digest, digest)
-		}
-	})
-}
-
-func hasFileContentsRange(file string, offset int, want string) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		f, err := r.OpenFile(file)
-		if err != nil {
-			t.Fatal(err)
-		}
-		got := make([]byte, len(want))
-		n, err := f.ReadAt(got, int64(offset))
-		if err != nil {
-			t.Fatalf("ReadAt(len %d, offset %d) = %v, %v", len(got), offset, n, err)
-		}
-		if string(got) != want {
-			t.Fatalf("ReadAt(len %d, offset %d) = %q, want %q", len(got), offset, got, want)
-		}
-	})
-}
-
-func hasChunkEntries(file string, wantChunks int) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		ent, ok := r.Lookup(file)
-		if !ok {
-			t.Fatalf("no file for %q", file)
-		}
-		if ent.Type != "reg" {
-			t.Fatalf("file %q has unexpected type %q; want reg", file, ent.Type)
-		}
-		chunks := r.getChunks(ent)
-		if len(chunks) != wantChunks {
-			t.Errorf("len(r.getChunks(%q)) = %d; want %d", file, len(chunks), wantChunks)
-			return
-		}
-		f := chunks[0]
-
-		var gotChunks []*TOCEntry
-		var last *TOCEntry
-		for off := int64(0); off < f.Size; off++ {
-			e, ok := r.ChunkEntryForOffset(file, off)
-			if !ok {
-				t.Errorf("no ChunkEntryForOffset at %d", off)
-				return
-			}
-			if last != e {
-				gotChunks = append(gotChunks, e)
-				last = e
-			}
-		}
-		if !reflect.DeepEqual(chunks, gotChunks) {
-			t.Errorf("gotChunks=%d, want=%d; contents mismatch", len(gotChunks), wantChunks)
-		}
-
-		// And verify the NextOffset
-		for i := 0; i < len(gotChunks)-1; i++ {
-			ci := gotChunks[i]
-			cnext := gotChunks[i+1]
-			if ci.NextOffset() != cnext.Offset {
-				t.Errorf("chunk %d NextOffset %d != next chunk's Offset of %d", i, ci.NextOffset(), cnext.Offset)
-			}
-		}
-	})
-}
-
-func entryHasChildren(dir string, want ...string) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		want := append([]string(nil), want...)
-		var got []string
-		ent, ok := r.Lookup(dir)
-		if !ok {
-			t.Fatalf("didn't find TOCEntry for dir node %q", dir)
-		}
-		for baseName := range ent.children {
-			got = append(got, baseName)
-		}
-		sort.Strings(got)
-		sort.Strings(want)
-		if !reflect.DeepEqual(got, want) {
-			t.Errorf("children of %q = %q; want %q", dir, got, want)
-		}
-	})
-}
-
-func hasDir(file string) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		for _, ent := range r.toc.Entries {
-			if ent.Name == cleanEntryName(file) {
-				if ent.Type != "dir" {
-					t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type)
-				}
-				return
-			}
-		}
-		t.Errorf("directory %q not found", file)
-	})
-}
-
-func hasDirLinkCount(file string, count int) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		for _, ent := range r.toc.Entries {
-			if ent.Name == cleanEntryName(file) {
-				if ent.Type != "dir" {
-					t.Errorf("file type of %q is %q; want \"dir\"", file, ent.Type)
-					return
-				}
-				if ent.NumLink != count {
-					t.Errorf("link count of %q = %d; want %d", file, ent.NumLink, count)
-				}
-				return
-			}
-		}
-		t.Errorf("directory %q not found", file)
-	})
-}
-
-func hasSymlink(file, target string) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		for _, ent := range r.toc.Entries {
-			if ent.Name == file {
-				if ent.Type != "symlink" {
-					t.Errorf("file type of %q is %q; want \"symlink\"", file, ent.Type)
-				} else if ent.LinkName != target {
-					t.Errorf("link target of symlink %q is %q; want %q", file, ent.LinkName, target)
-				}
-				return
-			}
-		}
-		t.Errorf("symlink %q not found", file)
-	})
-}
-
-func lookupMatch(name string, want *TOCEntry) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		e, ok := r.Lookup(name)
-		if !ok {
-			t.Fatalf("failed to Lookup entry %q", name)
-		}
-		if !reflect.DeepEqual(e, want) {
-			t.Errorf("entry %q mismatch.\n got: %+v\nwant: %+v\n", name, e, want)
-		}
-
-	})
-}
-
-func hasEntryOwner(entry string, owner owner) stargzCheck {
-	return stargzCheckFn(func(t *testing.T, r *Reader) {
-		ent, ok := r.Lookup(strings.TrimSuffix(entry, "/"))
-		if !ok {
-			t.Errorf("entry %q not found", entry)
-			return
-		}
-		if ent.UID != owner.uid || ent.GID != owner.gid {
-			t.Errorf("entry %q has invalid owner (uid:%d, gid:%d) instead of (uid:%d, gid:%d)", entry, ent.UID, ent.GID, owner.uid, owner.gid)
-			return
-		}
-	})
-}
-
-type tarEntry interface {
-	appendTar(tw *tar.Writer, prefix string) error
-}
-
-type tarEntryFunc func(*tar.Writer, string) error
-
-func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string) error { return f(tw, prefix) }
-
-func buildTar(t *testing.T, ents []tarEntry, prefix string) (r io.Reader, cancel func()) {
-	pr, pw := io.Pipe()
-	go func() {
-		tw := tar.NewWriter(pw)
-		for _, ent := range ents {
-			if err := ent.appendTar(tw, prefix); err != nil {
-				t.Errorf("building input tar: %v", err)
-				pw.Close()
-				return
-			}
-		}
-		if err := tw.Close(); err != nil {
-			t.Errorf("closing write of input tar: %v", err)
-		}
-		pw.Close()
-	}()
-	return pr, func() { go pr.Close(); go pw.Close() }
-}
-
-func buildTarStatic(t *testing.T, ents []tarEntry, prefix string) *io.SectionReader {
-	buf := new(bytes.Buffer)
-	tw := tar.NewWriter(buf)
-	for _, ent := range ents {
-		if err := ent.appendTar(tw, prefix); err != nil {
-			t.Fatalf("building input tar: %v", err)
-		}
-	}
-	if err := tw.Close(); err != nil {
-		t.Errorf("closing write of input tar: %v", err)
-	}
-	data := buf.Bytes()
-	return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data)))
-}
-
-func dir(name string, opts ...interface{}) tarEntry {
-	return tarEntryFunc(func(tw *tar.Writer, prefix string) error {
-		var o owner
-		for _, opt := range opts {
-			if v, ok := opt.(owner); ok {
-				o = v
-			} else {
-				return errors.New("unsupported opt")
-			}
-		}
-		if !strings.HasSuffix(name, "/") {
-			panic(fmt.Sprintf("missing trailing slash in dir %q ", name))
-		}
-		return tw.WriteHeader(&tar.Header{
-			Typeflag: tar.TypeDir,
-			Name:     prefix + name,
-			Mode:     0755,
-			Uid:      o.uid,
-			Gid:      o.gid,
-		})
-	})
-}
-
-// xAttr are extended attributes to set on test files created with the file func.
-type xAttr map[string]string
-
-// owner is owner ot set on test files and directories with the file and dir functions.
-type owner struct {
-	uid int
-	gid int
-}
-
-func file(name, contents string, opts ...interface{}) tarEntry {
-	return tarEntryFunc(func(tw *tar.Writer, prefix string) error {
-		var xattrs xAttr
-		var o owner
-		for _, opt := range opts {
-			switch v := opt.(type) {
-			case xAttr:
-				xattrs = v
-			case owner:
-				o = v
-			default:
-				return errors.New("unsupported opt")
-			}
-		}
-		if strings.HasSuffix(name, "/") {
-			return fmt.Errorf("bogus trailing slash in file %q", name)
-		}
-		if err := tw.WriteHeader(&tar.Header{
-			Typeflag: tar.TypeReg,
-			Name:     prefix + name,
-			Mode:     0644,
-			Xattrs:   xattrs,
-			Size:     int64(len(contents)),
-			Uid:      o.uid,
-			Gid:      o.gid,
-		}); err != nil {
-			return err
-		}
-		_, err := io.WriteString(tw, contents)
-		return err
-	})
-}
-
-func symlink(name, target string) tarEntry {
-	return tarEntryFunc(func(tw *tar.Writer, prefix string) error {
-		return tw.WriteHeader(&tar.Header{
-			Typeflag: tar.TypeSymlink,
-			Name:     prefix + name,
-			Linkname: target,
-			Mode:     0644,
-		})
-	})
-}
+import "testing"
 
 // Tests *Reader.ChunkEntryForOffset about offset and size calculation.
 func TestChunkEntryForOffset(t *testing.T) {
@@ -832,7 +81,7 @@ func TestChunkEntryForOffset(t *testing.T) {
 			if ok != te.wantOk {
 				t.Errorf("ok = %v; want (%v)", ok, te.wantOk)
 			} else if ok {
-				if !(ce.ChunkOffset == te.wantChunkOffset && ce.ChunkSize == te.wantChunkSize) {
+				if ce.ChunkOffset != te.wantChunkOffset || ce.ChunkSize != te.wantChunkSize {
 					t.Errorf("chunkOffset = %d, ChunkSize = %d; want (chunkOffset = %d, chunkSize = %d)",
 						ce.ChunkOffset, ce.ChunkSize, te.wantChunkOffset, te.wantChunkSize)
 				}

estargz/externaltoc/externaltoc.go

@@ -0,0 +1,278 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright 2019 The Go Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style
+   license that can be found in the LICENSE file.
+*/
+
+package externaltoc
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"hash"
+	"io"
+	"sync"
+
+	"github.com/containerd/stargz-snapshotter/estargz"
+	digest "github.com/opencontainers/go-digest"
+)
+
+type GzipCompression struct {
+	*GzipCompressor
+	*GzipDecompressor
+}
+
+func NewGzipCompressionWithLevel(provideTOC func() ([]byte, error), level int) estargz.Compression {
+	return &GzipCompression{
+		NewGzipCompressorWithLevel(level),
+		NewGzipDecompressor(provideTOC),
+	}
+}
+
+func NewGzipCompressor() *GzipCompressor {
+	return &GzipCompressor{compressionLevel: gzip.BestCompression}
+}
+
+func NewGzipCompressorWithLevel(level int) *GzipCompressor {
+	return &GzipCompressor{compressionLevel: level}
+}
+
+type GzipCompressor struct {
+	compressionLevel int
+	buf              *bytes.Buffer
+}
+
+func (gc *GzipCompressor) WriteTOCTo(w io.Writer) (int, error) {
+	if len(gc.buf.Bytes()) == 0 {
+		return 0, fmt.Errorf("TOC hasn't been registered")
+	}
+	return w.Write(gc.buf.Bytes())
+}
+
+func (gc *GzipCompressor) Writer(w io.Writer) (estargz.WriteFlushCloser, error) {
+	return gzip.NewWriterLevel(w, gc.compressionLevel)
+}
+
+func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *estargz.JTOC, diffHash hash.Hash) (digest.Digest, error) {
+	tocJSON, err := json.MarshalIndent(toc, "", "\t")
+	if err != nil {
+		return "", err
+	}
+	buf := new(bytes.Buffer)
+	gz, _ := gzip.NewWriterLevel(buf, gc.compressionLevel)
+	// TOC isn't written to layer so no effect to diff ID
+	tw := tar.NewWriter(gz)
+	if err := tw.WriteHeader(&tar.Header{
+		Typeflag: tar.TypeReg,
+		Name:     estargz.TOCTarName,
+		Size:     int64(len(tocJSON)),
+	}); err != nil {
+		return "", err
+	}
+	if _, err := tw.Write(tocJSON); err != nil {
+		return "", err
+	}
+
+	if err := tw.Close(); err != nil {
+		return "", err
+	}
+	if err := gz.Close(); err != nil {
+		return "", err
+	}
+	gc.buf = buf
+	footerBytes, err := gzipFooterBytes()
+	if err != nil {
+		return "", err
+	}
+	if _, err := w.Write(footerBytes); err != nil {
+		return "", err
+	}
+	return digest.FromBytes(tocJSON), nil
+}
+
+// The footer is an empty gzip stream with no compression and an Extra header.
+//
+// 46 comes from:
+//
+// 10 bytes  gzip header
+// 2  bytes  XLEN (length of Extra field) = 21 (4 bytes header + len("STARGZEXTERNALTOC"))
+// 2  bytes  Extra: SI1 = 'S', SI2 = 'G'
+// 2  bytes  Extra: LEN = 17 (len("STARGZEXTERNALTOC"))
+// 17 bytes  Extra: subfield = "STARGZEXTERNALTOC"
+// 5  bytes  flate header
+// 8  bytes  gzip footer
+// (End of the eStargz blob)
+const FooterSize = 46
+
+// gzipFooterBytes returns the 104 bytes footer.
+func gzipFooterBytes() ([]byte, error) {
+	buf := bytes.NewBuffer(make([]byte, 0, FooterSize))
+	gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes
+
+	// Extra header indicating the offset of TOCJSON
+	// https://tools.ietf.org/html/rfc1952#section-2.3.1.1
+	header := make([]byte, 4)
+	header[0], header[1] = 'S', 'G'
+	subfield := "STARGZEXTERNALTOC"                                   // len("STARGZEXTERNALTOC") = 17
+	binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952
+	gz.Extra = append(header, []byte(subfield)...)
+	if err := gz.Close(); err != nil {
+		return nil, err
+	}
+	if buf.Len() != FooterSize {
+		panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize))
+	}
+	return buf.Bytes(), nil
+}
+
+func NewGzipDecompressor(provideTOCFunc func() ([]byte, error)) *GzipDecompressor {
+	return &GzipDecompressor{provideTOCFunc: provideTOCFunc}
+}
+
+type GzipDecompressor struct {
+	provideTOCFunc func() ([]byte, error)
+	rawTOC         []byte // Do not access this field directly. Get this through getTOC() method.
+	getTOCOnce     sync.Once
+}
+
+func (gz *GzipDecompressor) getTOC() ([]byte, error) {
+	if len(gz.rawTOC) == 0 {
+		var retErr error
+		gz.getTOCOnce.Do(func() {
+			if gz.provideTOCFunc == nil {
+				retErr = fmt.Errorf("TOC hasn't been provided")
+				return
+			}
+			rawTOC, err := gz.provideTOCFunc()
+			if err != nil {
+				retErr = err
+				return
+			}
+			gz.rawTOC = rawTOC
+		})
+		if retErr != nil {
+			return nil, retErr
+		}
+		if len(gz.rawTOC) == 0 {
+			return nil, fmt.Errorf("no TOC is provided")
+		}
+	}
+	return gz.rawTOC, nil
+}
+
+func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
+	return gzip.NewReader(r)
+}
+
+func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *estargz.JTOC, tocDgst digest.Digest, err error) {
+	if r != nil {
+		return nil, "", fmt.Errorf("TOC must be provided externally but got internal one")
+	}
+	rawTOC, err := gz.getTOC()
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to get TOC: %v", err)
+	}
+	return parseTOCEStargz(bytes.NewReader(rawTOC))
+}
+
+func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
+	if len(p) != FooterSize {
+		return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p))
+	}
+	zr, err := gzip.NewReader(bytes.NewReader(p))
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	defer zr.Close()
+	extra := zr.Extra
+	si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
+	if si1 != 'S' || si2 != 'G' {
+		return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
+	}
+	if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(len("STARGZEXTERNALTOC")) {
+		return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ"))
+	}
+	if string(subfield) != "STARGZEXTERNALTOC" {
+		return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield")
+	}
+	// tocOffset < 0 indicates external TOC.
+	// blobPayloadSize < 0 indicates the entire blob size.
+	return -1, -1, 0, nil
+}
+
+func (gz *GzipDecompressor) FooterSize() int64 {
+	return FooterSize
+}
+
+func (gz *GzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
+	if r != nil {
+		return nil, fmt.Errorf("TOC must be provided externally but got internal one")
+	}
+	rawTOC, err := gz.getTOC()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get TOC: %v", err)
+	}
+	return decompressTOCEStargz(bytes.NewReader(rawTOC))
+}
+
+func parseTOCEStargz(r io.Reader) (toc *estargz.JTOC, tocDgst digest.Digest, err error) {
+	tr, err := decompressTOCEStargz(r)
+	if err != nil {
+		return nil, "", err
+	}
+	dgstr := digest.Canonical.Digester()
+	toc = new(estargz.JTOC)
+	if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil {
+		return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err)
+	}
+	if err := tr.Close(); err != nil {
+		return nil, "", err
+	}
+	return toc, dgstr.Digest(), nil
+}
+
+func decompressTOCEStargz(r io.Reader) (tocJSON io.ReadCloser, err error) {
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("malformed TOC gzip header: %v", err)
+	}
+	zr.Multistream(false)
+	tr := tar.NewReader(zr)
+	h, err := tr.Next()
+	if err != nil {
+		return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err)
+	}
+	if h.Name != estargz.TOCTarName {
+		return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, estargz.TOCTarName)
+	}
+	return readCloser{tr, zr.Close}, nil
+}
+
+type readCloser struct {
+	io.Reader
+	closeFunc func() error
+}
+
+func (rc readCloser) Close() error {
+	return rc.closeFunc()
+}

estargz/externaltoc/externaltoc_test.go

@@ -0,0 +1,102 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package externaltoc
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"testing"
+
+	"github.com/containerd/stargz-snapshotter/estargz"
+)
+
+// TestGzipEStargz tests gzip-based external TOC eStargz
+func TestGzipEStargz(t *testing.T) {
+	testRunner := &estargz.TestRunner{
+		TestingT: t,
+		Runner: func(testingT estargz.TestingT, name string, run func(t estargz.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	estargz.CompressionTestSuite(testRunner,
+		gzipControllerWithLevel(gzip.NoCompression),
+		gzipControllerWithLevel(gzip.BestSpeed),
+		gzipControllerWithLevel(gzip.BestCompression),
+		gzipControllerWithLevel(gzip.DefaultCompression),
+		gzipControllerWithLevel(gzip.HuffmanOnly),
+	)
+}
+
+func gzipControllerWithLevel(compressionLevel int) estargz.TestingControllerFactory {
+	return func() estargz.TestingController {
+		compressor := NewGzipCompressorWithLevel(compressionLevel)
+		decompressor := NewGzipDecompressor(func() ([]byte, error) {
+			buf := new(bytes.Buffer)
+			if _, err := compressor.WriteTOCTo(buf); err != nil {
+				return nil, err
+			}
+			return buf.Bytes(), nil
+		})
+		return &gzipController{compressor, decompressor}
+	}
+}
+
+type gzipController struct {
+	*GzipCompressor
+	*GzipDecompressor
+}
+
+func (gc *gzipController) String() string {
+	return fmt.Sprintf("externaltoc_gzip_compression_level=%v", gc.compressionLevel)
+}
+
+// TestStream tests the passed estargz blob contains the specified list of streams.
+func (gc *gzipController) TestStreams(t estargz.TestingT, b []byte, streams []int64) {
+	estargz.CheckGzipHasStreams(t, b, streams)
+}
+
+func (gc *gzipController) DiffIDOf(t estargz.TestingT, b []byte) string {
+	return estargz.GzipDiffIDOf(t, b)
+}
+
+// Tests footer encoding, size, and parsing of gzip-based eStargz.
+func TestGzipFooter(t *testing.T) {
+	footer, err := gzipFooterBytes()
+	if err != nil {
+		t.Fatalf("failed gzipFooterBytes: %v", err)
+	}
+	if len(footer) != FooterSize {
+		t.Fatalf("footer length was %d, not expected %d. got bytes: %q", len(footer), FooterSize, footer)
+	}
+	_, gotTOCOffset, _, err := (&GzipDecompressor{}).ParseFooter(footer)
+	if err != nil {
+		t.Fatalf("failed to parse footer, footer: %x: err: %v", footer, err)
+	}
+	if gotTOCOffset != -1 {
+		t.Fatalf("ParseFooter(footerBytes) must return -1 for external toc but got %d", gotTOCOffset)
+	}
+}

estargz/go.mod

@@ -1,9 +1,10 @@
 module github.com/containerd/stargz-snapshotter/estargz
 
-go 1.13
+go 1.23.0
 
 require (
+	github.com/klauspost/compress v1.18.0
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/pkg/errors v0.9.1
-	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
+	github.com/vbatts/tar-split v0.12.1
+	golang.org/x/sync v0.16.0
 )

estargz/go.sum

@@ -1,6 +1,8 @@
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo=
+github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=

estargz/gzip.go

@@ -0,0 +1,237 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright 2019 The Go Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style
+   license that can be found in the LICENSE file.
+*/
+
+package estargz
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"hash"
+	"io"
+	"strconv"
+
+	digest "github.com/opencontainers/go-digest"
+)
+
+type gzipCompression struct {
+	*GzipCompressor
+	*GzipDecompressor
+}
+
+func newGzipCompressionWithLevel(level int) Compression {
+	return &gzipCompression{
+		&GzipCompressor{level},
+		&GzipDecompressor{},
+	}
+}
+
+func NewGzipCompressor() *GzipCompressor {
+	return &GzipCompressor{gzip.BestCompression}
+}
+
+func NewGzipCompressorWithLevel(level int) *GzipCompressor {
+	return &GzipCompressor{level}
+}
+
+type GzipCompressor struct {
+	compressionLevel int
+}
+
+func (gc *GzipCompressor) Writer(w io.Writer) (WriteFlushCloser, error) {
+	return gzip.NewWriterLevel(w, gc.compressionLevel)
+}
+
+func (gc *GzipCompressor) WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (digest.Digest, error) {
+	tocJSON, err := json.MarshalIndent(toc, "", "\t")
+	if err != nil {
+		return "", err
+	}
+	gz, _ := gzip.NewWriterLevel(w, gc.compressionLevel)
+	gw := io.Writer(gz)
+	if diffHash != nil {
+		gw = io.MultiWriter(gz, diffHash)
+	}
+	tw := tar.NewWriter(gw)
+	if err := tw.WriteHeader(&tar.Header{
+		Typeflag: tar.TypeReg,
+		Name:     TOCTarName,
+		Size:     int64(len(tocJSON)),
+	}); err != nil {
+		return "", err
+	}
+	if _, err := tw.Write(tocJSON); err != nil {
+		return "", err
+	}
+
+	if err := tw.Close(); err != nil {
+		return "", err
+	}
+	if err := gz.Close(); err != nil {
+		return "", err
+	}
+	if _, err := w.Write(gzipFooterBytes(off)); err != nil {
+		return "", err
+	}
+	return digest.FromBytes(tocJSON), nil
+}
+
+// gzipFooterBytes returns the 51 bytes footer.
+func gzipFooterBytes(tocOff int64) []byte {
+	buf := bytes.NewBuffer(make([]byte, 0, FooterSize))
+	gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) // MUST be NoCompression to keep 51 bytes
+
+	// Extra header indicating the offset of TOCJSON
+	// https://tools.ietf.org/html/rfc1952#section-2.3.1.1
+	header := make([]byte, 4)
+	header[0], header[1] = 'S', 'G'
+	subfield := fmt.Sprintf("%016xSTARGZ", tocOff)
+	binary.LittleEndian.PutUint16(header[2:4], uint16(len(subfield))) // little-endian per RFC1952
+	gz.Extra = append(header, []byte(subfield)...)
+	gz.Close()
+	if buf.Len() != FooterSize {
+		panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), FooterSize))
+	}
+	return buf.Bytes()
+}
+
+type GzipDecompressor struct{}
+
+func (gz *GzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
+	return gzip.NewReader(r)
+}
+
+func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
+	return parseTOCEStargz(r)
+}
+
+func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
+	if len(p) != FooterSize {
+		return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p))
+	}
+	zr, err := gzip.NewReader(bytes.NewReader(p))
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	defer zr.Close()
+	extra := zr.Extra
+	si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:]
+	if si1 != 'S' || si2 != 'G' {
+		return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2)
+	}
+	if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) {
+		return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ"))
+	}
+	if string(subfield[16:]) != "STARGZ" {
+		return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield")
+	}
+	tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64)
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err)
+	}
+	return tocOffset, tocOffset, 0, nil
+}
+
+func (gz *GzipDecompressor) FooterSize() int64 {
+	return FooterSize
+}
+
+func (gz *GzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
+	return decompressTOCEStargz(r)
+}
+
+type LegacyGzipDecompressor struct{}
+
+func (gz *LegacyGzipDecompressor) Reader(r io.Reader) (io.ReadCloser, error) {
+	return gzip.NewReader(r)
+}
+
+func (gz *LegacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
+	return parseTOCEStargz(r)
+}
+
+func (gz *LegacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
+	if len(p) != legacyFooterSize {
+		return 0, 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p))
+	}
+	zr, err := gzip.NewReader(bytes.NewReader(p))
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("legacy: failed to get footer gzip reader: %w", err)
+	}
+	defer zr.Close()
+	extra := zr.Extra
+	if len(extra) != 16+len("STARGZ") {
+		return 0, 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size")
+	}
+	if string(extra[16:]) != "STARGZ" {
+		return 0, 0, 0, fmt.Errorf("legacy: magic string STARGZ not found")
+	}
+	tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64)
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("legacy: failed to parse toc offset: %w", err)
+	}
+	return tocOffset, tocOffset, 0, nil
+}
+
+func (gz *LegacyGzipDecompressor) FooterSize() int64 {
+	return legacyFooterSize
+}
+
+func (gz *LegacyGzipDecompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
+	return decompressTOCEStargz(r)
+}
+
+func parseTOCEStargz(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error) {
+	tr, err := decompressTOCEStargz(r)
+	if err != nil {
+		return nil, "", err
+	}
+	dgstr := digest.Canonical.Digester()
+	toc = new(JTOC)
+	if err := json.NewDecoder(io.TeeReader(tr, dgstr.Hash())).Decode(&toc); err != nil {
+		return nil, "", fmt.Errorf("error decoding TOC JSON: %v", err)
+	}
+	if err := tr.Close(); err != nil {
+		return nil, "", err
+	}
+	return toc, dgstr.Digest(), nil
+}
+
+func decompressTOCEStargz(r io.Reader) (tocJSON io.ReadCloser, err error) {
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("malformed TOC gzip header: %v", err)
+	}
+	zr.Multistream(false)
+	tr := tar.NewReader(zr)
+	h, err := tr.Next()
+	if err != nil {
+		return nil, fmt.Errorf("failed to find tar header in TOC gzip stream: %v", err)
+	}
+	if h.Name != TOCTarName {
+		return nil, fmt.Errorf("TOC tar entry had name %q; expected %q", h.Name, TOCTarName)
+	}
+	return readCloser{tr, zr.Close}, nil
+}

estargz/gzip_test.go

@@ -0,0 +1,130 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright 2019 The Go Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style
+   license that can be found in the LICENSE file.
+*/
+
+package estargz
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"testing"
+)
+
+// TestGzipEStargz tests gzip-based eStargz
+func TestGzipEStargz(t *testing.T) {
+	testRunner := &TestRunner{
+		TestingT: t,
+		Runner: func(testingT TestingT, name string, run func(t TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	CompressionTestSuite(testRunner,
+		gzipControllerWithLevel(gzip.NoCompression),
+		gzipControllerWithLevel(gzip.BestSpeed),
+		gzipControllerWithLevel(gzip.BestCompression),
+		gzipControllerWithLevel(gzip.DefaultCompression),
+		gzipControllerWithLevel(gzip.HuffmanOnly),
+	)
+}
+
+func gzipControllerWithLevel(compressionLevel int) TestingControllerFactory {
+	return func() TestingController {
+		return &gzipController{&GzipCompressor{compressionLevel}, &GzipDecompressor{}}
+	}
+}
+
+type gzipController struct {
+	*GzipCompressor
+	*GzipDecompressor
+}
+
+func (gc *gzipController) String() string {
+	return fmt.Sprintf("gzip_compression_level=%v", gc.compressionLevel)
+}
+
+// TestStream tests the passed estargz blob contains the specified list of streams.
+func (gc *gzipController) TestStreams(t TestingT, b []byte, streams []int64) {
+	CheckGzipHasStreams(t, b, streams)
+}
+
+func (gc *gzipController) DiffIDOf(t TestingT, b []byte) string {
+	return GzipDiffIDOf(t, b)
+}
+
+// Tests footer encoding, size, and parsing of gzip-based eStargz.
+func TestGzipFooter(t *testing.T) {
+	for off := int64(0); off <= 200000; off += 1023 {
+		checkFooter(t, off)
+		checkLegacyFooter(t, off)
+	}
+}
+
+// TODO: check fallback
+func checkFooter(t *testing.T, off int64) {
+	footer := gzipFooterBytes(off)
+	if len(footer) != FooterSize {
+		t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), FooterSize, footer)
+	}
+	_, got, _, err := (&GzipDecompressor{}).ParseFooter(footer)
+	if err != nil {
+		t.Fatalf("failed to parse footer for offset %d, footer: %x: err: %v",
+			off, footer, err)
+	}
+	if got != off {
+		t.Fatalf("ParseFooter(footerBytes(offset %d)) = %d; want %d", off, got, off)
+	}
+}
+
+func checkLegacyFooter(t *testing.T, off int64) {
+	footer := legacyFooterBytes(off)
+	if len(footer) != legacyFooterSize {
+		t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), legacyFooterSize, footer)
+	}
+	_, got, _, err := (&LegacyGzipDecompressor{}).ParseFooter(footer)
+	if err != nil {
+		t.Fatalf("failed to parse legacy footer for offset %d, footer: %x: err: %v",
+			off, footer, err)
+	}
+	if got != off {
+		t.Fatalf("ParseFooter(legacyFooterBytes(offset %d)) = %d; want %d", off, got, off)
+	}
+}
+
+func legacyFooterBytes(tocOff int64) []byte {
+	buf := bytes.NewBuffer(make([]byte, 0, legacyFooterSize))
+	gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression)
+	gz.Extra = []byte(fmt.Sprintf("%016xSTARGZ", tocOff))
+	gz.Close()
+	if buf.Len() != legacyFooterSize {
+		panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), legacyFooterSize))
+	}
+	return buf.Bytes()
+}

estargz/types.go

@@ -23,6 +23,9 @@
 package estargz
 
 import (
+	"archive/tar"
+	"hash"
+	"io"
 	"os"
 	"path"
 	"time"
@@ -72,6 +75,12 @@ const (
 	// of an image manifest.
 	TOCJSONDigestAnnotation = "containerd.io/snapshot/stargz/toc.digest"
 
+	// StoreUncompressedSizeAnnotation is an additional annotation key for eStargz to enable lazy
+	// pulling on containers/storage. Stargz Store is required to expose the layer's uncompressed size
+	// to the runtime but current OCI image doesn't ship this information by default. So we store this
+	// to the special annotation.
+	StoreUncompressedSizeAnnotation = "io.containers.estargz.uncompressed-size"
+
 	// PrefetchLandmark is a file entry which indicates the end position of
 	// prefetch in the stargz file.
 	PrefetchLandmark = ".prefetch.landmark"
@@ -83,8 +92,8 @@ const (
 	landmarkContents = 0xf
 )
 
-// jtoc is the JSON-serialized table of contents index of the files in the stargz file.
-type jtoc struct {
+// JTOC is the JSON-serialized table of contents index of the files in the stargz file.
+type JTOC struct {
 	Version int         `json:"version"`
 	Entries []*TOCEntry `json:"entries"`
 }
@@ -140,6 +149,12 @@ type TOCEntry struct {
 	// ChunkSize.
 	Offset int64 `json:"offset,omitempty"`
 
+	// InnerOffset is an optional field indicates uncompressed offset
+	// of this "reg" or "chunk" payload in a stream starts from Offset.
+	// This field enables to put multiple "reg" or "chunk" payloads
+	// in one chunk with having the same Offset but different InnerOffset.
+	InnerOffset int64 `json:"innerOffset,omitempty"`
+
 	nextOffset int64 // the Offset of the next entry with a non-zero Offset
 
 	// DevMajor is the major device number for "char" and "block" types.
@@ -150,7 +165,8 @@ type TOCEntry struct {
 
 	// NumLink is the number of entry names pointing to this entry.
 	// Zero means one name references this entry.
-	NumLink int
+	// This field is calculated during runtime and not recorded in TOC JSON.
+	NumLink int `json:"-"`
 
 	// Xattrs are the extended attribute for the entry.
 	Xattrs map[string][]byte `json:"xattrs,omitempty"`
@@ -176,6 +192,9 @@ type TOCEntry struct {
 	ChunkDigest string `json:"chunkDigest,omitempty"`
 
 	children map[string]*TOCEntry
+
+	// chunkTopIndex is index of the entry where Offset starts in the blob.
+	chunkTopIndex int
 }
 
 // ModTime returns the entry's modification time.
@@ -229,7 +248,9 @@ func (fi fileInfo) Size() int64        { return fi.e.Size }
 func (fi fileInfo) ModTime() time.Time { return fi.e.ModTime() }
 func (fi fileInfo) Sys() interface{}   { return fi.e }
 func (fi fileInfo) Mode() (m os.FileMode) {
-	m = os.FileMode(fi.e.Mode) & os.ModePerm
+	// TOCEntry.Mode is tar.Header.Mode so we can understand the these bits using `tar` pkg.
+	m = (&tar.Header{Mode: fi.e.Mode}).FileInfo().Mode() &
+		(os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky)
 	switch fi.e.Type {
 	case "dir":
 		m |= os.ModeDir
@@ -242,7 +263,6 @@ func (fi fileInfo) Mode() (m os.FileMode) {
 	case "fifo":
 		m |= os.ModeNamedPipe
 	}
-	// TODO: ModeSetuid, ModeSetgid, if/as needed.
 	return m
 }
 
@@ -254,3 +274,69 @@ type TOCEntryVerifier interface {
 	// contents of the specified TOCEntry.
 	Verifier(ce *TOCEntry) (digest.Verifier, error)
 }
+
+// Compression provides the compression helper to be used creating and parsing eStargz.
+// This package provides gzip-based Compression by default, but any compression
+// algorithm (e.g. zstd) can be used as long as it implements Compression.
+type Compression interface {
+	Compressor
+	Decompressor
+}
+
+// Compressor represents the helper mothods to be used for creating eStargz.
+type Compressor interface {
+	// Writer returns WriteCloser to be used for writing a chunk to eStargz.
+	// Everytime a chunk is written, the WriteCloser is closed and Writer is
+	// called again for writing the next chunk.
+	//
+	// The returned writer should implement "Flush() error" function that flushes
+	// any pending compressed data to the underlying writer.
+	Writer(w io.Writer) (WriteFlushCloser, error)
+
+	// WriteTOCAndFooter is called to write JTOC to the passed Writer.
+	// diffHash calculates the DiffID (uncompressed sha256 hash) of the blob
+	// WriteTOCAndFooter can optionally write anything that affects DiffID calculation
+	// (e.g. uncompressed TOC JSON).
+	//
+	// This function returns tocDgst that represents the digest of TOC that will be used
+	// to verify this blob when it's parsed.
+	WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (tocDgst digest.Digest, err error)
+}
+
+// Decompressor represents the helper mothods to be used for parsing eStargz.
+type Decompressor interface {
+	// Reader returns ReadCloser to be used for decompressing file payload.
+	Reader(r io.Reader) (io.ReadCloser, error)
+
+	// FooterSize returns the size of the footer of this blob.
+	FooterSize() int64
+
+	// ParseFooter parses the footer and returns the offset and (compressed) size of TOC.
+	// payloadBlobSize is the (compressed) size of the blob payload (i.e. the size between
+	// the top until the TOC JSON).
+	//
+	// If tocOffset < 0, we assume that TOC isn't contained in the blob and pass nil reader
+	// to ParseTOC. We expect that ParseTOC acquire TOC from the external location and return it.
+	//
+	// tocSize is optional. If tocSize <= 0, it's by default the size of the range from tocOffset until the beginning of the
+	// footer (blob size - tocOff - FooterSize).
+	// If blobPayloadSize < 0, blobPayloadSize become the blob size.
+	ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error)
+
+	// ParseTOC parses TOC from the passed reader. The reader provides the partial contents
+	// of the underlying blob that has the range specified by ParseFooter method.
+	//
+	// This function returns tocDgst that represents the digest of TOC that will be used
+	// to verify this blob. This must match to the value returned from
+	// Compressor.WriteTOCAndFooter that is used when creating this blob.
+	//
+	// If tocOffset returned by ParseFooter is < 0, we assume that TOC isn't contained in the blob.
+	// Pass nil reader to ParseTOC then we expect that ParseTOC acquire TOC from the external location
+	// and return it.
+	ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Digest, err error)
+}
+
+type WriteFlushCloser interface {
+	io.WriteCloser
+	Flush() error
+}

estargz/zstdchunked/zstdchunked.go

@@ -0,0 +1,201 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package zstdchunked
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"hash"
+	"io"
+	"sync"
+
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/klauspost/compress/zstd"
+	digest "github.com/opencontainers/go-digest"
+)
+
+const (
+	// ManifestChecksumAnnotation is an annotation that contains the compressed TOC Digset
+	ManifestChecksumAnnotation = "io.containers.zstd-chunked.manifest-checksum"
+
+	// ManifestPositionAnnotation is an annotation that contains the offset to the TOC.
+	ManifestPositionAnnotation = "io.containers.zstd-chunked.manifest-position"
+
+	// FooterSize is the size of the footer
+	FooterSize = 40
+
+	manifestTypeCRFS = 1
+)
+
+var (
+	skippableFrameMagic   = []byte{0x50, 0x2a, 0x4d, 0x18}
+	zstdFrameMagic        = []byte{0x28, 0xb5, 0x2f, 0xfd}
+	zstdChunkedFrameMagic = []byte{0x47, 0x6e, 0x55, 0x6c, 0x49, 0x6e, 0x55, 0x78}
+)
+
+type Decompressor struct{}
+
+func (zz *Decompressor) Reader(r io.Reader) (io.ReadCloser, error) {
+	decoder, err := zstd.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	return &zstdReadCloser{decoder}, nil
+}
+
+func (zz *Decompressor) ParseTOC(r io.Reader) (toc *estargz.JTOC, tocDgst digest.Digest, err error) {
+	zr, err := zstd.NewReader(r)
+	if err != nil {
+		return nil, "", err
+	}
+	defer zr.Close()
+	dgstr := digest.Canonical.Digester()
+	toc = new(estargz.JTOC)
+	if err := json.NewDecoder(io.TeeReader(zr, dgstr.Hash())).Decode(&toc); err != nil {
+		return nil, "", fmt.Errorf("error decoding TOC JSON: %w", err)
+	}
+	return toc, dgstr.Digest(), nil
+}
+
+func (zz *Decompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) {
+	offset := binary.LittleEndian.Uint64(p[0:8])
+	compressedLength := binary.LittleEndian.Uint64(p[8:16])
+	if !bytes.Equal(zstdChunkedFrameMagic, p[32:40]) {
+		return 0, 0, 0, fmt.Errorf("invalid magic number")
+	}
+	// 8 is the size of the zstd skippable frame header + the frame size (see WriteTOCAndFooter)
+	return int64(offset - 8), int64(offset), int64(compressedLength), nil
+}
+
+func (zz *Decompressor) FooterSize() int64 {
+	return FooterSize
+}
+
+func (zz *Decompressor) DecompressTOC(r io.Reader) (tocJSON io.ReadCloser, err error) {
+	decoder, err := zstd.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	br := bufio.NewReader(decoder)
+	if _, err := br.Peek(1); err != nil {
+		return nil, err
+	}
+	return &reader{br, decoder.Close}, nil
+}
+
+type reader struct {
+	io.Reader
+	closeFunc func()
+}
+
+func (r *reader) Close() error { r.closeFunc(); return nil }
+
+type zstdReadCloser struct{ *zstd.Decoder }
+
+func (z *zstdReadCloser) Close() error {
+	z.Decoder.Close()
+	return nil
+}
+
+type Compressor struct {
+	CompressionLevel zstd.EncoderLevel
+	Metadata         map[string]string
+
+	pool sync.Pool
+}
+
+func (zc *Compressor) Writer(w io.Writer) (estargz.WriteFlushCloser, error) {
+	if wc := zc.pool.Get(); wc != nil {
+		ec := wc.(*zstd.Encoder)
+		ec.Reset(w)
+		return &poolEncoder{ec, zc}, nil
+	}
+	ec, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zc.CompressionLevel), zstd.WithLowerEncoderMem(true))
+	if err != nil {
+		return nil, err
+	}
+	return &poolEncoder{ec, zc}, nil
+}
+
+type poolEncoder struct {
+	*zstd.Encoder
+	zc *Compressor
+}
+
+func (w *poolEncoder) Close() error {
+	if err := w.Encoder.Close(); err != nil {
+		return err
+	}
+	w.zc.pool.Put(w.Encoder)
+	return nil
+}
+
+func (zc *Compressor) WriteTOCAndFooter(w io.Writer, off int64, toc *estargz.JTOC, diffHash hash.Hash) (digest.Digest, error) {
+	tocJSON, err := json.MarshalIndent(toc, "", "\t")
+	if err != nil {
+		return "", err
+	}
+	buf := new(bytes.Buffer)
+	encoder, err := zstd.NewWriter(buf, zstd.WithEncoderLevel(zc.CompressionLevel))
+	if err != nil {
+		return "", err
+	}
+	if _, err := encoder.Write(tocJSON); err != nil {
+		return "", err
+	}
+	if err := encoder.Close(); err != nil {
+		return "", err
+	}
+	compressedTOC := buf.Bytes()
+	_, err = io.Copy(w, bytes.NewReader(appendSkippableFrameMagic(compressedTOC)))
+
+	// 8 is the size of the zstd skippable frame header + the frame size
+	tocOff := uint64(off) + 8
+	if _, err := w.Write(appendSkippableFrameMagic(
+		zstdFooterBytes(tocOff, uint64(len(tocJSON)), uint64(len(compressedTOC)))),
+	); err != nil {
+		return "", err
+	}
+
+	if zc.Metadata != nil {
+		zc.Metadata[ManifestChecksumAnnotation] = digest.FromBytes(compressedTOC).String()
+		zc.Metadata[ManifestPositionAnnotation] = fmt.Sprintf("%d:%d:%d:%d",
+			tocOff, len(compressedTOC), len(tocJSON), manifestTypeCRFS)
+	}
+
+	return digest.FromBytes(tocJSON), err
+}
+
+// zstdFooterBytes returns the 40 bytes footer.
+func zstdFooterBytes(tocOff, tocRawSize, tocCompressedSize uint64) []byte {
+	footer := make([]byte, FooterSize)
+	binary.LittleEndian.PutUint64(footer, tocOff)
+	binary.LittleEndian.PutUint64(footer[8:], tocCompressedSize)
+	binary.LittleEndian.PutUint64(footer[16:], tocRawSize)
+	binary.LittleEndian.PutUint64(footer[24:], manifestTypeCRFS)
+	copy(footer[32:40], zstdChunkedFrameMagic)
+	return footer
+}
+
+func appendSkippableFrameMagic(b []byte) []byte {
+	size := make([]byte, 4)
+	binary.LittleEndian.PutUint32(size, uint32(len(b)))
+	return append(append(skippableFrameMagic, size...), b...)
+}

estargz/zstdchunked/zstdchunked_test.go

@@ -0,0 +1,182 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package zstdchunked
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"sort"
+	"testing"
+
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/klauspost/compress/zstd"
+)
+
+// TestZstdChunked tests zstd:chunked
+func TestZstdChunked(t *testing.T) {
+	testRunner := &estargz.TestRunner{
+		TestingT: t,
+		Runner: func(testingT estargz.TestingT, name string, run func(t estargz.TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	estargz.CompressionTestSuite(testRunner,
+		zstdControllerWithLevel(zstd.SpeedFastest),
+		zstdControllerWithLevel(zstd.SpeedDefault),
+		zstdControllerWithLevel(zstd.SpeedBetterCompression),
+		// zstdControllerWithLevel(zstd.SpeedBestCompression), // consumes too much memory to pass on CI
+	)
+}
+
+func zstdControllerWithLevel(compressionLevel zstd.EncoderLevel) estargz.TestingControllerFactory {
+	return func() estargz.TestingController {
+		return &zstdController{&Compressor{CompressionLevel: compressionLevel}, &Decompressor{}}
+	}
+}
+
+type zstdController struct {
+	*Compressor
+	*Decompressor
+}
+
+func (zc *zstdController) String() string {
+	return fmt.Sprintf("zstd_compression_level=%v", zc.CompressionLevel)
+}
+
+// TestStream tests the passed zstdchunked blob contains the specified list of streams.
+// The last entry of streams must be the offset of footer (len(b) - footerSize).
+func (zc *zstdController) TestStreams(t estargz.TestingT, b []byte, streams []int64) {
+	t.Logf("got zstd streams (compressed size: %d):", len(b))
+
+	if len(streams) == 0 {
+		return // nop
+	}
+
+	// We expect the last offset is footer offset.
+	// 8 is the size of the zstd skippable frame header + the frame size (see WriteTOCAndFooter)
+	sort.Slice(streams, func(i, j int) bool {
+		return streams[i] < streams[j]
+	})
+	streams[len(streams)-1] = streams[len(streams)-1] - 8
+	wants := map[int64]struct{}{}
+	for _, s := range streams {
+		wants[s] = struct{}{}
+	}
+
+	magicLen := 4 // length of magic bytes and skippable frame magic bytes
+	zoff := 0
+	numStreams := 0
+	for {
+		if len(b) <= zoff {
+			break
+		} else if len(b)-zoff <= magicLen {
+			t.Fatalf("invalid frame size %d is too small", len(b)-zoff)
+		}
+		delete(wants, int64(zoff)) // offset found
+		remainingFrames := b[zoff:]
+
+		// Check if zoff points to the beginning of a frame
+		if !bytes.Equal(remainingFrames[:magicLen], zstdFrameMagic) {
+			if !bytes.Equal(remainingFrames[:magicLen], skippableFrameMagic) {
+				t.Fatalf("frame must start from magic bytes; but %x",
+					remainingFrames[:magicLen])
+			}
+		}
+		searchBase := magicLen
+		nextMagicIdx := nextIndex(remainingFrames[searchBase:], zstdFrameMagic)
+		nextSkippableIdx := nextIndex(remainingFrames[searchBase:], skippableFrameMagic)
+		nextFrame := len(remainingFrames)
+		for _, i := range []int{nextMagicIdx, nextSkippableIdx} {
+			if 0 < i && searchBase+i < nextFrame {
+				nextFrame = searchBase + i
+			}
+		}
+		t.Logf("  [%d] at %d in stargz (nextFrame: %d/%d): %v, %v",
+			numStreams, zoff, zoff+nextFrame, len(b), nextMagicIdx, nextSkippableIdx)
+		zoff += nextFrame
+		numStreams++
+	}
+	if len(wants) != 0 {
+		t.Fatalf("some stream offsets not found in the blob: %v", wants)
+	}
+}
+
+func nextIndex(s1, sub []byte) int {
+	for i := 0; i < len(s1); i++ {
+		if len(s1)-i < len(sub) {
+			return -1
+		} else if bytes.Equal(s1[i:i+len(sub)], sub) {
+			return i
+		}
+	}
+	return -1
+}
+
+func (zc *zstdController) DiffIDOf(t estargz.TestingT, b []byte) string {
+	h := sha256.New()
+	zr, err := zstd.NewReader(bytes.NewReader(b))
+	if err != nil {
+		t.Fatalf("diffIDOf(zstd): %v", err)
+	}
+	defer zr.Close()
+	if _, err := io.Copy(h, zr); err != nil {
+		t.Fatalf("diffIDOf(zstd).Copy: %v", err)
+	}
+	return fmt.Sprintf("sha256:%x", h.Sum(nil))
+}
+
+// Tests footer encoding, size, and parsing of zstd:chunked.
+func TestZstdChunkedFooter(t *testing.T) {
+	max := int64(200000)
+	for off := int64(0); off <= max; off += 1023 {
+		size := max - off
+		checkZstdChunkedFooter(t, off, size, size/2)
+	}
+}
+
+func checkZstdChunkedFooter(t *testing.T, off, size, cSize int64) {
+	footer := zstdFooterBytes(uint64(off), uint64(size), uint64(cSize))
+	if len(footer) != FooterSize {
+		t.Fatalf("for offset %v, footer length was %d, not expected %d. got bytes: %q", off, len(footer), FooterSize, footer)
+	}
+	gotBlobPayloadSize, gotOff, gotSize, err := (&Decompressor{}).ParseFooter(footer)
+	if err != nil {
+		t.Fatalf("failed to parse footer for offset %d, footer: %x: err: %v",
+			off, footer, err)
+	}
+	if gotBlobPayloadSize != off-8 {
+		// 8 is the size of the zstd skippable frame header + the frame size (see WriteTOCAndFooter)
+		t.Fatalf("ParseFooter(footerBytes(offset %d)) = blobPayloadSize %d; want %d", off, gotBlobPayloadSize, off-8)
+	}
+	if gotOff != off {
+		t.Fatalf("ParseFooter(footerBytes(offset %d)) = off %d; want %d", off, gotOff, off)
+	}
+	if gotSize != cSize {
+		t.Fatalf("ParseFooter(footerBytes(offset %d)) = size %d; want %d", off, gotSize, cSize)
+	}
+}

fs/config/config.go

@@ -33,35 +33,131 @@ const (
 	TargetPrefetchSizeLabel = "containerd.io/snapshot/remote/stargz.prefetch"
 )
 
+// Config is configuration for stargz snapshotter filesystem.
 type Config struct {
-	HTTPCacheType       string `toml:"http_cache_type"`
-	FSCacheType         string `toml:"filesystem_cache_type"`
-	ResolveResultEntry  int    `toml:"resolve_result_entry"`
-	PrefetchSize        int64  `toml:"prefetch_size"`
-	PrefetchTimeoutSec  int64  `toml:"prefetch_timeout_sec"`
-	NoPrefetch          bool   `toml:"noprefetch"`
-	NoBackgroundFetch   bool   `toml:"no_background_fetch"`
-	Debug               bool   `toml:"debug"`
-	AllowNoVerification bool   `toml:"allow_no_verification"`
-	DisableVerification bool   `toml:"disable_verification"`
-	MaxConcurrency      int64  `toml:"max_concurrency"`
+	// Type of cache for compressed contents fetched from the registry. "memory" stores them on memory.
+	// Other values default to cache them on disk.
+	HTTPCacheType string `toml:"http_cache_type" json:"http_cache_type"`
+
+	// Type of cache for uncompressed files contents. "memory" stores them on memory. Other values
+	// default to cache them on disk.
+	FSCacheType string `toml:"filesystem_cache_type" json:"filesystem_cache_type"`
+
+	// ResolveResultEntryTTLSec is TTL (in sec) to cache resolved layers for
+	// future use. (default 120s)
+	ResolveResultEntryTTLSec int `toml:"resolve_result_entry_ttl_sec" json:"resolve_result_entry_ttl_sec"`
+
+	// PrefetchSize is the default size (in bytes) to prefetch when mounting a layer. Default is 0. Stargz-snapshotter still
+	// uses the value specified by the image using "containerd.io/snapshot/remote/stargz.prefetch" or the landmark file.
+	PrefetchSize int64 `toml:"prefetch_size" json:"prefetch_size"`
+
+	// PrefetchTimeoutSec is the default timeout (in seconds) when the prefetching takes long. Default is 10s.
+	PrefetchTimeoutSec int64 `toml:"prefetch_timeout_sec" json:"prefetch_timeout_sec"`
+
+	// NoPrefetch disables prefetching. Default is false.
+	NoPrefetch bool `toml:"noprefetch" json:"noprefetch"`
+
+	// NoBackgroundFetch disables the behaviour of fetching the entire layer contents in background. Default is false.
+	NoBackgroundFetch bool `toml:"no_background_fetch" json:"no_background_fetch"`
+
+	// Debug enables filesystem debug log.
+	Debug bool `toml:"debug" json:"debug"`
+
+	// AllowNoVerification allows mouting images without verification. Default is false.
+	AllowNoVerification bool `toml:"allow_no_verification" json:"allow_no_verification"`
+
+	// DisableVerification disables verifying layer contents. Default is false.
+	DisableVerification bool `toml:"disable_verification" json:"disable_verification"`
+
+	// MaxConcurrency is max number of concurrent background tasks for fetching layer contents. Default is 2.
+	MaxConcurrency int64 `toml:"max_concurrency" json:"max_concurrency"`
+
+	// NoPrometheus disables exposing filesystem-related metrics. Default is false.
+	NoPrometheus bool `toml:"no_prometheus" json:"no_prometheus"`
 
 	// BlobConfig is config for layer blob management.
-	BlobConfig `toml:"blob"`
+	BlobConfig `toml:"blob" json:"blob"`
 
 	// DirectoryCacheConfig is config for directory-based cache.
-	DirectoryCacheConfig `toml:"directory_cache"`
+	DirectoryCacheConfig `toml:"directory_cache" json:"directory_cache"`
+
+	// FuseConfig is configurations for FUSE fs.
+	FuseConfig `toml:"fuse" json:"fuse"`
+
+	// ResolveResultEntry is a deprecated field.
+	ResolveResultEntry int `toml:"resolve_result_entry" json:"resolve_result_entry"` // deprecated
 }
 
+// BlobConfig is configuration for the logic to fetching blobs.
 type BlobConfig struct {
-	ValidInterval   int64 `toml:"valid_interval"`
-	CheckAlways     bool  `toml:"check_always"`
-	ChunkSize       int64 `toml:"chunk_size"`
-	FetchTimeoutSec int64 `toml:"fetching_timeout_sec"`
+	// ValidInterval specifies a duration (in seconds) during which the layer can be reused without
+	// checking the connection to the registry. Default is 60.
+	ValidInterval int64 `toml:"valid_interval" json:"valid_interval"`
+
+	// CheckAlways overwrites ValidInterval to 0 if it's true. Default is false.
+	CheckAlways bool `toml:"check_always" json:"check_always"`
+
+	// ChunkSize is the granularity (in bytes) at which background fetch and on-demand reads
+	// are fetched from the remote registry. Default is 50000.
+	ChunkSize int64 `toml:"chunk_size" json:"chunk_size"`
+
+	// FetchTimeoutSec is a timeout duration (in seconds) for fetching chunks from the registry. Default is 300.
+	FetchTimeoutSec int64 `toml:"fetching_timeout_sec" json:"fetching_tieout_sec"`
+
+	// ForceSingleRangeMode disables using of multiple ranges in a Range Request and always specifies one larger
+	// region that covers them. Default is false.
+	ForceSingleRangeMode bool `toml:"force_single_range_mode" json:"force_single_range_mode"`
+
+	// PrefetchChunkSize is the maximum bytes transferred per http GET from remote registry
+	// during prefetch. It is recommended to have PrefetchChunkSize > ChunkSize.
+	// If PrefetchChunkSize < ChunkSize prefetch bytes will be fetched as a single http GET,
+	// else total GET requests for prefetch = ceil(PrefetchSize / PrefetchChunkSize).
+	// Default is 0.
+	PrefetchChunkSize int64 `toml:"prefetch_chunk_size" json:"prefetch_chunk_size"`
+
+	// MaxRetries is a max number of reries of a HTTP request. Default is 5.
+	MaxRetries int `toml:"max_retries" json:"max_retries"`
+
+	// MinWaitMSec is minimal delay (in seconds) for the next retrying after a request failure. Default is 30.
+	MinWaitMSec int `toml:"min_wait_msec" json:"min_wait_msec"`
+
+	// MinWaitMSec is maximum delay (in seconds) for the next retrying after a request failure. Default is 30.
+	MaxWaitMSec int `toml:"max_wait_msec" json:"max_wait_msec"`
 }
 
+// DirectoryCacheConfig is configuration for the disk-based cache.
 type DirectoryCacheConfig struct {
-	MaxLRUCacheEntry int  `toml:"max_lru_cache_entry"`
-	MaxCacheFds      int  `toml:"max_cache_fds"`
-	SyncAdd          bool `toml:"sync_add"`
+	// MaxLRUCacheEntry is the number of entries of LRU cache to cache data on memory. Default is 10.
+	MaxLRUCacheEntry int `toml:"max_lru_cache_entry" json:"max_lru_cache_entry"`
+
+	// MaxCacheFds is the number of entries of LRU cache to hold fds of files of cached contents. Default is 10.
+	MaxCacheFds int `toml:"max_cache_fds" json:"max_cache_fds"`
+
+	// SyncAdd being true means that each adding of data to the cache blocks until the data is fully written to the
+	// cache directory. Default is false.
+	SyncAdd bool `toml:"sync_add" json:"sync_add"`
+
+	// Direct disables on-memory data cache. Default is true for saving memory usage.
+	Direct bool `toml:"direct" default:"true" json:"direct"`
+
+	// FadvDontNeed forcefully clean fscache pagecache for saving memory. Default is false.
+	FadvDontNeed bool `toml:"fadv_dontneed" json:"fadv_dontneed"`
+}
+
+// FuseConfig is configuration for FUSE fs.
+type FuseConfig struct {
+	// AttrTimeout defines overall timeout attribute for a file system in seconds.
+	AttrTimeout int64 `toml:"attr_timeout" json:"attr_timeout"`
+
+	// EntryTimeout defines TTL for directory, name lookup in seconds.
+	EntryTimeout int64 `toml:"entry_timeout" json:"entry_timeout"`
+
+	// PassThrough indicates whether to enable FUSE passthrough mode to improve local file read performance. Default is false.
+	PassThrough bool `toml:"passthrough" default:"false" json:"passthrough"`
+
+	// MergeBufferSize is the size of the buffer to merge chunks (in bytes) for passthrough mode. Default is 400MB.
+	MergeBufferSize int64 `toml:"merge_buffer_size" default:"419430400" json:"merge_buffer_size"`
+
+	// MergeWorkerCount is the number of workers to merge chunks for passthrough mode. Default is 10.
+	MergeWorkerCount int `toml:"merge_worker_count" default:"10" json:"merge_worker_count"`
 }

fs/layer/layer.go

@@ -0,0 +1,710 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright 2019 The Go Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style
+   license that can be found in the NOTICE.md file.
+*/
+
+package layer
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/log"
+	"github.com/containerd/stargz-snapshotter/cache"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/estargz/zstdchunked"
+	"github.com/containerd/stargz-snapshotter/fs/config"
+	commonmetrics "github.com/containerd/stargz-snapshotter/fs/metrics/common"
+	"github.com/containerd/stargz-snapshotter/fs/reader"
+	"github.com/containerd/stargz-snapshotter/fs/remote"
+	"github.com/containerd/stargz-snapshotter/fs/source"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	"github.com/containerd/stargz-snapshotter/task"
+	"github.com/containerd/stargz-snapshotter/util/cacheutil"
+	"github.com/containerd/stargz-snapshotter/util/namedmutex"
+	fusefs "github.com/hanwen/go-fuse/v2/fs"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+const (
+	defaultResolveResultEntryTTLSec = 120
+	defaultMaxLRUCacheEntry         = 10
+	defaultMaxCacheFds              = 10
+	defaultPrefetchTimeoutSec       = 10
+	memoryCacheType                 = "memory"
+)
+
+// passThroughConfig contains configuration for FUSE passthrough mode
+type passThroughConfig struct {
+	// enable indicates whether to enable FUSE passthrough mode
+	enable bool
+
+	// mergeBufferSize is the size of the buffer to merge chunks (in bytes)
+	mergeBufferSize int64
+
+	// mergeWorkerCount is the number of workers to merge chunks
+	mergeWorkerCount int
+}
+
+// Layer represents a layer.
+type Layer interface {
+	// Info returns the information of this layer.
+	Info() Info
+
+	// RootNode returns the root node of this layer.
+	RootNode(baseInode uint32) (fusefs.InodeEmbedder, error)
+
+	// Check checks if the layer is still connectable.
+	Check() error
+
+	// Refresh refreshes the layer connection.
+	Refresh(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) error
+
+	// Verify verifies this layer using the passed TOC Digest.
+	// Nop if Verify() or SkipVerify() was already called.
+	Verify(tocDigest digest.Digest) (err error)
+
+	// SkipVerify skips verification for this layer.
+	// Nop if Verify() or SkipVerify() was already called.
+	SkipVerify()
+
+	// Prefetch prefetches the specified size. If the layer is eStargz and contains landmark files,
+	// the range indicated by these files is respected.
+	Prefetch(prefetchSize int64) error
+
+	// ReadAt reads this layer.
+	ReadAt([]byte, int64, ...remote.Option) (int, error)
+
+	// WaitForPrefetchCompletion waits untils Prefetch completes.
+	WaitForPrefetchCompletion() error
+
+	// BackgroundFetch fetches the entire layer contents to the cache.
+	// Fetching contents is done as a background task.
+	BackgroundFetch() error
+
+	// Done releases the reference to this layer. The resources related to this layer will be
+	// discarded sooner or later. Queries after calling this function won't be serviced.
+	Done()
+
+	// Close is the same as Done. But this evicts the resources related to this Layer immediately.
+	// This can be used for cleaning up resources on unmount.
+	Close() error
+}
+
+// Info is the current status of a layer.
+type Info struct {
+	Digest       digest.Digest
+	Size         int64     // layer size in bytes
+	FetchedSize  int64     // layer fetched size in bytes
+	PrefetchSize int64     // layer prefetch size in bytes
+	ReadTime     time.Time // last time the layer was read
+	TOCDigest    digest.Digest
+}
+
+// Resolver resolves the layer location and provieds the handler of that layer.
+type Resolver struct {
+	rootDir                 string
+	resolver                *remote.Resolver
+	prefetchTimeout         time.Duration
+	layerCache              *cacheutil.TTLCache
+	layerCacheMu            sync.Mutex
+	blobCache               *cacheutil.TTLCache
+	blobCacheMu             sync.Mutex
+	backgroundTaskManager   *task.BackgroundTaskManager
+	resolveLock             *namedmutex.NamedMutex
+	config                  config.Config
+	metadataStore           metadata.Store
+	overlayOpaqueType       OverlayOpaqueType
+	additionalDecompressors func(context.Context, source.RegistryHosts, reference.Spec, ocispec.Descriptor) []metadata.Decompressor
+}
+
+// NewResolver returns a new layer resolver.
+func NewResolver(root string, backgroundTaskManager *task.BackgroundTaskManager, cfg config.Config, resolveHandlers map[string]remote.Handler, metadataStore metadata.Store, overlayOpaqueType OverlayOpaqueType, additionalDecompressors func(context.Context, source.RegistryHosts, reference.Spec, ocispec.Descriptor) []metadata.Decompressor) (*Resolver, error) {
+	resolveResultEntryTTL := time.Duration(cfg.ResolveResultEntryTTLSec) * time.Second
+	if resolveResultEntryTTL == 0 {
+		resolveResultEntryTTL = defaultResolveResultEntryTTLSec * time.Second
+	}
+	prefetchTimeout := time.Duration(cfg.PrefetchTimeoutSec) * time.Second
+	if prefetchTimeout == 0 {
+		prefetchTimeout = defaultPrefetchTimeoutSec * time.Second
+	}
+
+	// layerCache caches resolved layers for future use. This is useful in a use-case where
+	// the filesystem resolves and caches all layers in an image (not only queried one) in parallel,
+	// before they are actually queried.
+	layerCache := cacheutil.NewTTLCache(resolveResultEntryTTL)
+	layerCache.OnEvicted = func(key string, value interface{}) {
+		if err := value.(*layer).close(); err != nil {
+			log.L.WithField("key", key).WithError(err).Warnf("failed to clean up layer")
+			return
+		}
+		log.L.WithField("key", key).Debugf("cleaned up layer")
+	}
+
+	// blobCache caches resolved blobs for futural use. This is especially useful when a layer
+	// isn't eStargz/stargz (the *layer object won't be created/cached in this case).
+	blobCache := cacheutil.NewTTLCache(resolveResultEntryTTL)
+	blobCache.OnEvicted = func(key string, value interface{}) {
+		if err := value.(remote.Blob).Close(); err != nil {
+			log.L.WithField("key", key).WithError(err).Warnf("failed to clean up blob")
+			return
+		}
+		log.L.WithField("key", key).Debugf("cleaned up blob")
+	}
+
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
+
+	return &Resolver{
+		rootDir:                 root,
+		resolver:                remote.NewResolver(cfg.BlobConfig, resolveHandlers),
+		layerCache:              layerCache,
+		blobCache:               blobCache,
+		prefetchTimeout:         prefetchTimeout,
+		backgroundTaskManager:   backgroundTaskManager,
+		config:                  cfg,
+		resolveLock:             new(namedmutex.NamedMutex),
+		metadataStore:           metadataStore,
+		overlayOpaqueType:       overlayOpaqueType,
+		additionalDecompressors: additionalDecompressors,
+	}, nil
+}
+
+func newCache(root string, cacheType string, cfg config.Config) (cache.BlobCache, error) {
+	if cacheType == memoryCacheType {
+		return cache.NewMemoryCache(), nil
+	}
+
+	dcc := cfg.DirectoryCacheConfig
+	maxDataEntry := dcc.MaxLRUCacheEntry
+	if maxDataEntry == 0 {
+		maxDataEntry = defaultMaxLRUCacheEntry
+	}
+	maxFdEntry := dcc.MaxCacheFds
+	if maxFdEntry == 0 {
+		maxFdEntry = defaultMaxCacheFds
+	}
+
+	bufPool := &sync.Pool{
+		New: func() interface{} {
+			return new(bytes.Buffer)
+		},
+	}
+	dCache, fCache := cacheutil.NewLRUCache(maxDataEntry), cacheutil.NewLRUCache(maxFdEntry)
+	dCache.OnEvicted = func(key string, value interface{}) {
+		value.(*bytes.Buffer).Reset()
+		bufPool.Put(value)
+	}
+	fCache.OnEvicted = func(key string, value interface{}) {
+		value.(*os.File).Close()
+	}
+	// create a cache on an unique directory
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
+	cachePath, err := os.MkdirTemp(root, "")
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize directory cache: %w", err)
+	}
+	return cache.NewDirectoryCache(
+		cachePath,
+		cache.DirectoryCacheConfig{
+			SyncAdd:      dcc.SyncAdd,
+			DataCache:    dCache,
+			FdCache:      fCache,
+			BufPool:      bufPool,
+			Direct:       dcc.Direct,
+			FadvDontNeed: dcc.FadvDontNeed,
+		},
+	)
+}
+
+// Resolve resolves a layer based on the passed layer blob information.
+func (r *Resolver) Resolve(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor, esgzOpts ...metadata.Option) (_ Layer, retErr error) {
+	name := refspec.String() + "/" + desc.Digest.String()
+
+	// Wait if resolving this layer is already running. The result
+	// can hopefully get from the cache.
+	r.resolveLock.Lock(name)
+	defer r.resolveLock.Unlock(name)
+
+	ctx = log.WithLogger(ctx, log.G(ctx).WithField("src", name))
+
+	// First, try to retrieve this layer from the underlying cache.
+	r.layerCacheMu.Lock()
+	c, done, ok := r.layerCache.Get(name)
+	r.layerCacheMu.Unlock()
+	if ok {
+		if l := c.(*layer); l.Check() == nil {
+			log.G(ctx).Debugf("hit layer cache %q", name)
+			return &layerRef{l, done}, nil
+		}
+		// Cached layer is invalid
+		done(true)
+		r.layerCacheMu.Lock()
+		r.layerCache.Remove(name)
+		r.layerCacheMu.Unlock()
+	}
+
+	log.G(ctx).Debugf("resolving")
+
+	// Resolve the blob.
+	blobR, err := r.resolveBlob(ctx, hosts, refspec, desc)
+	if err != nil {
+		return nil, fmt.Errorf("failed to resolve the blob: %w", err)
+	}
+	defer func() {
+		if retErr != nil {
+			blobR.done(true)
+		}
+	}()
+
+	fsCache, err := newCache(filepath.Join(r.rootDir, "fscache"), r.config.FSCacheType, r.config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create fs cache: %w", err)
+	}
+	defer func() {
+		if retErr != nil {
+			fsCache.Close()
+		}
+	}()
+
+	// Get a reader for stargz archive.
+	// Each file's read operation is a prioritized task and all background tasks
+	// will be stopped during the execution so this can avoid being disturbed for
+	// NW traffic by background tasks.
+	sr := io.NewSectionReader(readerAtFunc(func(p []byte, offset int64) (n int, err error) {
+		r.backgroundTaskManager.DoPrioritizedTask()
+		defer r.backgroundTaskManager.DonePrioritizedTask()
+		return blobR.ReadAt(p, offset)
+	}), 0, blobR.Size())
+	// define telemetry hooks to measure latency metrics inside estargz package
+	telemetry := metadata.Telemetry{
+		GetFooterLatency: func(start time.Time) {
+			commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.StargzFooterGet, desc.Digest, start)
+		},
+		GetTocLatency: func(start time.Time) {
+			commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.StargzTocGet, desc.Digest, start)
+		},
+		DeserializeTocLatency: func(start time.Time) {
+			commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.DeserializeTocJSON, desc.Digest, start)
+		},
+	}
+
+	additionalDecompressors := []metadata.Decompressor{new(zstdchunked.Decompressor)}
+	if r.additionalDecompressors != nil {
+		additionalDecompressors = append(additionalDecompressors, r.additionalDecompressors(ctx, hosts, refspec, desc)...)
+	}
+	meta, err := r.metadataStore(sr,
+		append(esgzOpts, metadata.WithTelemetry(&telemetry), metadata.WithDecompressors(additionalDecompressors...))...)
+	if err != nil {
+		return nil, err
+	}
+	vr, err := reader.NewReader(meta, fsCache, desc.Digest)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read layer: %w", err)
+	}
+
+	// Combine layer information together and cache it.
+	l := newLayer(r, desc, blobR, vr, passThroughConfig{
+		enable:           r.config.PassThrough,
+		mergeBufferSize:  r.config.MergeBufferSize,
+		mergeWorkerCount: r.config.MergeWorkerCount,
+	})
+	r.layerCacheMu.Lock()
+	cachedL, done2, added := r.layerCache.Add(name, l)
+	r.layerCacheMu.Unlock()
+	if !added {
+		l.close() // layer already exists in the cache. discrad this.
+	}
+
+	log.G(ctx).Debugf("resolved")
+	return &layerRef{cachedL.(*layer), done2}, nil
+}
+
+// resolveBlob resolves a blob based on the passed layer blob information.
+func (r *Resolver) resolveBlob(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) (_ *blobRef, retErr error) {
+	name := refspec.String() + "/" + desc.Digest.String()
+
+	// Try to retrieve the blob from the underlying cache.
+	r.blobCacheMu.Lock()
+	c, done, ok := r.blobCache.Get(name)
+	r.blobCacheMu.Unlock()
+	if ok {
+		if blob := c.(remote.Blob); blob.Check() == nil {
+			return &blobRef{blob, done}, nil
+		}
+		// invalid blob. discard this.
+		done(true)
+		r.blobCacheMu.Lock()
+		r.blobCache.Remove(name)
+		r.blobCacheMu.Unlock()
+	}
+
+	httpCache, err := newCache(filepath.Join(r.rootDir, "httpcache"), r.config.HTTPCacheType, r.config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create http cache: %w", err)
+	}
+	defer func() {
+		if retErr != nil {
+			httpCache.Close()
+		}
+	}()
+
+	// Resolve the blob and cache the result.
+	b, err := r.resolver.Resolve(ctx, hosts, refspec, desc, httpCache)
+	if err != nil {
+		return nil, fmt.Errorf("failed to resolve the source: %w", err)
+	}
+	r.blobCacheMu.Lock()
+	cachedB, done, added := r.blobCache.Add(name, b)
+	r.blobCacheMu.Unlock()
+	if !added {
+		b.Close() // blob already exists in the cache. discard this.
+	}
+	return &blobRef{cachedB.(remote.Blob), done}, nil
+}
+
+func newLayer(
+	resolver *Resolver,
+	desc ocispec.Descriptor,
+	blob *blobRef,
+	vr *reader.VerifiableReader,
+	pth passThroughConfig,
+) *layer {
+	return &layer{
+		resolver:         resolver,
+		desc:             desc,
+		blob:             blob,
+		verifiableReader: vr,
+		prefetchWaiter:   newWaiter(),
+		passThrough:      pth,
+	}
+}
+
+type layer struct {
+	resolver         *Resolver
+	desc             ocispec.Descriptor
+	blob             *blobRef
+	verifiableReader *reader.VerifiableReader
+	prefetchWaiter   *waiter
+
+	prefetchSize   int64
+	prefetchSizeMu sync.Mutex
+
+	r reader.Reader
+
+	closed   bool
+	closedMu sync.Mutex
+
+	prefetchOnce        sync.Once
+	backgroundFetchOnce sync.Once
+	passThrough         passThroughConfig
+}
+
+func (l *layer) Info() Info {
+	var readTime time.Time
+	if l.r != nil {
+		readTime = l.r.LastOnDemandReadTime()
+	}
+	return Info{
+		Digest:       l.desc.Digest,
+		Size:         l.blob.Size(),
+		FetchedSize:  l.blob.FetchedSize(),
+		PrefetchSize: l.prefetchedSize(),
+		ReadTime:     readTime,
+		TOCDigest:    l.verifiableReader.Metadata().TOCDigest(),
+	}
+}
+
+func (l *layer) prefetchedSize() int64 {
+	l.prefetchSizeMu.Lock()
+	sz := l.prefetchSize
+	l.prefetchSizeMu.Unlock()
+	return sz
+}
+
+func (l *layer) Check() error {
+	if l.isClosed() {
+		return fmt.Errorf("layer is already closed")
+	}
+	return l.blob.Check()
+}
+
+func (l *layer) Refresh(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) error {
+	if l.isClosed() {
+		return fmt.Errorf("layer is already closed")
+	}
+	return l.blob.Refresh(ctx, hosts, refspec, desc)
+}
+
+func (l *layer) Verify(tocDigest digest.Digest) (err error) {
+	if l.isClosed() {
+		return fmt.Errorf("layer is already closed")
+	}
+	if l.r != nil {
+		return nil
+	}
+	l.r, err = l.verifiableReader.VerifyTOC(tocDigest)
+	return
+}
+
+func (l *layer) SkipVerify() {
+	if l.r != nil {
+		return
+	}
+	l.r = l.verifiableReader.SkipVerify()
+}
+
+func (l *layer) Prefetch(prefetchSize int64) (err error) {
+	l.prefetchOnce.Do(func() {
+		ctx := context.Background()
+		l.resolver.backgroundTaskManager.DoPrioritizedTask()
+		defer l.resolver.backgroundTaskManager.DonePrioritizedTask()
+		err = l.prefetch(ctx, prefetchSize)
+		if err != nil {
+			log.G(ctx).WithError(err).Warnf("failed to prefetch layer=%v", l.desc.Digest)
+			return
+		}
+		log.G(ctx).Debug("completed to prefetch")
+	})
+	return
+}
+
+func (l *layer) prefetch(ctx context.Context, prefetchSize int64) error {
+	defer l.prefetchWaiter.done() // Notify the completion
+	// Measuring the total time to complete prefetch (use defer func() because l.Info().PrefetchSize is set later)
+	start := time.Now()
+	defer func() {
+		commonmetrics.WriteLatencyWithBytesLogValue(ctx, l.desc.Digest, commonmetrics.PrefetchTotal, start, commonmetrics.PrefetchSize, l.prefetchedSize())
+	}()
+
+	if l.isClosed() {
+		return fmt.Errorf("layer is already closed")
+	}
+	rootID := l.verifiableReader.Metadata().RootID()
+	if _, _, err := l.verifiableReader.Metadata().GetChild(rootID, estargz.NoPrefetchLandmark); err == nil {
+		// do not prefetch this layer
+		return nil
+	} else if id, _, err := l.verifiableReader.Metadata().GetChild(rootID, estargz.PrefetchLandmark); err == nil {
+		offset, err := l.verifiableReader.Metadata().GetOffset(id)
+		if err != nil {
+			return fmt.Errorf("failed to get offset of prefetch landmark: %w", err)
+		}
+		// override the prefetch size with optimized value
+		prefetchSize = offset
+	} else if prefetchSize > l.blob.Size() {
+		// adjust prefetch size not to exceed the whole layer size
+		prefetchSize = l.blob.Size()
+	}
+
+	// Fetch the target range
+	downloadStart := time.Now()
+	err := l.blob.Cache(0, prefetchSize)
+	commonmetrics.WriteLatencyLogValue(ctx, l.desc.Digest, commonmetrics.PrefetchDownload, downloadStart) // time to download prefetch data
+
+	if err != nil {
+		return fmt.Errorf("failed to prefetch layer: %w", err)
+	}
+
+	// Set prefetch size for metrics after prefetch completed
+	l.prefetchSizeMu.Lock()
+	l.prefetchSize = prefetchSize
+	l.prefetchSizeMu.Unlock()
+
+	// Cache uncompressed contents of the prefetched range
+	decompressStart := time.Now()
+	err = l.verifiableReader.Cache(reader.WithFilter(func(offset int64) bool {
+		return offset < prefetchSize // Cache only prefetch target
+	}))
+	commonmetrics.WriteLatencyLogValue(ctx, l.desc.Digest, commonmetrics.PrefetchDecompress, decompressStart) // time to decompress prefetch data
+	if err != nil {
+		return fmt.Errorf("failed to cache prefetched layer: %w", err)
+	}
+
+	return nil
+}
+
+func (l *layer) WaitForPrefetchCompletion() error {
+	if l.isClosed() {
+		return fmt.Errorf("layer is already closed")
+	}
+	return l.prefetchWaiter.wait(l.resolver.prefetchTimeout)
+}
+
+func (l *layer) BackgroundFetch() (err error) {
+	l.backgroundFetchOnce.Do(func() {
+		ctx := context.Background()
+		err = l.backgroundFetch(ctx)
+		if err != nil {
+			log.G(ctx).WithError(err).Warnf("failed to fetch whole layer=%v", l.desc.Digest)
+			return
+		}
+		log.G(ctx).Debug("completed to fetch all layer data in background")
+	})
+	return
+}
+
+func (l *layer) backgroundFetch(ctx context.Context) error {
+	defer commonmetrics.WriteLatencyLogValue(ctx, l.desc.Digest, commonmetrics.BackgroundFetchTotal, time.Now())
+	if l.isClosed() {
+		return fmt.Errorf("layer is already closed")
+	}
+	br := io.NewSectionReader(readerAtFunc(func(p []byte, offset int64) (retN int, retErr error) {
+		l.resolver.backgroundTaskManager.InvokeBackgroundTask(func(ctx context.Context) {
+			// Measuring the time to download background fetch data (in milliseconds)
+			defer commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.BackgroundFetchDownload, l.Info().Digest, time.Now()) // time to download background fetch data
+			retN, retErr = l.blob.ReadAt(
+				p,
+				offset,
+				remote.WithContext(ctx),              // Make cancellable
+				remote.WithCacheOpts(cache.Direct()), // Do not pollute mem cache
+			)
+		}, 120*time.Second)
+		return
+	}), 0, l.blob.Size())
+	defer commonmetrics.WriteLatencyLogValue(ctx, l.desc.Digest, commonmetrics.BackgroundFetchDecompress, time.Now()) // time to decompress background fetch data (in milliseconds)
+	return l.verifiableReader.Cache(
+		reader.WithReader(br),                // Read contents in background
+		reader.WithCacheOpts(cache.Direct()), // Do not pollute mem cache
+	)
+}
+
+func (l *layerRef) Done() {
+	l.done(false) // leave chances to reuse this
+}
+
+func (l *layerRef) Close() error {
+	l.done(true) // evict this from the cache
+	return nil
+}
+
+func (l *layer) RootNode(baseInode uint32) (fusefs.InodeEmbedder, error) {
+	if l.isClosed() {
+		return nil, fmt.Errorf("layer is already closed")
+	}
+	if l.r == nil {
+		return nil, fmt.Errorf("layer hasn't been verified yet")
+	}
+	return newNode(l.desc.Digest, l.r, l.blob, baseInode, l.resolver.overlayOpaqueType, l.passThrough)
+}
+
+func (l *layer) ReadAt(p []byte, offset int64, opts ...remote.Option) (int, error) {
+	return l.blob.ReadAt(p, offset, opts...)
+}
+
+func (l *layer) close() error {
+	l.closedMu.Lock()
+	defer l.closedMu.Unlock()
+	if l.closed {
+		return nil
+	}
+	l.closed = true
+	defer l.blob.done(true) // Close reader first, then close the blob
+	l.verifiableReader.Close()
+	if l.r != nil {
+		return l.r.Close()
+	}
+	return nil
+}
+
+func (l *layer) isClosed() bool {
+	l.closedMu.Lock()
+	closed := l.closed
+	l.closedMu.Unlock()
+	return closed
+}
+
+// blobRef is a reference to the blob in the cache. Calling `done` decreases the reference counter
+// of this blob in the underlying cache. When nobody refers to the blob in the cache, resources bound
+// to this blob will be discarded.
+type blobRef struct {
+	remote.Blob
+	done func(bool)
+}
+
+// layerRef is a reference to the layer in the cache. Calling `Done` or `done` decreases the
+// reference counter of this blob in the underlying cache. When nobody refers to the layer in the
+// cache, resources bound to this layer will be discarded.
+type layerRef struct {
+	*layer
+	done func(bool)
+}
+
+func newWaiter() *waiter {
+	return &waiter{
+		completionCond: sync.NewCond(&sync.Mutex{}),
+	}
+}
+
+type waiter struct {
+	isDone         bool
+	isDoneMu       sync.Mutex
+	completionCond *sync.Cond
+}
+
+func (w *waiter) done() {
+	w.isDoneMu.Lock()
+	w.isDone = true
+	w.isDoneMu.Unlock()
+	w.completionCond.Broadcast()
+}
+
+func (w *waiter) wait(timeout time.Duration) error {
+	wait := func() <-chan struct{} {
+		ch := make(chan struct{})
+		go func() {
+			w.isDoneMu.Lock()
+			isDone := w.isDone
+			w.isDoneMu.Unlock()
+
+			w.completionCond.L.Lock()
+			if !isDone {
+				w.completionCond.Wait()
+			}
+			w.completionCond.L.Unlock()
+			ch <- struct{}{}
+		}()
+		return ch
+	}
+	select {
+	case <-time.After(timeout):
+		w.isDoneMu.Lock()
+		w.isDone = true
+		w.isDoneMu.Unlock()
+		w.completionCond.Broadcast()
+		return fmt.Errorf("timeout(%v)", timeout)
+	case <-wait():
+		return nil
+	}
+}
+
+type readerAtFunc func([]byte, int64) (int, error)
+
+func (f readerAtFunc) ReadAt(p []byte, offset int64) (int, error) { return f(p, offset) }

fs/layer/layer_test.go

@@ -0,0 +1,76 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright 2019 The Go Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style
+   license that can be found in the NOTICE.md file.
+*/
+
+package layer
+
+import (
+	"testing"
+	"time"
+
+	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
+)
+
+func TestLayer(t *testing.T) {
+	testRunner := &TestRunner{
+		TestingT: t,
+		Runner: func(testingT TestingT, name string, run func(t TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
+			}
+
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
+		},
+	}
+
+	TestSuiteLayer(testRunner, memorymetadata.NewReader)
+}
+
+func TestWaiter(t *testing.T) {
+	var (
+		w         = newWaiter()
+		waitTime  = time.Second
+		startTime = time.Now()
+		doneTime  time.Time
+		done      = make(chan struct{})
+	)
+
+	go func() {
+		defer close(done)
+		if err := w.wait(10 * time.Second); err != nil {
+			t.Errorf("failed to wait: %v", err)
+			return
+		}
+		doneTime = time.Now()
+	}()
+
+	time.Sleep(waitTime)
+	w.done()
+	<-done
+
+	if doneTime.Sub(startTime) < waitTime {
+		t.Errorf("wait time is too short: %v; want %v", doneTime.Sub(startTime), waitTime)
+	}
+}

fs/layer/node.go

@@ -0,0 +1,839 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+   Copyright 2019 The Go Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style
+   license that can be found in the NOTICE.md file.
+*/
+
+package layer
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/containerd/log"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	commonmetrics "github.com/containerd/stargz-snapshotter/fs/metrics/common"
+	"github.com/containerd/stargz-snapshotter/fs/reader"
+	"github.com/containerd/stargz-snapshotter/fs/remote"
+	"github.com/containerd/stargz-snapshotter/metadata"
+	fusefs "github.com/hanwen/go-fuse/v2/fs"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	digest "github.com/opencontainers/go-digest"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	blockSize         = 4096
+	physicalBlockSize = 512
+	// physicalBlockRatio is the ratio of blockSize to physicalBlockSize.
+	// It can be used to convert from # blockSize-byte blocks to # physicalBlockSize-byte blocks
+	physicalBlockRatio = blockSize / physicalBlockSize
+	whiteoutPrefix     = ".wh."
+	whiteoutOpaqueDir  = whiteoutPrefix + whiteoutPrefix + ".opq"
+	opaqueXattrValue   = "y"
+	stateDirName       = ".stargz-snapshotter"
+	statFileMode       = syscall.S_IFREG | 0400 // -r--------
+	stateDirMode       = syscall.S_IFDIR | 0500 // dr-x------
+)
+
+type OverlayOpaqueType int
+
+const (
+	OverlayOpaqueAll OverlayOpaqueType = iota
+	OverlayOpaqueTrusted
+	OverlayOpaqueUser
+)
+
+var opaqueXattrs = map[OverlayOpaqueType][]string{
+	OverlayOpaqueAll:     {"trusted.overlay.opaque", "user.overlay.opaque"},
+	OverlayOpaqueTrusted: {"trusted.overlay.opaque"},
+	OverlayOpaqueUser:    {"user.overlay.opaque"},
+}
+
+func newNode(layerDgst digest.Digest, r reader.Reader, blob remote.Blob, baseInode uint32, opaque OverlayOpaqueType, pth passThroughConfig) (fusefs.InodeEmbedder, error) {
+	rootID := r.Metadata().RootID()
+	rootAttr, err := r.Metadata().GetAttr(rootID)
+	if err != nil {
+		return nil, err
+	}
+	opq, ok := opaqueXattrs[opaque]
+	if !ok {
+		return nil, fmt.Errorf("unknown overlay opaque type")
+	}
+	ffs := &fs{
+		r:            r,
+		layerDigest:  layerDgst,
+		baseInode:    baseInode,
+		rootID:       rootID,
+		opaqueXattrs: opq,
+		passThrough:  pth,
+	}
+	ffs.s = ffs.newState(layerDgst, blob)
+	return &node{
+		id:   rootID,
+		attr: rootAttr,
+		fs:   ffs,
+	}, nil
+}
+
+// fs contains global metadata used by nodes
+type fs struct {
+	r            reader.Reader
+	s            *state
+	layerDigest  digest.Digest
+	baseInode    uint32
+	rootID       uint32
+	opaqueXattrs []string
+	passThrough  passThroughConfig
+}
+
+func (fs *fs) inodeOfState() uint64 {
+	return (uint64(fs.baseInode) << 32) | 1 // reserved
+}
+
+func (fs *fs) inodeOfStatFile() uint64 {
+	return (uint64(fs.baseInode) << 32) | 2 // reserved
+}
+
+func (fs *fs) inodeOfID(id uint32) (uint64, error) {
+	// 0 is reserved by go-fuse 1 and 2 are reserved by the state dir
+	if id > ^uint32(0)-3 {
+		return 0, fmt.Errorf("too many inodes")
+	}
+	return (uint64(fs.baseInode) << 32) | uint64(3+id), nil
+}
+
+// node is a filesystem inode abstraction.
+type node struct {
+	fusefs.Inode
+	fs   *fs
+	id   uint32
+	attr metadata.Attr
+
+	ents       []fuse.DirEntry
+	entsCached bool
+	entsMu     sync.Mutex
+}
+
+func (n *node) isRootNode() bool {
+	return n.id == n.fs.rootID
+}
+
+func (n *node) isOpaque() bool {
+	if _, _, err := n.fs.r.Metadata().GetChild(n.id, whiteoutOpaqueDir); err == nil {
+		return true
+	}
+	return false
+}
+
+var _ = (fusefs.InodeEmbedder)((*node)(nil))
+
+var _ = (fusefs.NodeReaddirer)((*node)(nil))
+
+func (n *node) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) {
+	ents, errno := n.readdir()
+	if errno != 0 {
+		return nil, errno
+	}
+	return fusefs.NewListDirStream(ents), 0
+}
+
+func (n *node) readdir() ([]fuse.DirEntry, syscall.Errno) {
+	// Measure how long node_readdir operation takes (in microseconds).
+	start := time.Now() // set start time
+	defer commonmetrics.MeasureLatencyInMicroseconds(commonmetrics.NodeReaddir, n.fs.layerDigest, start)
+
+	n.entsMu.Lock()
+	if n.entsCached {
+		ents := n.ents
+		n.entsMu.Unlock()
+		return ents, 0
+	}
+	n.entsMu.Unlock()
+
+	isRoot := n.isRootNode()
+
+	var ents []fuse.DirEntry
+	whiteouts := map[string]uint32{}
+	normalEnts := map[string]bool{}
+	var lastErr error
+	if err := n.fs.r.Metadata().ForeachChild(n.id, func(name string, id uint32, mode os.FileMode) bool {
+
+		// We don't want to show prefetch landmarks in "/".
+		if isRoot && (name == estargz.PrefetchLandmark || name == estargz.NoPrefetchLandmark) {
+			return true
+		}
+
+		// We don't want to show whiteouts.
+		if strings.HasPrefix(name, whiteoutPrefix) {
+			if name == whiteoutOpaqueDir {
+				return true
+			}
+			// Add the overlayfs-compiant whiteout later.
+			whiteouts[name] = id
+			return true
+		}
+
+		// This is a normal entry.
+		normalEnts[name] = true
+		ino, err := n.fs.inodeOfID(id)
+		if err != nil {
+			lastErr = err
+			return false
+		}
+		ents = append(ents, fuse.DirEntry{
+			Mode: fileModeToSystemMode(mode),
+			Name: name,
+			Ino:  ino,
+		})
+		return true
+	}); err != nil || lastErr != nil {
+		n.fs.s.report(fmt.Errorf("node.Readdir: err = %v; lastErr = %v", err, lastErr))
+		return nil, syscall.EIO
+	}
+
+	// Append whiteouts if no entry replaces the target entry in the lower layer.
+	for w, id := range whiteouts {
+		if !normalEnts[w[len(whiteoutPrefix):]] {
+			ino, err := n.fs.inodeOfID(id)
+			if err != nil {
+				n.fs.s.report(fmt.Errorf("node.Readdir: err = %v; lastErr = %v", err, lastErr))
+				return nil, syscall.EIO
+			}
+			ents = append(ents, fuse.DirEntry{
+				Mode: syscall.S_IFCHR,
+				Name: w[len(whiteoutPrefix):],
+				Ino:  ino,
+			})
+
+		}
+	}
+
+	// Avoid undeterministic order of entries on each call
+	sort.Slice(ents, func(i, j int) bool {
+		return ents[i].Name < ents[j].Name
+	})
+	n.entsMu.Lock()
+	defer n.entsMu.Unlock()
+	n.ents, n.entsCached = ents, true // cache it
+
+	return ents, 0
+}
+
+var _ = (fusefs.NodeLookuper)((*node)(nil))
+
+func (n *node) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) {
+
+	isRoot := n.isRootNode()
+
+	// We don't want to show prefetch landmarks in "/".
+	if isRoot && (name == estargz.PrefetchLandmark || name == estargz.NoPrefetchLandmark) {
+		return nil, syscall.ENOENT
+	}
+
+	// We don't want to show whiteouts.
+	if strings.HasPrefix(name, whiteoutPrefix) {
+		return nil, syscall.ENOENT
+	}
+
+	// state directory
+	if isRoot && name == stateDirName {
+		return n.NewInode(ctx, n.fs.s, n.fs.stateToAttr(&out.Attr)), 0
+	}
+
+	// lookup on memory nodes
+	if cn := n.GetChild(name); cn != nil {
+		switch tn := cn.Operations().(type) {
+		case *node:
+			ino, err := n.fs.inodeOfID(tn.id)
+			if err != nil {
+				n.fs.s.report(fmt.Errorf("node.Lookup: %v", err))
+				return nil, syscall.EIO
+			}
+			entryToAttr(ino, tn.attr, &out.Attr)
+		case *whiteout:
+			ino, err := n.fs.inodeOfID(tn.id)
+			if err != nil {
+				n.fs.s.report(fmt.Errorf("node.Lookup: %v", err))
+				return nil, syscall.EIO
+			}
+			entryToAttr(ino, tn.attr, &out.Attr)
+		default:
+			n.fs.s.report(fmt.Errorf("node.Lookup: uknown node type detected"))
+			return nil, syscall.EIO
+		}
+		return cn, 0
+	}
+
+	// early return if this entry doesn't exist
+	n.entsMu.Lock()
+	if n.entsCached {
+		var found bool
+		for _, e := range n.ents {
+			if e.Name == name {
+				found = true
+			}
+		}
+		if !found {
+			n.entsMu.Unlock()
+			return nil, syscall.ENOENT
+		}
+	}
+	n.entsMu.Unlock()
+
+	id, ce, err := n.fs.r.Metadata().GetChild(n.id, name)
+	if err != nil {
+		// If the entry exists as a whiteout, show an overlayfs-styled whiteout node.
+		if whID, wh, err := n.fs.r.Metadata().GetChild(n.id, fmt.Sprintf("%s%s", whiteoutPrefix, name)); err == nil {
+			ino, err := n.fs.inodeOfID(whID)
+			if err != nil {
+				n.fs.s.report(fmt.Errorf("node.Lookup: %v", err))
+				return nil, syscall.EIO
+			}
+			return n.NewInode(ctx, &whiteout{
+				id:   whID,
+				fs:   n.fs,
+				attr: wh,
+			}, entryToWhAttr(ino, wh, &out.Attr)), 0
+		}
+		n.readdir() // This code path is very expensive. Cache child entries here so that the next call don't reach here.
+		return nil, syscall.ENOENT
+	}
+
+	ino, err := n.fs.inodeOfID(id)
+	if err != nil {
+		n.fs.s.report(fmt.Errorf("node.Lookup: %v", err))
+		return nil, syscall.EIO
+	}
+	return n.NewInode(ctx, &node{
+		id:   id,
+		fs:   n.fs,
+		attr: ce,
+	}, entryToAttr(ino, ce, &out.Attr)), 0
+}
+
+var _ = (fusefs.NodeOpener)((*node)(nil))
+
+func (n *node) Open(ctx context.Context, flags uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) {
+	ra, err := n.fs.r.OpenFile(n.id)
+	if err != nil {
+		n.fs.s.report(fmt.Errorf("node.Open: %v", err))
+		return nil, 0, syscall.EIO
+	}
+
+	f := &file{
+		n:  n,
+		ra: ra,
+		fd: -1,
+	}
+
+	if n.fs.passThrough.enable {
+		if getter, ok := ra.(reader.PassthroughFdGetter); ok {
+			fd, err := getter.GetPassthroughFd(n.fs.passThrough.mergeBufferSize, n.fs.passThrough.mergeWorkerCount)
+			if err != nil {
+				n.fs.s.report(fmt.Errorf("passThrough model failed due to node.Open: %v", err))
+				n.fs.passThrough.enable = false
+			} else {
+				f.InitFd(int(fd))
+			}
+		}
+	}
+
+	return f, fuse.FOPEN_KEEP_CACHE, 0
+}
+
+var _ = (fusefs.NodeGetattrer)((*node)(nil))
+
+func (n *node) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+	ino, err := n.fs.inodeOfID(n.id)
+	if err != nil {
+		n.fs.s.report(fmt.Errorf("node.Getattr: %v", err))
+		return syscall.EIO
+	}
+	entryToAttr(ino, n.attr, &out.Attr)
+	return 0
+}
+
+var _ = (fusefs.NodeGetxattrer)((*node)(nil))
+
+func (n *node) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) {
+	ent := n.attr
+	opq := n.isOpaque()
+	for _, opaqueXattr := range n.fs.opaqueXattrs {
+		if attr == opaqueXattr && opq {
+			// This node is an opaque directory so give overlayfs-compliant indicator.
+			if len(dest) < len(opaqueXattrValue) {
+				return uint32(len(opaqueXattrValue)), syscall.ERANGE
+			}
+			return uint32(copy(dest, opaqueXattrValue)), 0
+		}
+	}
+	if v, ok := ent.Xattrs[attr]; ok {
+		if len(dest) < len(v) {
+			return uint32(len(v)), syscall.ERANGE
+		}
+		return uint32(copy(dest, v)), 0
+	}
+	return 0, syscall.ENODATA
+}
+
+var _ = (fusefs.NodeListxattrer)((*node)(nil))
+
+func (n *node) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errno) {
+	ent := n.attr
+	opq := n.isOpaque()
+	var attrs []byte
+	if opq {
+		// This node is an opaque directory so add overlayfs-compliant indicator.
+		for _, opaqueXattr := range n.fs.opaqueXattrs {
+			attrs = append(attrs, []byte(opaqueXattr+"\x00")...)
+		}
+	}
+	for k := range ent.Xattrs {
+		attrs = append(attrs, []byte(k+"\x00")...)
+	}
+	if len(dest) < len(attrs) {
+		return uint32(len(attrs)), syscall.ERANGE
+	}
+	return uint32(copy(dest, attrs)), 0
+}
+
+var _ = (fusefs.NodeReadlinker)((*node)(nil))
+
+func (n *node) Readlink(ctx context.Context) ([]byte, syscall.Errno) {
+	ent := n.attr
+	return []byte(ent.LinkName), 0
+}
+
+var _ = (fusefs.NodeStatfser)((*node)(nil))
+
+func (n *node) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno {
+	defaultStatfs(out)
+	return 0
+}
+
+// file is a file abstraction which implements file handle in go-fuse.
+type file struct {
+	n  *node
+	ra io.ReaderAt
+	fd int
+}
+
+var _ = (fusefs.FileReader)((*file)(nil))
+
+func (f *file) Read(ctx context.Context, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) {
+	defer commonmetrics.MeasureLatencyInMicroseconds(commonmetrics.ReadOnDemand, f.n.fs.layerDigest, time.Now()) // measure time for on-demand file reads (in microseconds)
+	defer commonmetrics.IncOperationCount(commonmetrics.OnDemandReadAccessCount, f.n.fs.layerDigest)             // increment the counter for on-demand file accesses
+	n, err := f.ra.ReadAt(dest, off)
+	if err != nil && err != io.EOF {
+		f.n.fs.s.report(fmt.Errorf("file.Read: %v", err))
+		return nil, syscall.EIO
+	}
+	return fuse.ReadResultData(dest[:n]), 0
+}
+
+var _ = (fusefs.FileGetattrer)((*file)(nil))
+
+func (f *file) Getattr(ctx context.Context, out *fuse.AttrOut) syscall.Errno {
+	ino, err := f.n.fs.inodeOfID(f.n.id)
+	if err != nil {
+		f.n.fs.s.report(fmt.Errorf("file.Getattr: %v", err))
+		return syscall.EIO
+	}
+	entryToAttr(ino, f.n.attr, &out.Attr)
+	return 0
+}
+
+// Implement PassthroughFd to enable go-fuse passthrough
+var _ = (fusefs.FilePassthroughFder)((*file)(nil))
+
+func (f *file) PassthroughFd() (int, bool) {
+	if f.fd <= 0 {
+		return -1, false
+	}
+	return f.fd, true
+}
+
+func (f *file) InitFd(fd int) {
+	f.fd = fd
+}
+
+// whiteout is a whiteout abstraction compliant to overlayfs.
+type whiteout struct {
+	fusefs.Inode
+	id   uint32
+	fs   *fs
+	attr metadata.Attr
+}
+
+var _ = (fusefs.NodeGetattrer)((*whiteout)(nil))
+
+func (w *whiteout) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+	ino, err := w.fs.inodeOfID(w.id)
+	if err != nil {
+		w.fs.s.report(fmt.Errorf("whiteout.Getattr: %v", err))
+		return syscall.EIO
+	}
+	entryToWhAttr(ino, w.attr, &out.Attr)
+	return 0
+}
+
+var _ = (fusefs.NodeStatfser)((*whiteout)(nil))
+
+func (w *whiteout) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno {
+	defaultStatfs(out)
+	return 0
+}
+
+// newState provides new state directory node.
+// It creates statFile at the same time to give it stable inode number.
+func (fs *fs) newState(layerDigest digest.Digest, blob remote.Blob) *state {
+	return &state{
+		statFile: &statFile{
+			name: layerDigest.String() + ".json",
+			statJSON: statJSON{
+				Digest: layerDigest.String(),
+				Size:   blob.Size(),
+			},
+			blob: blob,
+			fs:   fs,
+		},
+		fs: fs,
+	}
+}
+
+// state is a directory which contain a "state file" of this layer aiming to
+// observability. This filesystem uses it to report something(e.g. error) to
+// the clients(e.g. Kubernetes's livenessProbe).
+// This directory has mode "dr-x------ root root".
+type state struct {
+	fusefs.Inode
+	statFile *statFile
+	fs       *fs
+}
+
+var _ = (fusefs.NodeReaddirer)((*state)(nil))
+
+func (s *state) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) {
+	return fusefs.NewListDirStream([]fuse.DirEntry{
+		{
+			Mode: statFileMode,
+			Name: s.statFile.name,
+			Ino:  s.fs.inodeOfStatFile(),
+		},
+	}), 0
+}
+
+var _ = (fusefs.NodeLookuper)((*state)(nil))
+
+func (s *state) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) {
+	if name != s.statFile.name {
+		return nil, syscall.ENOENT
+	}
+	attr, errno := s.statFile.attr(&out.Attr)
+	if errno != 0 {
+		return nil, errno
+	}
+	return s.NewInode(ctx, s.statFile, attr), 0
+}
+
+var _ = (fusefs.NodeGetattrer)((*state)(nil))
+
+func (s *state) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+	s.fs.stateToAttr(&out.Attr)
+	return 0
+}
+
+var _ = (fusefs.NodeStatfser)((*state)(nil))
+
+func (s *state) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno {
+	defaultStatfs(out)
+	return 0
+}
+
+func (s *state) report(err error) {
+	s.statFile.report(err)
+}
+
+type statJSON struct {
+	Error  string `json:"error,omitempty"`
+	Digest string `json:"digest"`
+	// URL is excluded for potential security reason
+	Size           int64   `json:"size"`
+	FetchedSize    int64   `json:"fetchedSize"`
+	FetchedPercent float64 `json:"fetchedPercent"` // Fetched / Size * 100.0
+}
+
+// statFile is a file which contain something to be reported from this layer.
+// This filesystem uses statFile.report() to report something(e.g. error) to
+// the clients(e.g. Kubernetes's livenessProbe).
+// This file has mode "-r-------- root root".
+type statFile struct {
+	fusefs.Inode
+	name     string
+	blob     remote.Blob
+	statJSON statJSON
+	mu       sync.Mutex
+	fs       *fs
+}
+
+var _ = (fusefs.NodeOpener)((*statFile)(nil))
+
+func (sf *statFile) Open(ctx context.Context, flags uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) {
+	return nil, 0, 0
+}
+
+var _ = (fusefs.NodeReader)((*statFile)(nil))
+
+func (sf *statFile) Read(ctx context.Context, f fusefs.FileHandle, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) {
+	sf.mu.Lock()
+	defer sf.mu.Unlock()
+	st, err := sf.updateStatUnlocked()
+	if err != nil {
+		return nil, syscall.EIO
+	}
+	n, err := bytes.NewReader(st).ReadAt(dest, off)
+	if err != nil && err != io.EOF {
+		return nil, syscall.EIO
+	}
+	return fuse.ReadResultData(dest[:n]), 0
+}
+
+var _ = (fusefs.NodeGetattrer)((*statFile)(nil))
+
+func (sf *statFile) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+	_, errno := sf.attr(&out.Attr)
+	return errno
+}
+
+var _ = (fusefs.NodeStatfser)((*statFile)(nil))
+
+func (sf *statFile) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno {
+	defaultStatfs(out)
+	return 0
+}
+
+// logContents puts the contents of statFile in the log
+// to keep that information accessible for troubleshooting.
+// The entries naming is kept to be consistend with the field naming in statJSON.
+func (sf *statFile) logContents() {
+	ctx := context.Background()
+	log.G(ctx).WithFields(log.Fields{
+		"digest": sf.statJSON.Digest, "size": sf.statJSON.Size,
+		"fetchedSize": sf.statJSON.FetchedSize, "fetchedPercent": sf.statJSON.FetchedPercent,
+	}).WithError(errors.New(sf.statJSON.Error)).Error("statFile error")
+}
+
+func (sf *statFile) report(err error) {
+	sf.mu.Lock()
+	defer sf.mu.Unlock()
+	sf.statJSON.Error = err.Error()
+	sf.logContents()
+}
+
+func (sf *statFile) attr(out *fuse.Attr) (fusefs.StableAttr, syscall.Errno) {
+	sf.mu.Lock()
+	defer sf.mu.Unlock()
+
+	st, err := sf.updateStatUnlocked()
+	if err != nil {
+		return fusefs.StableAttr{}, syscall.EIO
+	}
+
+	return sf.fs.statFileToAttr(uint64(len(st)), out), 0
+}
+
+func (sf *statFile) updateStatUnlocked() ([]byte, error) {
+	sf.statJSON.FetchedSize = sf.blob.FetchedSize()
+	sf.statJSON.FetchedPercent = float64(sf.statJSON.FetchedSize) / float64(sf.statJSON.Size) * 100.0
+	j, err := json.Marshal(&sf.statJSON)
+	if err != nil {
+		return nil, err
+	}
+	j = append(j, []byte("\n")...)
+	return j, nil
+}
+
+// entryToAttr converts metadata.Attr to go-fuse's Attr.
+func entryToAttr(ino uint64, e metadata.Attr, out *fuse.Attr) fusefs.StableAttr {
+	out.Ino = ino
+	out.Size = uint64(e.Size)
+	if e.Mode&os.ModeSymlink != 0 {
+		out.Size = uint64(len(e.LinkName))
+	}
+	out.Blksize = blockSize
+	out.Blocks = (out.Size + uint64(out.Blksize) - 1) / uint64(out.Blksize) * physicalBlockRatio
+	mtime := e.ModTime
+	out.SetTimes(nil, &mtime, nil)
+	out.Mode = fileModeToSystemMode(e.Mode)
+	out.Owner = fuse.Owner{Uid: uint32(e.UID), Gid: uint32(e.GID)}
+	out.Rdev = uint32(unix.Mkdev(uint32(e.DevMajor), uint32(e.DevMinor)))
+	out.Nlink = uint32(e.NumLink)
+	if out.Nlink == 0 {
+		out.Nlink = 1 // zero "NumLink" means one.
+	}
+	out.Padding = 0 // TODO
+
+	return fusefs.StableAttr{
+		Mode: out.Mode,
+		Ino:  out.Ino,
+		// NOTE: The inode number is unique throughout the lifetime of
+		// this filesystem so we don't consider about generation at this
+		// moment.
+	}
+}
+
+// entryToWhAttr converts metadata.Attr to go-fuse's Attr of whiteouts.
+func entryToWhAttr(ino uint64, e metadata.Attr, out *fuse.Attr) fusefs.StableAttr {
+	out.Ino = ino
+	out.Size = 0
+	out.Blksize = blockSize
+	out.Blocks = 0
+	mtime := e.ModTime
+	out.SetTimes(nil, &mtime, nil)
+	out.Mode = syscall.S_IFCHR
+	out.Owner = fuse.Owner{Uid: 0, Gid: 0}
+	out.Rdev = uint32(unix.Mkdev(0, 0))
+	out.Nlink = 1
+	out.Padding = 0 // TODO
+
+	return fusefs.StableAttr{
+		Mode: out.Mode,
+		Ino:  out.Ino,
+		// NOTE: The inode number is unique throughout the lifetime of
+		// this filesystem so we don't consider about generation at this
+		// moment.
+	}
+}
+
+// stateToAttr converts state directory to go-fuse's Attr.
+func (fs *fs) stateToAttr(out *fuse.Attr) fusefs.StableAttr {
+	out.Ino = fs.inodeOfState()
+	out.Size = 0
+	out.Blksize = blockSize
+	out.Blocks = 0
+	out.Nlink = 1
+
+	// root can read and open it (dr-x------ root root).
+	out.Mode = stateDirMode
+	out.Owner = fuse.Owner{Uid: 0, Gid: 0}
+
+	// dummy
+	out.Mtime = 0
+	out.Mtimensec = 0
+	out.Rdev = 0
+	out.Padding = 0
+
+	return fusefs.StableAttr{
+		Mode: out.Mode,
+		Ino:  out.Ino,
+		// NOTE: The inode number is unique throughout the lifetime of
+		// this filesystem so we don't consider about generation at this
+		// moment.
+	}
+}
+
+// statFileToAttr converts stat file to go-fuse's Attr.
+// func statFileToAttr(id uint64, sf *statFile, size uint64, out *fuse.Attr) fusefs.StableAttr {
+func (fs *fs) statFileToAttr(size uint64, out *fuse.Attr) fusefs.StableAttr {
+	out.Ino = fs.inodeOfStatFile()
+	out.Size = size
+	out.Blksize = blockSize
+	out.Blocks = (out.Size + uint64(out.Blksize) - 1) / uint64(out.Blksize) * physicalBlockRatio
+	out.Nlink = 1
+
+	// Root can read it ("-r-------- root root").
+	out.Mode = statFileMode
+	out.Owner = fuse.Owner{Uid: 0, Gid: 0}
+
+	// dummy
+	out.Mtime = 0
+	out.Mtimensec = 0
+	out.Rdev = 0
+	out.Padding = 0
+
+	return fusefs.StableAttr{
+		Mode: out.Mode,
+		Ino:  out.Ino,
+		// NOTE: The inode number is unique throughout the lifetime of
+		// this filesystem so we don't consider about generation at this
+		// moment.
+	}
+}
+
+func fileModeToSystemMode(m os.FileMode) uint32 {
+	// Permission bits
+	res := uint32(m & os.ModePerm)
+
+	// File type bits
+	switch m & os.ModeType {
+	case os.ModeDevice:
+		res |= syscall.S_IFBLK
+	case os.ModeDevice | os.ModeCharDevice:
+		res |= syscall.S_IFCHR
+	case os.ModeDir:
+		res |= syscall.S_IFDIR
+	case os.ModeNamedPipe:
+		res |= syscall.S_IFIFO
+	case os.ModeSymlink:
+		res |= syscall.S_IFLNK
+	case os.ModeSocket:
+		res |= syscall.S_IFSOCK
+	default: // regular file.
+		res |= syscall.S_IFREG
+	}
+
+	// suid, sgid, sticky bits
+	if m&os.ModeSetuid != 0 {
+		res |= syscall.S_ISUID
+	}
+	if m&os.ModeSetgid != 0 {
+		res |= syscall.S_ISGID
+	}
+	if m&os.ModeSticky != 0 {
+		res |= syscall.S_ISVTX
+	}
+
+	return res
+}
+
+func defaultStatfs(stat *fuse.StatfsOut) {
+
+	// http://man7.org/linux/man-pages/man2/statfs.2.html
+	stat.Blocks = 0 // dummy
+	stat.Bfree = 0
+	stat.Bavail = 0
+	stat.Files = 0 // dummy
+	stat.Ffree = 0
+	stat.Bsize = blockSize
+	stat.NameLen = 255 // Standard max filename length for most filesystems (ext4, etc.) for compatibility
+	stat.Frsize = blockSize
+	stat.Padding = 0
+	stat.Spare = [6]uint32{}
+}

fs/metrics/common/metrics.go

@@ -0,0 +1,216 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package commonmetrics
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/containerd/log"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// OperationLatencyKeyMilliseconds is the key for stargz operation latency metrics in milliseconds.
+	OperationLatencyKeyMilliseconds = "operation_duration_milliseconds"
+
+	// OperationLatencyKeyMicroseconds is the key for stargz operation latency metrics in microseconds.
+	OperationLatencyKeyMicroseconds = "operation_duration_microseconds"
+
+	// OperationCountKey is the key for stargz operation count metrics.
+	OperationCountKey = "operation_count"
+
+	// BytesServedKey is the key for any metric related to counting bytes served as the part of specific operation.
+	BytesServedKey = "bytes_served"
+
+	// Keep namespace as stargz and subsystem as fs.
+	namespace = "stargz"
+	subsystem = "fs"
+)
+
+// Lists all metric labels.
+const (
+	// prometheus metrics
+	Mount                         = "mount"
+	RemoteRegistryGet             = "remote_registry_get"
+	NodeReaddir                   = "node_readdir"
+	StargzHeaderGet               = "stargz_header_get"
+	StargzFooterGet               = "stargz_footer_get"
+	StargzTocGet                  = "stargz_toc_get"
+	DeserializeTocJSON            = "stargz_toc_json_deserialize"
+	PrefetchesCompleted           = "all_prefetches_completed"
+	ReadOnDemand                  = "read_on_demand"
+	MountLayerToLastOnDemandFetch = "mount_layer_to_last_on_demand_fetch"
+
+	OnDemandReadAccessCount          = "on_demand_read_access_count"
+	OnDemandRemoteRegistryFetchCount = "on_demand_remote_registry_fetch_count"
+	OnDemandBytesServed              = "on_demand_bytes_served"
+	OnDemandBytesFetched             = "on_demand_bytes_fetched"
+
+	// logs metrics
+	PrefetchTotal             = "prefetch_total"
+	PrefetchDownload          = "prefetch_download"
+	PrefetchDecompress        = "prefetch_decompress"
+	BackgroundFetchTotal      = "background_fetch_total"
+	BackgroundFetchDownload   = "background_fetch_download"
+	BackgroundFetchDecompress = "background_fetch_decompress"
+	PrefetchSize              = "prefetch_size"
+)
+
+var (
+	// Buckets for OperationLatency metrics.
+	latencyBucketsMilliseconds = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384} // in milliseconds
+	latencyBucketsMicroseconds = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024}                          // in microseconds
+
+	// operationLatencyMilliseconds collects operation latency numbers in milliseconds grouped by
+	// operation, type and layer digest.
+	operationLatencyMilliseconds = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      OperationLatencyKeyMilliseconds,
+			Help:      "Latency in milliseconds of stargz snapshotter operations. Broken down by operation type and layer sha.",
+			Buckets:   latencyBucketsMilliseconds,
+		},
+		[]string{"operation_type", "layer"},
+	)
+
+	// operationLatencyMicroseconds collects operation latency numbers in microseconds grouped by
+	// operation, type and layer digest.
+	operationLatencyMicroseconds = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      OperationLatencyKeyMicroseconds,
+			Help:      "Latency in microseconds of stargz snapshotter operations. Broken down by operation type and layer sha.",
+			Buckets:   latencyBucketsMicroseconds,
+		},
+		[]string{"operation_type", "layer"},
+	)
+
+	// operationCount collects operation count numbers by operation
+	// type and layer sha.
+	operationCount = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      OperationCountKey,
+			Help:      "The count of stargz snapshotter operations. Broken down by operation type and layer sha.",
+		},
+		[]string{"operation_type", "layer"},
+	)
+
+	// bytesCount reflects the number of bytes served as the part of specitic operation type per layer sha.
+	bytesCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      BytesServedKey,
+			Help:      "The number of bytes served per stargz snapshotter operations. Broken down by operation type and layer sha.",
+		},
+		[]string{"operation_type", "layer"},
+	)
+)
+
+var register sync.Once
+var logLevel = log.DebugLevel
+
+// sinceInMilliseconds gets the time since the specified start in milliseconds.
+// The division by 1e6 is made to have the milliseconds value as floating point number, since the native method
+// .Milliseconds() returns an integer value and you can lost a precision for sub-millisecond values.
+func sinceInMilliseconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds()) / 1e6
+}
+
+// sinceInMicroseconds gets the time since the specified start in microseconds.
+// The division by 1e3 is made to have the microseconds value as floating point number, since the native method
+// .Microseconds() returns an integer value and you can lost a precision for sub-microsecond values.
+func sinceInMicroseconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds()) / 1e3
+}
+
+// Register registers metrics. This is always called only once.
+func Register(l log.Level) {
+	register.Do(func() {
+		logLevel = l
+		prometheus.MustRegister(operationLatencyMilliseconds)
+		prometheus.MustRegister(operationLatencyMicroseconds)
+		prometheus.MustRegister(operationCount)
+		prometheus.MustRegister(bytesCount)
+	})
+}
+
+// MeasureLatencyInMilliseconds wraps the labels attachment as well as calling Observe into a single method.
+// Right now we attach the operation and layer digest, so it's possible to see the breakdown for latency
+// by operation and individual layers.
+// If you want this to be layer agnostic, just pass the digest from empty string, e.g.
+// layerDigest := digest.FromString("")
+func MeasureLatencyInMilliseconds(operation string, layer digest.Digest, start time.Time) {
+	operationLatencyMilliseconds.WithLabelValues(operation, layer.String()).Observe(sinceInMilliseconds(start))
+}
+
+// MeasureLatencyInMicroseconds wraps the labels attachment as well as calling Observe into a single method.
+// Right now we attach the operation and layer digest, so it's possible to see the breakdown for latency
+// by operation and individual layers.
+// If you want this to be layer agnostic, just pass the digest from empty string, e.g.
+// layerDigest := digest.FromString("")
+func MeasureLatencyInMicroseconds(operation string, layer digest.Digest, start time.Time) {
+	operationLatencyMicroseconds.WithLabelValues(operation, layer.String()).Observe(sinceInMicroseconds(start))
+}
+
+// IncOperationCount wraps the labels attachment as well as calling Inc into a single method.
+func IncOperationCount(operation string, layer digest.Digest) {
+	operationCount.WithLabelValues(operation, layer.String()).Inc()
+}
+
+// AddBytesCount wraps the labels attachment as well as calling Add into a single method.
+func AddBytesCount(operation string, layer digest.Digest, bytes int64) {
+	bytesCount.WithLabelValues(operation, layer.String()).Add(float64(bytes))
+}
+
+// WriteLatencyLogValue wraps writing the log info record for latency in milliseconds. The log record breaks down by operation and layer digest.
+func WriteLatencyLogValue(ctx context.Context, layer digest.Digest, operation string, start time.Time) {
+	ctx = log.WithLogger(ctx, log.G(ctx).WithField("metrics", "latency").WithField("operation", operation).WithField("layer_sha", layer.String()))
+	log.G(ctx).Logf(logLevel, "value=%v milliseconds", sinceInMilliseconds(start))
+}
+
+// WriteLatencyWithBytesLogValue wraps writing the log info record for latency in milliseconds with adding the size in bytes.
+// The log record breaks down by operation, layer digest and byte value.
+func WriteLatencyWithBytesLogValue(ctx context.Context, layer digest.Digest, latencyOperation string, start time.Time, bytesMetricName string, bytesMetricValue int64) {
+	ctx = log.WithLogger(ctx, log.G(ctx).WithField("metrics", "latency").WithField("operation", latencyOperation).WithField("layer_sha", layer.String()))
+	log.G(ctx).Logf(logLevel, "value=%v milliseconds; %v=%v bytes", sinceInMilliseconds(start), bytesMetricName, bytesMetricValue)
+}
+
+// LogLatencyForLastOnDemandFetch implements a special case for measuring the latency of last on demand fetch, which must be invoked at the end of
+// background fetch operation only. Since this is expected to happen only once per container launch, it writes a log line,
+// instead of directly emitting a metric.
+// We do that in the following way:
+// 1. We record the mount start time
+// 2. We constantly record the timestamps when we do on demand fetch for each layer sha
+// 3. On background fetch completed we measure the difference between the last on demand fetch and mount start time
+// and record it as a metric
+func LogLatencyForLastOnDemandFetch(ctx context.Context, layer digest.Digest, start time.Time, end time.Time) {
+	diffInMilliseconds := float64(end.Sub(start).Milliseconds())
+	// value can be negative if we pass the default value for time.Time as `end`
+	// this can happen if there were no on-demand fetch for the particular layer
+	if diffInMilliseconds > 0 {
+		ctx = log.WithLogger(ctx, log.G(ctx).WithField("metrics", "latency").WithField("operation", MountLayerToLastOnDemandFetch).WithField("layer_sha", layer.String()))
+		log.G(ctx).Logf(logLevel, "value=%v milliseconds", diffInMilliseconds)
+	}
+}

fs/metrics/layer/layer.go

@@ -0,0 +1,65 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package layermetrics
+
+import (
+	"github.com/containerd/stargz-snapshotter/fs/layer"
+	metrics "github.com/docker/go-metrics"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var layerMetrics = []*metric{
+	{
+		name: "layer_fetched_size",
+		help: "Total fetched size of the layer",
+		unit: metrics.Bytes,
+		vt:   prometheus.CounterValue,
+		getValues: func(l layer.Layer) []value {
+			return []value{
+				{
+					v: float64(l.Info().FetchedSize),
+				},
+			}
+		},
+	},
+	{
+		name: "layer_prefetch_size",
+		help: "Total prefetched size of the layer",
+		unit: metrics.Bytes,
+		vt:   prometheus.CounterValue,
+		getValues: func(l layer.Layer) []value {
+			return []value{
+				{
+					v: float64(l.Info().PrefetchSize),
+				},
+			}
+		},
+	},
+	{
+		name: "layer_size",
+		help: "Total size of the layer",
+		unit: metrics.Bytes,
+		vt:   prometheus.CounterValue,
+		getValues: func(l layer.Layer) []value {
+			return []value{
+				{
+					v: float64(l.Info().Size),
+				},
+			}
+		},
+	},
+}

fs/metrics/layer/metrics.go

@@ -0,0 +1,113 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package layermetrics
+
+import (
+	"sync"
+
+	"github.com/containerd/stargz-snapshotter/fs/layer"
+	metrics "github.com/docker/go-metrics"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func NewLayerMetrics(ns *metrics.Namespace) *Controller {
+	if ns == nil {
+		return &Controller{}
+	}
+	c := &Controller{
+		ns:    ns,
+		layer: make(map[string]layer.Layer),
+	}
+	c.metrics = append(c.metrics, layerMetrics...)
+	ns.Add(c)
+	return c
+}
+
+type Controller struct {
+	ns      *metrics.Namespace
+	metrics []*metric
+
+	layer   map[string]layer.Layer
+	layerMu sync.RWMutex
+}
+
+func (c *Controller) Describe(ch chan<- *prometheus.Desc) {
+	for _, e := range c.metrics {
+		ch <- e.desc(c.ns)
+	}
+}
+
+func (c *Controller) Collect(ch chan<- prometheus.Metric) {
+	c.layerMu.RLock()
+	wg := &sync.WaitGroup{}
+	for mp, l := range c.layer {
+		mp, l := mp, l
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for _, e := range c.metrics {
+				e.collect(mp, l, c.ns, ch)
+			}
+		}()
+	}
+	c.layerMu.RUnlock()
+	wg.Wait()
+}
+
+func (c *Controller) Add(key string, l layer.Layer) {
+	if c.ns == nil {
+		return
+	}
+	c.layerMu.Lock()
+	c.layer[key] = l
+	c.layerMu.Unlock()
+}
+
+func (c *Controller) Remove(key string) {
+	if c.ns == nil {
+		return
+	}
+	c.layerMu.Lock()
+	delete(c.layer, key)
+	c.layerMu.Unlock()
+}
+
+type value struct {
+	v float64
+	l []string
+}
+
+type metric struct {
+	name   string
+	help   string
+	unit   metrics.Unit
+	vt     prometheus.ValueType
+	labels []string
+	// getValues returns the value and labels for the data
+	getValues func(l layer.Layer) []value
+}
+
+func (m *metric) desc(ns *metrics.Namespace) *prometheus.Desc {
+	return ns.NewDesc(m.name, m.help, m.unit, append([]string{"digest", "mountpoint"}, m.labels...)...)
+}
+
+func (m *metric) collect(mountpoint string, l layer.Layer, ns *metrics.Namespace, ch chan<- prometheus.Metric) {
+	values := m.getValues(l)
+	for _, v := range values {
+		ch <- prometheus.MustNewConstMetric(m.desc(ns), m.vt, v.v, append([]string{l.Info().Digest.String(), mountpoint}, v.l...)...)
+	}
+}

fs/reader/reader_test.go

@@ -23,382 +23,26 @@
 package reader
 
 import (
-	"archive/tar"
-	"bytes"
-	"fmt"
-	"io"
-	"strings"
-	"sync"
 	"testing"
 
-	"github.com/containerd/stargz-snapshotter/cache"
-	"github.com/containerd/stargz-snapshotter/estargz"
-	digest "github.com/opencontainers/go-digest"
+	memorymetadata "github.com/containerd/stargz-snapshotter/metadata/memory"
 )
 
-const (
-	sampleChunkSize    = 3
-	sampleMiddleOffset = sampleChunkSize / 2
-	sampleData1        = "0123456789"
-	lastChunkOffset1   = sampleChunkSize * (int64(len(sampleData1)) / sampleChunkSize)
-)
-
-// Tests Reader for failure cases.
-func TestFailReader(t *testing.T) {
-	testFileName := "test"
-	stargzFile, _ := buildStargz(t, []tarent{
-		regfile(testFileName, sampleData1),
-	}, chunkSizeInfo(sampleChunkSize))
-	br := &breakReaderAt{
-		ReaderAt: stargzFile,
-		success:  true,
-	}
-	bev := &testTOCEntryVerifier{true}
-	gr, _, err := newReader(io.NewSectionReader(br, 0, stargzFile.Size()), &nopCache{}, bev)
-	if err != nil {
-		t.Fatalf("Failed to open stargz file: %v", err)
-	}
-
-	// tests for opening file
-	_, err = gr.OpenFile("dummy")
-	if err == nil {
-		t.Errorf("succeeded to open file but wanted to fail")
-		return
-	}
-
-	fr, err := gr.OpenFile(testFileName)
-	if err != nil {
-		t.Errorf("failed to open file but wanted to succeed: %v", err)
-	}
-
-	for _, rs := range []bool{true, false} {
-		for _, vs := range []bool{true, false} {
-			br.success = rs
-			bev.success = vs
-
-			// tests for reading file
-			p := make([]byte, len(sampleData1))
-			n, err := fr.ReadAt(p, 0)
-			if rs && vs {
-				if err != nil || n != len(sampleData1) || !bytes.Equal([]byte(sampleData1), p) {
-					t.Errorf("failed to read data but wanted to succeed: %v", err)
-					return
-				}
-			} else {
-				if err == nil {
-					t.Errorf("succeeded to read data but wanted to fail (reader:%v,verify:%v)", rs, vs)
-					return
-				}
+func TestReader(t *testing.T) {
+	testRunner := &TestRunner{
+		TestingT: t,
+		Runner: func(testingT TestingT, name string, run func(t TestingT)) {
+			tt, ok := testingT.(*testing.T)
+			if !ok {
+				testingT.Fatal("TestingT is not a *testing.T")
+				return
 			}
 
-			// tests for caching reader
-			err = gr.Cache()
-			if rs && vs {
-				if err != nil {
-					t.Errorf("failed to cache reader but wanted to succeed")
-				}
-			} else {
-				if err == nil {
-					t.Errorf("succeeded to cache reader but wanted to fail (reader:%v,verify:%v)", rs, vs)
-				}
-			}
-
-		}
-	}
-}
-
-type breakReaderAt struct {
-	io.ReaderAt
-	success bool
-}
-
-func (br *breakReaderAt) ReadAt(p []byte, off int64) (int, error) {
-	if br.success {
-		return br.ReaderAt.ReadAt(p, off)
-	}
-	return 0, fmt.Errorf("failed")
-}
-
-type testTOCEntryVerifier struct {
-	success bool
-}
-
-func (bev *testTOCEntryVerifier) Verifier(ce *estargz.TOCEntry) (digest.Verifier, error) {
-	return &testVerifier{bev.success}, nil
-}
-
-type testVerifier struct {
-	success bool
-}
-
-func (bv *testVerifier) Write(p []byte) (n int, err error) {
-	return len(p), nil
-}
-
-func (bv *testVerifier) Verified() bool {
-	return bv.success
-}
-
-type nopCache struct{}
-
-func (nc *nopCache) FetchAt(key string, offset int64, p []byte, opts ...cache.Option) (int, error) {
-	return 0, fmt.Errorf("Missed cache: %q", key)
-}
-
-func (nc *nopCache) Add(key string, p []byte, opts ...cache.Option) {}
-
-type testCache struct {
-	membuf map[string]string
-	t      *testing.T
-	mu     sync.Mutex
-}
-
-func (tc *testCache) FetchAt(key string, offset int64, p []byte, opts ...cache.Option) (int, error) {
-	tc.mu.Lock()
-	defer tc.mu.Unlock()
-
-	cache, ok := tc.membuf[key]
-	if !ok {
-		return 0, fmt.Errorf("Missed cache: %q", key)
-	}
-	return copy(p, cache[offset:]), nil
-}
-
-func (tc *testCache) Add(key string, p []byte, opts ...cache.Option) {
-	tc.mu.Lock()
-	defer tc.mu.Unlock()
-	tc.membuf[key] = string(p)
-	tc.t.Logf("  cached [%s...]: %q", key[:8], string(p))
-}
-
-type region struct{ b, e int64 }
-
-// Tests ReadAt method of each file.
-func TestFileReadAt(t *testing.T) {
-	sizeCond := map[string]int64{
-		"single_chunk": sampleChunkSize - sampleMiddleOffset,
-		"multi_chunks": sampleChunkSize + sampleMiddleOffset,
-	}
-	innerOffsetCond := map[string]int64{
-		"at_top":    0,
-		"at_middle": sampleMiddleOffset,
-	}
-	baseOffsetCond := map[string]int64{
-		"of_1st_chunk":  sampleChunkSize * 0,
-		"of_2nd_chunk":  sampleChunkSize * 1,
-		"of_last_chunk": lastChunkOffset1,
-	}
-	fileSizeCond := map[string]int64{
-		"in_1_chunk_file":  sampleChunkSize * 1,
-		"in_2_chunks_file": sampleChunkSize * 2,
-		"in_max_size_file": int64(len(sampleData1)),
-	}
-	cacheCond := map[string][]region{
-		"with_clean_cache": nil,
-		"with_edge_filled_cache": {
-			region{0, sampleChunkSize - 1},
-			region{lastChunkOffset1, int64(len(sampleData1)) - 1},
-		},
-		"with_sparse_cache": {
-			region{0, sampleChunkSize - 1},
-			region{2 * sampleChunkSize, 3*sampleChunkSize - 1},
+			tt.Run(name, func(t *testing.T) {
+				run(t)
+			})
 		},
 	}
-	for sn, size := range sizeCond {
-		for in, innero := range innerOffsetCond {
-			for bo, baseo := range baseOffsetCond {
-				for fn, filesize := range fileSizeCond {
-					for cc, cacheExcept := range cacheCond {
-						t.Run(fmt.Sprintf("reading_%s_%s_%s_%s_%s", sn, in, bo, fn, cc), func(t *testing.T) {
-							if filesize > int64(len(sampleData1)) {
-								t.Fatal("sample file size is larger than sample data")
-							}
 
-							wantN := size
-							offset := baseo + innero
-							if remain := filesize - offset; remain < wantN {
-								if wantN = remain; wantN < 0 {
-									wantN = 0
-								}
-							}
-
-							// use constant string value as a data source.
-							want := strings.NewReader(sampleData1)
-
-							// data we want to get.
-							wantData := make([]byte, wantN)
-							_, err := want.ReadAt(wantData, offset)
-							if err != nil && err != io.EOF {
-								t.Fatalf("want.ReadAt (offset=%d,size=%d): %v", offset, wantN, err)
-							}
-
-							// data we get through a file.
-							f := makeFile(t, []byte(sampleData1)[:filesize], sampleChunkSize)
-							f.ra = newExceptSectionReader(t, f.ra, cacheExcept...)
-							for _, reg := range cacheExcept {
-								f.cache.Add(genID(f.digest, reg.b, reg.e-reg.b+1), []byte(sampleData1[reg.b:reg.e+1]))
-							}
-							respData := make([]byte, size)
-							n, err := f.ReadAt(respData, offset)
-							if err != nil {
-								t.Errorf("failed to read off=%d, size=%d, filesize=%d: %v", offset, size, filesize, err)
-								return
-							}
-							respData = respData[:n]
-
-							if !bytes.Equal(wantData, respData) {
-								t.Errorf("off=%d, filesize=%d; read data{size=%d,data=%q}; want (size=%d,data=%q)",
-									offset, filesize, len(respData), string(respData), wantN, string(wantData))
-								return
-							}
-
-							// check cache has valid contents.
-							cn := 0
-							nr := 0
-							for int64(nr) < wantN {
-								ce, ok := f.r.ChunkEntryForOffset(f.name, offset+int64(nr))
-								if !ok {
-									break
-								}
-								data := make([]byte, ce.ChunkSize)
-								n, err := f.cache.FetchAt(genID(f.digest, ce.ChunkOffset, ce.ChunkSize), 0, data)
-								if err != nil || n != int(ce.ChunkSize) {
-									t.Errorf("missed cache of offset=%d, size=%d: %v(got size=%d)", ce.ChunkOffset, ce.ChunkSize, err, n)
-									return
-								}
-								nr += n
-								cn++
-							}
-						})
-					}
-				}
-			}
-		}
-	}
-}
-
-type exceptSectionReader struct {
-	ra     io.ReaderAt
-	except map[region]bool
-	t      *testing.T
-}
-
-func newExceptSectionReader(t *testing.T, ra io.ReaderAt, except ...region) io.ReaderAt {
-	er := exceptSectionReader{ra: ra, t: t}
-	er.except = map[region]bool{}
-	for _, reg := range except {
-		er.except[reg] = true
-	}
-	return &er
-}
-
-func (er *exceptSectionReader) ReadAt(p []byte, offset int64) (int, error) {
-	if er.except[region{offset, offset + int64(len(p)) - 1}] {
-		er.t.Fatalf("Requested prohibited region of chunk: (%d, %d)", offset, offset+int64(len(p))-1)
-	}
-	return er.ra.ReadAt(p, offset)
-}
-
-func makeFile(t *testing.T, contents []byte, chunkSize int64) *file {
-	testName := "test"
-	sr, dgst := buildStargz(t, []tarent{
-		regfile(testName, string(contents)),
-	}, chunkSizeInfo(chunkSize))
-
-	sgz, err := estargz.Open(sr)
-	if err != nil {
-		t.Fatalf("failed to parse converted stargz: %v", err)
-	}
-	ev, err := sgz.VerifyTOC(dgst)
-	if err != nil {
-		t.Fatalf("failed to verify stargz: %v", err)
-	}
-
-	r, _, err := newReader(sr, &testCache{membuf: map[string]string{}, t: t}, ev)
-	if err != nil {
-		t.Fatalf("Failed to open stargz file: %v", err)
-	}
-	ra, err := r.OpenFile(testName)
-	if err != nil {
-		t.Fatalf("Failed to open testing file: %v", err)
-	}
-	f, ok := ra.(*file)
-	if !ok {
-		t.Fatalf("invalid type of file %q", testName)
-	}
-	return f
-}
-
-type tarent struct {
-	header   *tar.Header
-	contents []byte
-}
-
-func regfile(name string, contents string) tarent {
-	if strings.HasSuffix(name, "/") {
-		panic(fmt.Sprintf("file %q has suffix /", name))
-	}
-	return tarent{
-		header: &tar.Header{
-			Typeflag: tar.TypeReg,
-			Name:     name,
-			Mode:     0644,
-			Size:     int64(len(contents)),
-		},
-		contents: []byte(contents),
-	}
-}
-
-type chunkSizeInfo int
-
-func buildStargz(t *testing.T, ents []tarent, opts ...interface{}) (*io.SectionReader, digest.Digest) {
-	var chunkSize chunkSizeInfo
-	for _, opt := range opts {
-		if v, ok := opt.(chunkSizeInfo); ok {
-			chunkSize = v
-		} else {
-			t.Fatalf("unsupported opt")
-		}
-	}
-
-	tarBuf := new(bytes.Buffer)
-	tw := tar.NewWriter(tarBuf)
-	for _, ent := range ents {
-		if err := tw.WriteHeader(ent.header); err != nil {
-			t.Fatalf("writing header to the input tar: %v", err)
-		}
-		if _, err := tw.Write(ent.contents); err != nil {
-			t.Fatalf("writing contents to the input tar: %v", err)
-		}
-	}
-	if err := tw.Close(); err != nil {
-		t.Fatalf("closing write of input tar: %v", err)
-	}
-
-	tarData := tarBuf.Bytes()
-
-	rc, err := estargz.Build(
-		io.NewSectionReader(bytes.NewReader(tarData), 0, int64(len(tarData))),
-		estargz.WithChunkSize(int(chunkSize)),
-	)
-	if err != nil {
-		t.Fatalf("failed to build verifiable stargz: %v", err)
-	}
-	vsb := new(bytes.Buffer)
-	if _, err := io.Copy(vsb, rc); err != nil {
-		t.Fatalf("failed to copy built stargz blob: %v", err)
-	}
-	vsbb := vsb.Bytes()
-
-	return io.NewSectionReader(bytes.NewReader(vsbb), 0, int64(len(vsbb))), rc.TOCDigest()
-}
-
-func newReader(sr *io.SectionReader, cache cache.BlobCache, ev estargz.TOCEntryVerifier) (*reader, *estargz.TOCEntry, error) {
-	var r *reader
-	vr, root, err := NewReader(sr, cache)
-	if vr != nil {
-		r = vr.r
-		r.verifier = ev
-	}
-	return r, root, err
+	TestSuiteReader(testRunner, memorymetadata.NewReader)
 }

fs/remote/blob.go

@@ -23,20 +23,21 @@
 package remote
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"regexp"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/reference"
 	"github.com/containerd/stargz-snapshotter/cache"
+	"github.com/containerd/stargz-snapshotter/fs/source"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/sync/singleflight"
 )
 
 var contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`)
@@ -47,39 +48,84 @@ type Blob interface {
 	FetchedSize() int64
 	ReadAt(p []byte, offset int64, opts ...Option) (int, error)
 	Cache(offset int64, size int64, opts ...Option) error
-	Refresh(ctx context.Context, host docker.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) error
+	Refresh(ctx context.Context, host source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) error
+	Close() error
 }
 
 type blob struct {
-	fetcher   *fetcher
+	fetcher   fetcher
 	fetcherMu sync.Mutex
 
-	size          int64
-	chunkSize     int64
-	cache         cache.BlobCache
-	lastCheck     time.Time
-	lastCheckMu   sync.Mutex
-	checkInterval time.Duration
-	fetchTimeout  time.Duration
+	size              int64
+	chunkSize         int64
+	prefetchChunkSize int64
+	cache             cache.BlobCache
+	lastCheck         time.Time
+	lastCheckMu       sync.Mutex
+	checkInterval     time.Duration
+	fetchTimeout      time.Duration
 
-	fetchedRegionSet   regionSet
-	fetchedRegionSetMu sync.Mutex
+	fetchedRegionSet    regionSet
+	fetchedRegionSetMu  sync.Mutex
+	fetchedRegionGroup  singleflight.Group
+	fetchedRegionCopyMu sync.Mutex
 
 	resolver *Resolver
+
+	closed   bool
+	closedMu sync.Mutex
 }
 
-func (b *blob) Refresh(ctx context.Context, hosts docker.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) error {
+func makeBlob(fetcher fetcher, size int64, chunkSize int64, prefetchChunkSize int64,
+	blobCache cache.BlobCache, lastCheck time.Time, checkInterval time.Duration,
+	r *Resolver, fetchTimeout time.Duration) *blob {
+	return &blob{
+		fetcher:           fetcher,
+		size:              size,
+		chunkSize:         chunkSize,
+		prefetchChunkSize: prefetchChunkSize,
+		cache:             blobCache,
+		lastCheck:         lastCheck,
+		checkInterval:     checkInterval,
+		resolver:          r,
+		fetchTimeout:      fetchTimeout,
+	}
+}
+
+func (b *blob) Close() error {
+	b.closedMu.Lock()
+	defer b.closedMu.Unlock()
+	if b.closed {
+		return nil
+	}
+	b.closed = true
+	return b.cache.Close()
+}
+
+func (b *blob) isClosed() bool {
+	b.closedMu.Lock()
+	closed := b.closed
+	b.closedMu.Unlock()
+	return closed
+}
+
+func (b *blob) Refresh(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) error {
+	if b.isClosed() {
+		return fmt.Errorf("blob is already closed")
+	}
+
 	// refresh the fetcher
-	new, newSize, err := newFetcher(ctx, hosts, refspec, desc)
+	f, newSize, err := b.resolver.resolveFetcher(ctx, hosts, refspec, desc)
 	if err != nil {
 		return err
-	} else if newSize != b.size {
-		return fmt.Errorf("Invalid size of new blob %d; want %d", newSize, b.size)
+	}
+	if newSize != b.size {
+		return fmt.Errorf("invalid size of new blob %d; want %d", newSize, b.size)
 	}
 
 	// update the blob's fetcher with new one
 	b.fetcherMu.Lock()
-	b.fetcher = new
+	b.fetcher = f
 	b.fetcherMu.Unlock()
 	b.lastCheckMu.Lock()
 	b.lastCheck = time.Now()
@@ -89,6 +135,10 @@ func (b *blob) Refresh(ctx context.Context, hosts docker.RegistryHosts, refspec
 }
 
 func (b *blob) Check() error {
+	if b.isClosed() {
+		return fmt.Errorf("blob is already closed")
+	}
+
 	now := time.Now()
 	b.lastCheckMu.Lock()
 	lastCheck := b.lastCheck
@@ -123,7 +173,40 @@ func (b *blob) FetchedSize() int64 {
 	return sz
 }
 
+func makeSyncKey(allData map[region]io.Writer) string {
+	keys := make([]string, len(allData))
+	keysIndex := 0
+	for key := range allData {
+		keys[keysIndex] = fmt.Sprintf("[%d,%d]", key.b, key.e)
+		keysIndex++
+	}
+	sort.Strings(keys)
+	return strings.Join(keys, ",")
+}
+
+func (b *blob) cacheAt(offset int64, size int64, fr fetcher, cacheOpts *options) error {
+	fetchReg := region{floor(offset, b.chunkSize), ceil(offset+size-1, b.chunkSize) - 1}
+	discard := make(map[region]io.Writer)
+
+	err := b.walkChunks(fetchReg, func(reg region) error {
+		if r, err := b.cache.Get(fr.genID(reg), cacheOpts.cacheOpts...); err == nil {
+			return r.Close() // nop if the cache hits
+		}
+		discard[reg] = io.Discard
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return b.fetchRange(discard, cacheOpts)
+}
+
 func (b *blob) Cache(offset int64, size int64, opts ...Option) error {
+	if b.isClosed() {
+		return fmt.Errorf("blob is already closed")
+	}
+
 	var cacheOpts options
 	for _, o := range opts {
 		o(&cacheOpts)
@@ -133,25 +216,36 @@ func (b *blob) Cache(offset int64, size int64, opts ...Option) error {
 	fr := b.fetcher
 	b.fetcherMu.Unlock()
 
-	fetchReg := region{floor(offset, b.chunkSize), ceil(offset+size-1, b.chunkSize) - 1}
-	discard := make(map[region]io.Writer)
-	b.walkChunks(fetchReg, func(reg region) error {
-		if _, err := b.cache.FetchAt(fr.genID(reg), 0, nil, cacheOpts.cacheOpts...); err != nil {
-			discard[reg] = ioutil.Discard
-		}
-		return nil
-	})
-	if err := b.fetchRange(discard, &cacheOpts); err != nil {
-		return err
+	if b.prefetchChunkSize <= b.chunkSize {
+		return b.cacheAt(offset, size, fr, &cacheOpts)
 	}
 
-	return nil
+	eg, _ := errgroup.WithContext(context.Background())
+
+	fetchSize := b.chunkSize * (b.prefetchChunkSize / b.chunkSize)
+
+	end := offset + size
+	for i := offset; i < end; i += fetchSize {
+		i, l := i, fetchSize
+		if i+l > end {
+			l = end - i
+		}
+		eg.Go(func() error {
+			return b.cacheAt(i, l, fr, &cacheOpts)
+		})
+	}
+
+	return eg.Wait()
 }
 
 // ReadAt reads remote chunks from specified offset for the buffer size.
 // It tries to fetch as many chunks as possible from local cache.
 // We can configure this function with options.
 func (b *blob) ReadAt(p []byte, offset int64, opts ...Option) (int, error) {
+	if b.isClosed() {
+		return 0, fmt.Errorf("blob is already closed")
+	}
+
 	if len(p) == 0 || offset > b.size {
 		return 0, nil
 	}
@@ -159,26 +253,29 @@ func (b *blob) ReadAt(p []byte, offset int64, opts ...Option) (int, error) {
 	// Make the buffer chunk aligned
 	allRegion := region{floor(offset, b.chunkSize), ceil(offset+int64(len(p))-1, b.chunkSize) - 1}
 	allData := make(map[region]io.Writer)
-	var putBufs []*bytes.Buffer
-	defer func() {
-		for _, bf := range putBufs {
-			b.resolver.bufPool.Put(bf)
-		}
-	}()
 
 	var readAtOpts options
 	for _, o := range opts {
 		o(&readAtOpts)
 	}
 
-	// Fetcher can be suddenly updated so we take and use the snapshot of it for
-	// consistency.
-	b.fetcherMu.Lock()
-	fr := b.fetcher
-	b.fetcherMu.Unlock()
+	fr := b.getFetcher()
 
-	var commits []func() error
-	b.walkChunks(allRegion, func(chunk region) error {
+	if err := b.prepareChunksForRead(allRegion, offset, p, fr, allData, &readAtOpts); err != nil {
+		return 0, err
+	}
+
+	// Read required data
+	if err := b.fetchRange(allData, &readAtOpts); err != nil {
+		return 0, err
+	}
+
+	return b.adjustBufferSize(p, offset), nil
+}
+
+// prepareChunksForRead prepares chunks for reading by checking cache and setting up writers
+func (b *blob) prepareChunksForRead(allRegion region, offset int64, p []byte, fr fetcher, allData map[region]io.Writer, opts *options) error {
+	return b.walkChunks(allRegion, func(chunk region) error {
 		var (
 			base         = positive(chunk.b - offset)
 			lowerUnread  = positive(offset - chunk.b)
@@ -186,91 +283,58 @@ func (b *blob) ReadAt(p []byte, offset int64, opts ...Option) (int, error) {
 			expectedSize = chunk.size() - upperUnread - lowerUnread
 		)
 
-		// Check if the content exists in the cache
-		n, err := b.cache.FetchAt(fr.genID(chunk), lowerUnread, p[base:base+expectedSize], readAtOpts.cacheOpts...)
-		if err == nil && n == int(expectedSize) {
+		// Try to read from cache first
+		if err := b.readFromCache(chunk, p[base:base+expectedSize], lowerUnread, fr, opts); err == nil {
 			return nil
 		}
 
 		// We missed cache. Take it from remote registry.
 		// We get the whole chunk here and add it to the cache so that following
 		// reads against neighboring chunks can take the data without making HTTP requests.
-		if lowerUnread == 0 && upperUnread == 0 {
-			// We can directly store the result in the given buffer
-			allData[chunk] = &byteWriter{
-				p: p[base : base+chunk.size()],
-			}
-		} else {
-			// Use temporally buffer for aligning this chunk
-			bf := b.resolver.bufPool.Get().(*bytes.Buffer)
-			putBufs = append(putBufs, bf)
-			bf.Reset()
-			bf.Grow(int(chunk.size()))
-			allData[chunk] = bf
-
-			// Function for committing the buffered chunk into the result slice.
-			commits = append(commits, func() error {
-				if int64(bf.Len()) != chunk.size() {
-					return fmt.Errorf("unexpected data size %d; want %d",
-						bf.Len(), chunk.size())
-				}
-				bb := bf.Bytes()[:chunk.size()]
-				n := copy(p[base:], bb[lowerUnread:chunk.size()-upperUnread])
-				if int64(n) != expectedSize {
-					return fmt.Errorf("invalid copied data size %d; want %d",
-						n, expectedSize)
-				}
-				return nil
-			})
-		}
+		allData[chunk] = newBytesWriter(p[base:base+expectedSize], lowerUnread)
 		return nil
 	})
-
-	// Read required data
-	if err := b.fetchRange(allData, &readAtOpts); err != nil {
-		return 0, err
-	}
-
-	// Write all data to the result buffer
-	for _, c := range commits {
-		if err := c(); err != nil {
-			return 0, err
-		}
-	}
-
-	// Adjust the buffer size according to the blob size
-	if remain := b.size - offset; int64(len(p)) >= remain {
-		if remain < 0 {
-			remain = 0
-		}
-		p = p[:remain]
-	}
-
-	return len(p), nil
 }
 
-// fetchRange fetches all specified chunks from local cache and remote blob.
-func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
+// readFromCache attempts to read chunk data from cache
+func (b *blob) readFromCache(chunk region, dest []byte, offset int64, fr fetcher, opts *options) error {
+	r, err := b.cache.Get(fr.genID(chunk), opts.cacheOpts...)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	n, err := r.ReadAt(dest, offset)
+	if err != nil && err != io.EOF {
+		return err
+	}
+	if n != len(dest) {
+		return fmt.Errorf("incomplete read from cache: read %d bytes, expected %d bytes", n, len(dest))
+	}
+	return nil
+}
+
+// fetchRegions fetches all specified chunks from remote blob and puts it in the local cache.
+// It must be called from within fetchRange and need to ensure that it is inside the singleflight `Do` operation.
+func (b *blob) fetchRegions(allData map[region]io.Writer, fetched map[region]bool, opts *options) error {
 	if len(allData) == 0 {
 		return nil
 	}
 
-	// Fetcher can be suddenly updated so we take and use the snapshot of it for
-	// consistency.
-	b.fetcherMu.Lock()
-	fr := b.fetcher
-	b.fetcherMu.Unlock()
+	fr := b.getFetcher()
 
 	// request missed regions
 	var req []region
-	fetched := make(map[region]bool)
 	for reg := range allData {
 		req = append(req, reg)
 		fetched[reg] = false
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), b.fetchTimeout)
+
+	fetchCtx, cancel := context.WithTimeout(context.Background(), b.fetchTimeout)
 	defer cancel()
-	mr, err := fr.fetch(ctx, req, true, opts)
+	if opts.ctx != nil {
+		fetchCtx = opts.ctx
+	}
+	mr, err := fr.fetch(fetchCtx, req, true)
 	if err != nil {
 		return err
 	}
@@ -288,40 +352,15 @@ func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
 		if err == io.EOF {
 			break
 		} else if err != nil {
-			return errors.Wrapf(err, "failed to read multipart resp")
+			return fmt.Errorf("failed to read multipart resp: %w", err)
 		}
-		if err := b.walkChunks(reg, func(chunk region) error {
-
-			// Prepare the temporary buffer
-			bf := b.resolver.bufPool.Get().(*bytes.Buffer)
-			defer b.resolver.bufPool.Put(bf)
-			bf.Reset()
-			bf.Grow(int(chunk.size()))
-			w := io.Writer(bf)
-
-			// If this chunk is one of the targets, write the content to the
-			// passed reader too.
-			if _, ok := fetched[chunk]; ok {
-				w = io.MultiWriter(bf, allData[chunk])
-			}
-
-			// Copy the target chunk
-			if _, err := io.CopyN(w, p, chunk.size()); err != nil {
+		if err := b.walkChunks(reg, func(chunk region) (retErr error) {
+			if err := b.cacheChunkData(chunk, p, fr, allData, fetched, opts); err != nil {
 				return err
-			} else if int64(bf.Len()) != chunk.size() {
-				return fmt.Errorf("unexpected fetched data size %d; want %d",
-					bf.Len(), chunk.size())
 			}
-
-			// Add the target chunk to the cache
-			b.cache.Add(fr.genID(chunk), bf.Bytes()[:chunk.size()], opts.cacheOpts...)
-			b.fetchedRegionSetMu.Lock()
-			b.fetchedRegionSet.add(chunk)
-			b.fetchedRegionSetMu.Unlock()
-			fetched[chunk] = true
 			return nil
 		}); err != nil {
-			return errors.Wrapf(err, "failed to get chunks")
+			return fmt.Errorf("failed to get chunks: %w", err)
 		}
 	}
 
@@ -339,6 +378,81 @@ func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
 	return nil
 }
 
+// fetchRange fetches all specified chunks from local cache and remote blob.
+func (b *blob) fetchRange(allData map[region]io.Writer, opts *options) error {
+	if len(allData) == 0 {
+		return nil
+	}
+
+	key := makeSyncKey(allData)
+	fetched := make(map[region]bool)
+	_, err, shared := b.fetchedRegionGroup.Do(key, func() (interface{}, error) {
+		return nil, b.fetchRegions(allData, fetched, opts)
+	})
+
+	// When unblocked try to read from cache in case if there were no errors
+	// If we fail reading from cache, fetch from remote registry again
+	if err == nil && shared {
+		if err := b.handleSharedFetch(allData, fetched, opts); err != nil {
+			return b.fetchRange(allData, opts) // retry on error
+		}
+	}
+
+	return err
+}
+
+// handleSharedFetch handles the case when multiple goroutines share the same fetch result
+func (b *blob) handleSharedFetch(allData map[region]io.Writer, fetched map[region]bool, opts *options) error {
+	for reg := range allData {
+		if _, ok := fetched[reg]; ok {
+			continue
+		}
+		if err := b.copyFetchedChunks(reg, allData, opts); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// copyFetchedChunks copies fetched chunks from cache to target writer
+func (b *blob) copyFetchedChunks(reg region, allData map[region]io.Writer, opts *options) error {
+	return b.walkChunks(reg, func(chunk region) error {
+		fr := b.getFetcher()
+		r, err := b.cache.Get(fr.genID(chunk), opts.cacheOpts...)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+
+		b.fetchedRegionCopyMu.Lock()
+		defer b.fetchedRegionCopyMu.Unlock()
+
+		if _, err := io.CopyN(allData[chunk], io.NewSectionReader(r, 0, chunk.size()), chunk.size()); err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+// getFetcher safely gets the current fetcher
+// Fetcher can be suddenly updated so we take and use the snapshot of it for consistency.
+func (b *blob) getFetcher() fetcher {
+	b.fetcherMu.Lock()
+	defer b.fetcherMu.Unlock()
+	return b.fetcher
+}
+
+// adjustBufferSize adjusts buffer size according to the blob size
+func (b *blob) adjustBufferSize(p []byte, offset int64) int {
+	if remain := b.size - offset; int64(len(p)) >= remain {
+		if remain < 0 {
+			remain = 0
+		}
+		p = p[:remain]
+	}
+	return len(p)
+}
+
 type walkFunc func(reg region) error
 
 // walkChunks walks chunks from begin to end in order in the specified region.
@@ -360,15 +474,42 @@ func (b *blob) walkChunks(allRegion region, walkFn walkFunc) error {
 	return nil
 }
 
-type byteWriter struct {
-	p []byte
-	n int
+func newBytesWriter(dest []byte, destOff int64) io.Writer {
+	return &bytesWriter{
+		dest:    dest,
+		destOff: destOff,
+		current: 0,
+	}
 }
 
-func (w *byteWriter) Write(p []byte) (int, error) {
-	n := copy(w.p[w.n:], p)
-	w.n += n
-	return n, nil
+type bytesWriter struct {
+	dest    []byte
+	destOff int64
+	current int64
+}
+
+func (bw *bytesWriter) Write(p []byte) (int, error) {
+	defer func() { bw.current = bw.current + int64(len(p)) }()
+
+	var (
+		destBase = positive(bw.current - bw.destOff)
+		pBegin   = positive(bw.destOff - bw.current)
+		pEnd     = positive(bw.destOff + int64(len(bw.dest)) - bw.current)
+	)
+
+	if destBase > int64(len(bw.dest)) {
+		return len(p), nil
+	}
+	if pBegin >= int64(len(p)) {
+		return len(p), nil
+	}
+	if pEnd > int64(len(p)) {
+		pEnd = int64(len(p))
+	}
+
+	copy(bw.dest[destBase:], p[pBegin:pEnd])
+
+	return len(p), nil
 }
 
 func floor(n int64, unit int64) int64 {
@@ -385,3 +526,34 @@ func positive(n int64) int64 {
 	}
 	return n
 }
+
+// cacheChunkData handles caching of chunk data
+func (b *blob) cacheChunkData(chunk region, r io.Reader, fr fetcher, allData map[region]io.Writer, fetched map[region]bool, opts *options) error {
+	id := fr.genID(chunk)
+	cw, err := b.cache.Add(id, opts.cacheOpts...)
+	if err != nil {
+		return fmt.Errorf("failed to create cache writer: %w", err)
+	}
+	defer cw.Close()
+
+	w := io.Writer(cw)
+	if _, ok := fetched[chunk]; ok {
+		w = io.MultiWriter(w, allData[chunk])
+	}
+
+	if _, err := io.CopyN(w, r, chunk.size()); err != nil {
+		cw.Abort()
+		return fmt.Errorf("failed to write chunk data: %w", err)
+	}
+
+	if err := cw.Commit(); err != nil {
+		return fmt.Errorf("failed to commit chunk: %w", err)
+	}
+
+	b.fetchedRegionSetMu.Lock()
+	b.fetchedRegionSet.add(chunk)
+	b.fetchedRegionSetMu.Unlock()
+	fetched[chunk] = true
+
+	return nil
+}

fs/remote/blob_test.go

@@ -26,7 +26,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"mime/multipart"
 	"net/http"
@@ -35,6 +34,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -42,12 +42,13 @@ import (
 )
 
 const (
-	testURL            = "http://testdummy.com/v2/library/test/blobs/sha256:deadbeaf"
-	rangeHeaderPrefix  = "bytes="
-	sampleChunkSize    = 3
-	sampleMiddleOffset = sampleChunkSize / 2
-	sampleData1        = "0123456789"
-	lastChunkOffset1   = sampleChunkSize * (int64(len(sampleData1)) / sampleChunkSize)
+	testURL                  = "http://testdummy.com/v2/library/test/blobs/sha256:deadbeaf"
+	rangeHeaderPrefix        = "bytes="
+	sampleChunkSize          = 3
+	sampleMiddleOffset       = sampleChunkSize / 2
+	sampleData1              = "0123456789"
+	lastChunkOffset1         = sampleChunkSize * (int64(len(sampleData1)) / sampleChunkSize)
+	defaultPrefetchChunkSize = 0
 )
 
 // Tests ReadAt and Cache method of each file.
@@ -70,6 +71,10 @@ func TestReadAt(t *testing.T) {
 		"in_3_chunks_blob": sampleChunkSize * 3,
 		"in_max_size_blob": int64(len(sampleData1)),
 	}
+	prefetchChunkSizeCond := map[string]int64{
+		"single_get_prefetch":   0,
+		"multiple_get_prefetch": sampleChunkSize * 2,
+	}
 	type cacheCond struct {
 		reg     region
 		mustHit bool
@@ -120,54 +125,57 @@ func TestReadAt(t *testing.T) {
 		for in, innero := range innerOffsetCond {
 			for bo, baseo := range baseOffsetCond {
 				for bs, blobsize := range blobSizeCond {
-					for tc, trCond := range transportCond {
-						t.Run(fmt.Sprintf("reading_%s_%s_%s_%s_%s", sn, in, bo, bs, tc), func(t *testing.T) {
-							if blobsize > int64(len(sampleData1)) {
-								t.Fatal("sample file size is larger than sample data")
-							}
-
-							wantN := size
-							offset := baseo + innero
-							if remain := blobsize - offset; remain < wantN {
-								if wantN = remain; wantN < 0 {
-									wantN = 0
+					for pc, prefetchchunksize := range prefetchChunkSizeCond {
+						for tc, trCond := range transportCond {
+							t.Run(fmt.Sprintf("reading_%s_%s_%s_%s_%s_%s", sn, in, bo, bs, pc, tc), func(t *testing.T) {
+								if blobsize > int64(len(sampleData1)) {
+									t.Fatal("sample file size is larger than sample data")
 								}
-							}
 
-							// use constant string value as a data source.
-							want := strings.NewReader(sampleData1)
-
-							// data we want to get.
-							wantData := make([]byte, wantN)
-							_, err := want.ReadAt(wantData, offset)
-							if err != nil && err != io.EOF {
-								t.Fatalf("want.ReadAt (offset=%d,size=%d): %v", offset, wantN, err)
-							}
-
-							// data we get through a remote blob.
-							blob := []byte(sampleData1)[:blobsize]
-
-							// Check with allowing multi range requests
-							var cacheChunks []region
-							var except []region
-							for _, cond := range trCond.cacheCond {
-								cacheChunks = append(cacheChunks, cond.reg)
-								if cond.mustHit {
-									except = append(except, cond.reg)
+								wantN := size
+								offset := baseo + innero
+								if remain := blobsize - offset; remain < wantN {
+									if wantN = remain; wantN < 0 {
+										wantN = 0
+									}
 								}
-							}
-							tr := multiRoundTripper(t, blob, allowMultiRange(trCond.allowMultiRange), exceptChunks(except))
 
-							// Check ReadAt method
-							bb1 := makeBlob(t, blobsize, sampleChunkSize, tr)
-							cacheAll(bb1, cacheChunks)
-							checkRead(t, wantData, bb1, offset, size)
+								// use constant string value as a data source.
+								want := strings.NewReader(sampleData1)
+
+								// data we want to get.
+								wantData := make([]byte, wantN)
+								_, err := want.ReadAt(wantData, offset)
+								if err != nil && err != io.EOF {
+									t.Fatalf("want.ReadAt (offset=%d,size=%d): %v", offset, wantN, err)
+								}
+
+								// data we get through a remote blob.
+								blob := []byte(sampleData1)[:blobsize]
+
+								// Check with allowing multi range requests
+								var cacheChunks []region
+								var except []region
+								for _, cond := range trCond.cacheCond {
+									cacheChunks = append(cacheChunks, cond.reg)
+									if cond.mustHit {
+										except = append(except, cond.reg)
+									}
+								}
+								tr := multiRoundTripper(t, blob, allowMultiRange(trCond.allowMultiRange), exceptChunks(except))
+
+								// Check ReadAt method
+								bb1 := makeTestBlob(t, blobsize, sampleChunkSize, prefetchchunksize, tr)
+								cacheAll(t, bb1, cacheChunks)
+								checkRead(t, wantData, bb1, offset, size)
+
+								// Check Cache method
+								bb2 := makeTestBlob(t, blobsize, sampleChunkSize, prefetchchunksize, tr)
+								cacheAll(t, bb2, cacheChunks)
+								checkCache(t, bb2, offset, size)
+							})
+						}
 
-							// Check Cache method
-							bb2 := makeBlob(t, blobsize, sampleChunkSize, tr)
-							cacheAll(bb2, cacheChunks)
-							checkCache(t, bb2, offset, size)
-						})
 					}
 				}
 			}
@@ -175,9 +183,23 @@ func TestReadAt(t *testing.T) {
 	}
 }
 
-func cacheAll(b *blob, chunks []region) {
+func cacheAll(t *testing.T, b *blob, chunks []region) {
 	for _, reg := range chunks {
-		b.cache.Add(b.fetcher.genID(reg), []byte(sampleData1[reg.b:reg.e+1]))
+		id := b.fetcher.genID(reg)
+		w, err := b.cache.Add(id)
+		if err != nil {
+			w.Close()
+			t.Fatalf("failed to add cache %v: %v", id, err)
+		}
+		if _, err := w.Write([]byte(sampleData1[reg.b : reg.e+1])); err != nil {
+			w.Close()
+			t.Fatalf("failed to write cache %v: %v", id, err)
+		}
+		if err := w.Commit(); err != nil {
+			w.Close()
+			t.Fatalf("failed to commit cache %v: %v", id, err)
+		}
+		w.Close()
 	}
 }
 
@@ -216,10 +238,15 @@ func checkAllCached(t *testing.T, r *blob, offset, size int64) {
 	whole := region{floor(offset, r.chunkSize), ceil(offset+size-1, r.chunkSize) - 1}
 	if err := r.walkChunks(whole, func(reg region) error {
 		data := make([]byte, reg.size())
-		n, err := r.cache.FetchAt(r.fetcher.genID(reg), 0, data)
-		if err != nil || int64(n) != reg.size() {
-			return fmt.Errorf("missed cache of region={%d,%d}(size=%d): %v",
-				reg.b, reg.e, reg.size(), err)
+		id := r.fetcher.genID(reg)
+
+		r, err := r.cache.Get(id)
+		if err != nil {
+			return fmt.Errorf("missed cache of region={%d,%d}(size=%d): %v", reg.b, reg.e, reg.size(), err)
+		}
+		defer r.Close()
+		if n, err := r.ReadAt(data, 0); (err != nil && err != io.EOF) || int64(n) != reg.size() {
+			return fmt.Errorf("failed to read cache of region={%d,%d}(size=%d): %v", reg.b, reg.e, reg.size(), err)
 		}
 		cn++
 		return nil
@@ -233,7 +260,7 @@ func checkAllCached(t *testing.T, r *blob, offset, size int64) {
 func TestFailReadAt(t *testing.T) {
 
 	// test failed http respose.
-	r := makeBlob(t, int64(len(sampleData1)), sampleChunkSize, failRoundTripper())
+	r := makeTestBlob(t, int64(len(sampleData1)), sampleChunkSize, defaultPrefetchChunkSize, failRoundTripper())
 	respData := make([]byte, len(sampleData1))
 	_, err := r.ReadAt(respData, 0)
 	if err == nil || err == io.EOF {
@@ -252,12 +279,12 @@ func TestFailReadAt(t *testing.T) {
 
 func checkBrokenBody(t *testing.T, allowMultiRange bool) {
 	respData := make([]byte, len(sampleData1))
-	r := makeBlob(t, int64(len(sampleData1)), sampleChunkSize, brokenBodyRoundTripper(t, []byte(sampleData1), allowMultiRange))
+	r := makeTestBlob(t, int64(len(sampleData1)), sampleChunkSize, defaultPrefetchChunkSize, brokenBodyRoundTripper(t, []byte(sampleData1), allowMultiRange))
 	if _, err := r.ReadAt(respData, 0); err == nil || err == io.EOF {
 		t.Errorf("must be fail for broken full body but err=%v (allowMultiRange=%v)", err, allowMultiRange)
 		return
 	}
-	r = makeBlob(t, int64(len(sampleData1)), sampleChunkSize, brokenBodyRoundTripper(t, []byte(sampleData1), allowMultiRange))
+	r = makeTestBlob(t, int64(len(sampleData1)), sampleChunkSize, defaultPrefetchChunkSize, brokenBodyRoundTripper(t, []byte(sampleData1), allowMultiRange))
 	if _, err := r.ReadAt(respData[0:len(sampleData1)/2], 0); err == nil || err == io.EOF {
 		t.Errorf("must be fail for broken multipart body but err=%v (allowMultiRange=%v)", err, allowMultiRange)
 		return
@@ -265,7 +292,7 @@ func checkBrokenBody(t *testing.T, allowMultiRange bool) {
 }
 
 func checkBrokenHeader(t *testing.T, allowMultiRange bool) {
-	r := makeBlob(t, int64(len(sampleData1)), sampleChunkSize, brokenHeaderRoundTripper(t, []byte(sampleData1), allowMultiRange))
+	r := makeTestBlob(t, int64(len(sampleData1)), sampleChunkSize, defaultPrefetchChunkSize, brokenHeaderRoundTripper(t, []byte(sampleData1), allowMultiRange))
 	respData := make([]byte, len(sampleData1))
 	if _, err := r.ReadAt(respData[0:len(sampleData1)/2], 0); err == nil || err == io.EOF {
 		t.Errorf("must be fail for broken multipart header but err=%v (allowMultiRange=%v)", err, allowMultiRange)
@@ -273,48 +300,283 @@ func checkBrokenHeader(t *testing.T, allowMultiRange bool) {
 	}
 }
 
-func makeBlob(t *testing.T, size int64, chunkSize int64, fn RoundTripFunc) *blob {
-	return &blob{
-		fetcher: &fetcher{
+func TestParallelDownloadingBehavior(t *testing.T) {
+	type regionsBoundaries struct {
+		regions []region
+		start   int64
+		end     int64
+	}
+
+	type testData struct {
+		name           string
+		regions        [3]regionsBoundaries
+		roundtripCount int64
+		chunkSize      int64
+		content        string
+	}
+
+	tests := []testData{
+		{
+			name:           "no_data",
+			regions:        [3]regionsBoundaries{},
+			roundtripCount: 0,
+			chunkSize:      4,
+		},
+		{
+			name: "same_regions",
+			regions: [3]regionsBoundaries{
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+					},
+					start: 0,
+					end:   3,
+				},
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+					},
+					start: 0,
+					end:   3,
+				},
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+					},
+					start: 0,
+					end:   3,
+				},
+			},
+			roundtripCount: 1,
+			chunkSize:      4,
+			content:        "test",
+		},
+		{
+			name: "same_regions_multiple_values",
+			regions: [3]regionsBoundaries{
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+						{
+							b: 4,
+							e: 7,
+						},
+					},
+					start: 0,
+					end:   7,
+				},
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+						{
+							b: 4,
+							e: 7,
+						},
+					},
+					start: 0,
+					end:   7,
+				},
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+						{
+							b: 4,
+							e: 7,
+						},
+					},
+					start: 0,
+					end:   7,
+				},
+			},
+			roundtripCount: 1,
+			chunkSize:      4,
+			content:        "test1234",
+		},
+		{
+			name: "different_regions",
+			regions: [3]regionsBoundaries{
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+					},
+					start: 0,
+					end:   3,
+				},
+				{
+					regions: []region{
+						{
+							b: 4,
+							e: 7,
+						},
+					},
+					start: 4,
+					end:   7,
+				},
+				{
+					regions: []region{
+						{
+							b: 8,
+							e: 11,
+						},
+					},
+					start: 8,
+					end:   11,
+				},
+			},
+			roundtripCount: 3,
+			chunkSize:      4,
+			content:        "test12345678",
+		},
+		{
+			name: "some_overlap",
+			regions: [3]regionsBoundaries{
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+					},
+					start: 0,
+					end:   3,
+				},
+				{
+					regions: []region{
+						{
+							b: 0,
+							e: 3,
+						},
+					},
+					start: 0,
+					end:   3,
+				},
+				{
+					regions: []region{
+						{
+							b: 4,
+							e: 7,
+						},
+					},
+					start: 4,
+					end:   7,
+				},
+			},
+			roundtripCount: 2,
+			chunkSize:      4,
+			content:        "test1234",
+		},
+	}
+
+	var wg sync.WaitGroup
+	// we always run 3 routines
+	routines := 3
+
+	for _, tst := range tests {
+		var (
+			tr = &callsCountRoundTripper{
+				content: tst.content,
+			}
+			b = &blob{
+				fetcher: &httpFetcher{
+					url: "test",
+					tr:  tr,
+				},
+				chunkSize: tst.chunkSize,
+				size:      int64(len(tst.content)),
+				cache:     cache.NewMemoryCache(),
+			}
+		)
+
+		start := make(chan struct{})
+		wg.Add(routines)
+		var contentBytes [3][]byte
+
+		for i := 0; i < routines; i++ {
+			p := make([]byte, len(tst.content))
+			contentBytes[i] = p
+			allData := make(map[region]io.Writer)
+			if i < len(tst.regions) {
+				offset := int64(0)
+				for j := range tst.regions[i].regions {
+					r := tst.regions[i].regions[j]
+					var (
+						base         = positive(r.b - offset)
+						lowerUnread  = positive(offset - r.b)
+						upperUnread  = positive(r.e + 1 - (offset + int64(len(p))))
+						expectedSize = r.size() - upperUnread - lowerUnread
+					)
+					allData[tst.regions[i].regions[j]] = newBytesWriter(p[base:base+expectedSize], lowerUnread)
+				}
+			}
+
+			go func() {
+				<-start // by blocking on channel start we can ensure that the goroutines will run at approximately the same time
+				defer wg.Done()
+				b.fetchRange(allData, &options{})
+			}()
+		}
+		close(start) // starting
+		wg.Wait()
+
+		// We expect the number of round trip calls to be 1, since we are making 5 calls to fetchRange with
+		// overlapping intervals.
+		if tr.count != tst.roundtripCount {
+			t.Errorf("%v test failed: the round trip count should be %v, but was %v", tst.name, tst.roundtripCount, tr.count)
+		}
+		// Check for contents
+		for j := range contentBytes {
+			start := tst.regions[j].start
+			end := tst.regions[j].end
+			for i := start; i < end; i++ {
+				if contentBytes[j][i] != []byte(tst.content)[i] {
+					t.Errorf("%v test failed: the output sequence is wrong, wanted %v, got %v", tst.name, []byte(tst.content)[start:end], contentBytes[j][start:end])
+					break
+				}
+			}
+		}
+	}
+}
+
+func makeTestBlob(t *testing.T, size int64, chunkSize int64, prefetchChunkSize int64, fn RoundTripFunc) *blob {
+	var (
+		lastCheck     time.Time
+		checkInterval time.Duration
+	)
+
+	return makeBlob(
+		&httpFetcher{
 			url: testURL,
 			tr:  fn,
 		},
-		size:      size,
-		chunkSize: chunkSize,
-		cache:     &testCache{membuf: map[string]string{}, t: t},
-		resolver: &Resolver{
-			bufPool: sync.Pool{
-				New: func() interface{} {
-					return new(bytes.Buffer)
-				},
-			},
-		},
-		fetchTimeout: time.Duration(defaultFetchTimeoutSec) * time.Second,
-	}
-}
-
-type testCache struct {
-	membuf map[string]string
-	t      *testing.T
-	mu     sync.Mutex
-}
-
-func (tc *testCache) FetchAt(key string, offset int64, p []byte, opts ...cache.Option) (int, error) {
-	tc.mu.Lock()
-	defer tc.mu.Unlock()
-
-	cache, ok := tc.membuf[key]
-	if !ok {
-		return 0, fmt.Errorf("Missed cache: %q", key)
-	}
-	return copy(p, cache[offset:]), nil
-}
-
-func (tc *testCache) Add(key string, p []byte, opts ...cache.Option) {
-	tc.mu.Lock()
-	defer tc.mu.Unlock()
-	tc.membuf[key] = string(p)
-	tc.t.Logf("  cached [%s...]: %q", key[:8], string(p))
+		size,
+		chunkSize,
+		prefetchChunkSize,
+		cache.NewMemoryCache(),
+		lastCheck,
+		checkInterval,
+		&Resolver{},
+		time.Duration(defaultFetchTimeoutSec)*time.Second)
 }
 
 func TestCheckInterval(t *testing.T) {
@@ -322,7 +584,7 @@ func TestCheckInterval(t *testing.T) {
 		tr        = &calledRoundTripper{}
 		firstTime = time.Now()
 		b         = &blob{
-			fetcher: &fetcher{
+			fetcher: &httpFetcher{
 				url: "test",
 				tr:  tr,
 			},
@@ -347,7 +609,7 @@ func TestCheckInterval(t *testing.T) {
 		if !tr.called {
 			return b.lastCheck, false
 		}
-		if !(b.lastCheck.After(beforeUpdate) && b.lastCheck.Before(afterUpdate)) {
+		if !b.lastCheck.After(beforeUpdate) || !b.lastCheck.Before(afterUpdate) {
 			t.Errorf("%q: updated time must be after %q and before %q but %q", name, beforeUpdate, afterUpdate, b.lastCheck)
 		}
 
@@ -369,6 +631,24 @@ func TestCheckInterval(t *testing.T) {
 	}
 }
 
+type callsCountRoundTripper struct {
+	count   int64
+	content string
+}
+
+func (c *callsCountRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
+	atomic.AddInt64(&c.count, 1)
+	time.Sleep(50 * time.Millisecond) // sleep for 50 milliseconds to emulate the http call and to make sure that we can run tests on parallel goroutines
+	convertBody := func(r io.ReadCloser) io.ReadCloser { return r }
+	header := make(http.Header)
+	header.Add("Content-Length", fmt.Sprintf("%d", len(c.content)))
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Header:     header,
+		Body:       convertBody(io.NopCloser(bytes.NewReader([]byte(c.content)))),
+	}, nil
+}
+
 type calledRoundTripper struct {
 	called bool
 }
@@ -378,7 +658,7 @@ func (c *calledRoundTripper) RoundTrip(req *http.Request) (res *http.Response, e
 	res = &http.Response{
 		StatusCode: http.StatusOK,
 		Header:     make(http.Header),
-		Body:       ioutil.NopCloser(bytes.NewReader([]byte("test"))),
+		Body:       io.NopCloser(bytes.NewReader([]byte("test"))),
 	}
 	return
 }
@@ -410,7 +690,7 @@ func multiRoundTripper(t *testing.T, contents []byte, opts ...interface{}) Round
 		return &http.Response{
 			StatusCode: statusCode,
 			Header:     make(http.Header),
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
 		}
 	}
 
@@ -459,7 +739,7 @@ func multiRoundTripper(t *testing.T, contents []byte, opts ...interface{}) Round
 				return &http.Response{
 					StatusCode: http.StatusOK,
 					Header:     header,
-					Body:       convertBody(ioutil.NopCloser(bytes.NewReader(contents))),
+					Body:       convertBody(io.NopCloser(bytes.NewReader(contents))),
 				}
 			}
 		}
@@ -487,7 +767,7 @@ func multiRoundTripper(t *testing.T, contents []byte, opts ...interface{}) Round
 			return &http.Response{
 				StatusCode: http.StatusPartialContent,
 				Header:     header,
-				Body:       convertBody(ioutil.NopCloser(bytes.NewReader(part))),
+				Body:       convertBody(io.NopCloser(bytes.NewReader(part))),
 			}
 		}
 
@@ -528,7 +808,7 @@ func multiRoundTripper(t *testing.T, contents []byte, opts ...interface{}) Round
 		return &http.Response{
 			StatusCode: http.StatusPartialContent,
 			Header:     header,
-			Body:       convertBody(ioutil.NopCloser(&buf)),
+			Body:       convertBody(io.NopCloser(&buf)),
 		}
 	}
 }
@@ -538,7 +818,7 @@ func failRoundTripper() RoundTripFunc {
 		return &http.Response{
 			StatusCode: http.StatusInternalServerError,
 			Header:     make(http.Header),
-			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+			Body:       io.NopCloser(bytes.NewReader([]byte{})),
 		}
 	}
 }
@@ -546,11 +826,11 @@ func failRoundTripper() RoundTripFunc {
 func brokenBodyRoundTripper(t *testing.T, contents []byte, multiRange bool) RoundTripFunc {
 	breakReadCloser := func(r io.ReadCloser) io.ReadCloser {
 		defer r.Close()
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		if err != nil {
 			t.Fatalf("failed to break read closer faild to read original: %v", err)
 		}
-		return ioutil.NopCloser(bytes.NewReader(data[:len(data)/2]))
+		return io.NopCloser(bytes.NewReader(data[:len(data)/2]))
 	}
 	tr := multiRoundTripper(t, contents, allowMultiRange(multiRange), bodyConverter(breakReadCloser))
 	return func(req *http.Request) *http.Response {

fs/remote/resolver.go

@@ -23,38 +23,46 @@
 package remote
 
 import (
-	"bytes"
 	"context"
+	"crypto/rand"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"math/big"
 	"mime"
 	"mime/multipart"
 	"net/http"
-	"net/url"
 	"path"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/reference"
-	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/v2/core/remotes/docker"
+	"github.com/containerd/containerd/v2/pkg/reference"
+	"github.com/containerd/errdefs"
+	"github.com/containerd/log"
 	"github.com/containerd/stargz-snapshotter/cache"
 	"github.com/containerd/stargz-snapshotter/fs/config"
+	commonmetrics "github.com/containerd/stargz-snapshotter/fs/metrics/common"
+	"github.com/containerd/stargz-snapshotter/fs/source"
+	rhttp "github.com/hashicorp/go-retryablehttp"
+	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 )
 
 const (
 	defaultChunkSize        = 50000
 	defaultValidIntervalSec = 60
 	defaultFetchTimeoutSec  = 300
+
+	defaultMaxRetries  = 5
+	defaultMinWaitMSec = 30
+	defaultMaxWaitMSec = 300000
 )
 
-func NewResolver(cache cache.BlobCache, cfg config.BlobConfig) *Resolver {
+func NewResolver(cfg config.BlobConfig, handlers map[string]Handler) *Resolver {
 	if cfg.ChunkSize == 0 { // zero means "use default chunk size"
 		cfg.ChunkSize = defaultChunkSize
 	}
@@ -67,51 +75,137 @@ func NewResolver(cache cache.BlobCache, cfg config.BlobConfig) *Resolver {
 	if cfg.FetchTimeoutSec == 0 {
 		cfg.FetchTimeoutSec = defaultFetchTimeoutSec
 	}
+	if cfg.MaxRetries == 0 {
+		cfg.MaxRetries = defaultMaxRetries
+	}
+	if cfg.MinWaitMSec == 0 {
+		cfg.MinWaitMSec = defaultMinWaitMSec
+	}
+	if cfg.MaxWaitMSec == 0 {
+		cfg.MaxWaitMSec = defaultMaxWaitMSec
+	}
 
 	return &Resolver{
-		bufPool: sync.Pool{
-			New: func() interface{} {
-				return new(bytes.Buffer)
-			},
-		},
-		blobCache:  cache,
 		blobConfig: cfg,
+		handlers:   handlers,
 	}
 }
 
 type Resolver struct {
-	blobCache  cache.BlobCache
 	blobConfig config.BlobConfig
-	bufPool    sync.Pool
+	handlers   map[string]Handler
 }
 
-func (r *Resolver) Resolve(ctx context.Context, hosts docker.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) (Blob, error) {
-	fetcher, size, err := newFetcher(ctx, hosts, refspec, desc)
+type fetcher interface {
+	fetch(ctx context.Context, rs []region, retry bool) (multipartReadCloser, error)
+	check() error
+	genID(reg region) string
+}
+
+func (r *Resolver) Resolve(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor, blobCache cache.BlobCache) (Blob, error) {
+	f, size, err := r.resolveFetcher(ctx, hosts, refspec, desc)
 	if err != nil {
 		return nil, err
 	}
-	return &blob{
-		fetcher:       fetcher,
-		size:          size,
-		chunkSize:     r.blobConfig.ChunkSize,
-		cache:         r.blobCache,
-		lastCheck:     time.Now(),
-		checkInterval: time.Duration(r.blobConfig.ValidInterval) * time.Second,
-		resolver:      r,
-		fetchTimeout:  time.Duration(r.blobConfig.FetchTimeoutSec) * time.Second,
-	}, nil
+	blobConfig := &r.blobConfig
+	return makeBlob(f,
+		size,
+		blobConfig.ChunkSize,
+		blobConfig.PrefetchChunkSize,
+		blobCache,
+		time.Now(),
+		time.Duration(blobConfig.ValidInterval)*time.Second,
+		r,
+		time.Duration(blobConfig.FetchTimeoutSec)*time.Second), nil
 }
 
-func newFetcher(ctx context.Context, hosts docker.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) (*fetcher, int64, error) {
-	reghosts, err := hosts(refspec.Hostname())
+func (r *Resolver) resolveFetcher(ctx context.Context, hosts source.RegistryHosts, refspec reference.Spec, desc ocispec.Descriptor) (f fetcher, size int64, err error) {
+	blobConfig := &r.blobConfig
+	fc := &fetcherConfig{
+		hosts:      hosts,
+		refspec:    refspec,
+		desc:       desc,
+		maxRetries: blobConfig.MaxRetries,
+		minWait:    time.Duration(blobConfig.MinWaitMSec) * time.Millisecond,
+		maxWait:    time.Duration(blobConfig.MaxWaitMSec) * time.Millisecond,
+	}
+	var errs []error
+	for name, p := range r.handlers {
+		// TODO: allow to configure the selection of readers based on the hostname in refspec
+		r, size, err := p.Handle(ctx, desc)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		log.G(ctx).WithField("handler name", name).WithField("ref", refspec.String()).WithField("digest", desc.Digest).
+			Debugf("contents is provided by a handler")
+		return &remoteFetcher{r}, size, nil
+	}
+
+	handlersErr := errors.Join(errs...)
+
+	log.G(ctx).WithError(handlersErr).WithField("ref", refspec.String()).WithField("digest", desc.Digest).Debugf("using default handler")
+	hf, size, err := newHTTPFetcher(ctx, fc)
 	if err != nil {
 		return nil, 0, err
 	}
+	if blobConfig.ForceSingleRangeMode {
+		hf.singleRangeMode()
+	}
+	return hf, size, err
+}
+
+type fetcherConfig struct {
+	hosts      source.RegistryHosts
+	refspec    reference.Spec
+	desc       ocispec.Descriptor
+	maxRetries int
+	minWait    time.Duration
+	maxWait    time.Duration
+}
+
+func jitter(duration time.Duration) time.Duration {
+	if duration <= 0 {
+		return duration
+	}
+	b, err := rand.Int(rand.Reader, big.NewInt(int64(duration)))
+	if err != nil {
+		panic(err)
+	}
+	return time.Duration(b.Int64() + int64(duration))
+}
+
+// backoffStrategy extends retryablehttp's DefaultBackoff to add a random jitter to avoid overwhelming the repository
+// when it comes back online
+// DefaultBackoff either tries to parse the 'Retry-After' header of the response; or, it uses an exponential backoff
+// 2 ^ numAttempts, limited by max
+func backoffStrategy(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
+	delayTime := rhttp.DefaultBackoff(min, max, attemptNum, resp)
+	return jitter(delayTime)
+}
+
+// retryStrategy extends retryablehttp's DefaultRetryPolicy to debug log the error when retrying
+// DefaultRetryPolicy retries whenever err is non-nil (except for some url errors) or if returned
+// status code is 429 or 5xx (except 501)
+func retryStrategy(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	retry, err2 := rhttp.DefaultRetryPolicy(ctx, resp, err)
+	if retry {
+		log.G(ctx).WithError(err).Debugf("Retrying request")
+	}
+	return retry, err2
+}
+
+func newHTTPFetcher(ctx context.Context, fc *fetcherConfig) (*httpFetcher, int64, error) {
+	reghosts, err := fc.hosts(fc.refspec)
+	if err != nil {
+		return nil, 0, err
+	}
+	desc := fc.desc
 	if desc.Digest.String() == "" {
-		return nil, 0, fmt.Errorf("Digest is mandatory in layer descriptor")
+		return nil, 0, fmt.Errorf("digest is mandatory in layer descriptor")
 	}
 	digest := desc.Digest
-	u, err := url.Parse("dummy://" + refspec.Locator)
+	pullScope, err := docker.RepositoryScope(fc.refspec, false)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -120,21 +214,29 @@ func newFetcher(ctx context.Context, hosts docker.RegistryHosts, refspec referen
 	rErr := fmt.Errorf("failed to resolve")
 	for _, host := range reghosts {
 		if host.Host == "" || strings.Contains(host.Host, "/") {
-			rErr = errors.Wrapf(rErr, "invalid destination (host %q, ref:%q, digest:%q)",
-				host.Host, refspec, digest)
+			rErr = fmt.Errorf("invalid destination (host %q, ref:%q, digest:%q): %w", host.Host, fc.refspec, digest, rErr)
 			continue // Try another
 
 		}
 
 		// Prepare transport with authorization functionality
 		tr := host.Client.Transport
+
+		timeout := host.Client.Timeout
+		if rt, ok := tr.(*rhttp.RoundTripper); ok {
+			rt.Client.RetryMax = fc.maxRetries
+			rt.Client.RetryWaitMin = fc.minWait
+			rt.Client.RetryWaitMax = fc.maxWait
+			rt.Client.Backoff = backoffStrategy
+			rt.Client.CheckRetry = retryStrategy
+			timeout = rt.Client.HTTPClient.Timeout
+		}
+
 		if host.Authorizer != nil {
 			tr = &transport{
 				inner: tr,
 				auth:  host.Authorizer,
-				// Specify pull scope
-				// TODO: The scope generator function in containerd (github.com/containerd/containerd/remotes/docker/scope.go) should be exported and used here.
-				scope: "repository:" + strings.TrimPrefix(u.Path, "/") + ":pull",
+				scope: pullScope,
 			}
 		}
 
@@ -142,33 +244,37 @@ func newFetcher(ctx context.Context, hosts docker.RegistryHosts, refspec referen
 		blobURL := fmt.Sprintf("%s://%s/%s/blobs/%s",
 			host.Scheme,
 			path.Join(host.Host, host.Path),
-			strings.TrimPrefix(refspec.Locator, refspec.Hostname()+"/"),
+			strings.TrimPrefix(fc.refspec.Locator, fc.refspec.Hostname()+"/"),
 			digest)
-		url, err := redirect(ctx, blobURL, tr)
+		url, header, err := redirect(ctx, blobURL, tr, timeout, host.Header)
 		if err != nil {
-			rErr = errors.Wrapf(rErr, "failed to redirect (host %q, ref:%q, digest:%q): %v",
-				host.Host, refspec, digest, err)
+			rErr = fmt.Errorf("failed to redirect (host %q, ref:%q, digest:%q): %v: %w", host.Host, fc.refspec, digest, err, rErr)
 			continue // Try another
 		}
 
 		// Get size information
 		// TODO: we should try to use the Size field in the descriptor here.
-		size, err := getSize(ctx, url, tr)
+		start := time.Now() // start time before getting layer header
+		size, err := getSize(ctx, url, tr, timeout, header)
+		commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.StargzHeaderGet, digest, start) // time to get layer header
 		if err != nil {
-			rErr = errors.Wrapf(rErr, "failed to get size (host %q, ref:%q, digest:%q): %v",
-				host.Host, refspec, digest, err)
+			rErr = fmt.Errorf("failed to get size (host %q, ref:%q, digest:%q): %v: %w", host.Host, fc.refspec, digest, err, rErr)
 			continue // Try another
 		}
 
 		// Hit one destination
-		return &fetcher{
-			url:     url,
-			tr:      tr,
-			blobURL: blobURL,
+		return &httpFetcher{
+			url:       url,
+			tr:        tr,
+			blobURL:   blobURL,
+			digest:    digest,
+			timeout:   timeout,
+			header:    header,
+			orgHeader: host.Header,
 		}, size, nil
 	}
 
-	return nil, 0, errors.Wrapf(rErr, "cannot resolve layer")
+	return nil, 0, fmt.Errorf("cannot resolve layer: %w", rErr)
 }
 
 type transport struct {
@@ -196,6 +302,7 @@ func (tr *transport) RoundTrip(req *http.Request) (*http.Response, error) {
 
 	// TODO: support more status codes and retries
 	if resp.StatusCode == http.StatusUnauthorized {
+		log.G(ctx).Infof("Received status code: %v. Refreshing creds...", resp.Status)
 
 		// prepare authorization for the target host using docker.Authorizer
 		if err := tr.auth.AddResponses(ctx, []*http.Response{resp}); err != nil {
@@ -212,47 +319,62 @@ func (tr *transport) RoundTrip(req *http.Request) (*http.Response, error) {
 	return resp, nil
 }
 
-func redirect(ctx context.Context, blobURL string, tr http.RoundTripper) (url string, err error) {
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-
+func redirect(ctx context.Context, blobURL string, tr http.RoundTripper, timeout time.Duration, header http.Header) (url string, withHeader http.Header, err error) {
+	if timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+		defer cancel()
+	}
 	// We use GET request for redirect.
 	// gcr.io returns 200 on HEAD without Location header (2020).
 	// ghcr.io returns 200 on HEAD without Location header (2020).
 	req, err := http.NewRequestWithContext(ctx, "GET", blobURL, nil)
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to make request to the registry")
+		return "", nil, fmt.Errorf("failed to make request to the registry: %w", err)
+	}
+	req.Header = http.Header{}
+	for k, v := range header {
+		req.Header[k] = v
 	}
 	req.Close = false
 	req.Header.Set("Range", "bytes=0-1")
 	res, err := tr.RoundTrip(req)
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to request")
+		return "", nil, fmt.Errorf("failed to request: %w", err)
 	}
 	defer func() {
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 		res.Body.Close()
 	}()
 
 	if res.StatusCode/100 == 2 {
 		url = blobURL
+		withHeader = header
 	} else if redir := res.Header.Get("Location"); redir != "" && res.StatusCode/100 == 3 {
 		// TODO: Support nested redirection
 		url = redir
+		// Do not pass headers to the redirected location.
 	} else {
-		return "", fmt.Errorf("failed to access to the registry with code %v", res.StatusCode)
+		return "", nil, fmt.Errorf("failed to access to the registry with code %v", res.StatusCode)
 	}
 
 	return
 }
 
-func getSize(ctx context.Context, url string, tr http.RoundTripper) (int64, error) {
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
+func getSize(ctx context.Context, url string, tr http.RoundTripper, timeout time.Duration, header http.Header) (int64, error) {
+	if timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+		defer cancel()
+	}
 	req, err := http.NewRequestWithContext(ctx, "HEAD", url, nil)
 	if err != nil {
 		return 0, err
 	}
+	req.Header = http.Header{}
+	for k, v := range header {
+		req.Header[k] = v
+	}
 	req.Close = false
 	res, err := tr.RoundTrip(req)
 	if err != nil {
@@ -269,22 +391,27 @@ func getSize(ctx context.Context, url string, tr http.RoundTripper) (int64, erro
 	// HEAD request (2020).
 	req, err = http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
-		return 0, errors.Wrapf(err, "failed to make request to the registry")
+		return 0, fmt.Errorf("failed to make request to the registry: %w", err)
+	}
+	req.Header = http.Header{}
+	for k, v := range header {
+		req.Header[k] = v
 	}
 	req.Close = false
 	req.Header.Set("Range", "bytes=0-1")
 	res, err = tr.RoundTrip(req)
 	if err != nil {
-		return 0, errors.Wrapf(err, "failed to request")
+		return 0, fmt.Errorf("failed to request: %w", err)
 	}
 	defer func() {
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 		res.Body.Close()
 	}()
 
-	if res.StatusCode == http.StatusOK {
+	switch res.StatusCode {
+	case http.StatusOK:
 		return strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
-	} else if res.StatusCode == http.StatusPartialContent {
+	case http.StatusPartialContent:
 		_, size, err := parseRange(res.Header.Get("Content-Range"))
 		return size, err
 	}
@@ -293,13 +420,17 @@ func getSize(ctx context.Context, url string, tr http.RoundTripper) (int64, erro
 		headStatusCode, res.StatusCode)
 }
 
-type fetcher struct {
+type httpFetcher struct {
 	url           string
 	urlMu         sync.Mutex
 	tr            http.RoundTripper
 	blobURL       string
+	digest        digest.Digest
 	singleRange   bool
 	singleRangeMu sync.Mutex
+	timeout       time.Duration
+	header        http.Header
+	orgHeader     http.Header
 }
 
 type multipartReadCloser interface {
@@ -307,7 +438,7 @@ type multipartReadCloser interface {
 	Close() error
 }
 
-func (f *fetcher) fetch(ctx context.Context, rs []region, retry bool, opts *options) (multipartReadCloser, error) {
+func (f *httpFetcher) fetch(ctx context.Context, rs []region, retry bool) (multipartReadCloser, error) {
 	if len(rs) == 0 {
 		return nil, fmt.Errorf("no request queried")
 	}
@@ -317,13 +448,6 @@ func (f *fetcher) fetch(ctx context.Context, rs []region, retry bool, opts *opti
 		singleRangeMode = f.isSingleRangeMode()
 	)
 
-	if opts.ctx != nil {
-		ctx = opts.ctx
-	}
-	if opts.tr != nil {
-		tr = opts.tr
-	}
-
 	// squash requesting chunks for reducing the total size of request header
 	// (servers generally have limits for the size of headers)
 	// TODO: when our request has too many ranges, we need to divide it into
@@ -346,6 +470,10 @@ func (f *fetcher) fetch(ctx context.Context, rs []region, retry bool, opts *opti
 	if err != nil {
 		return nil, err
 	}
+	req.Header = http.Header{}
+	for k, v := range f.header {
+		req.Header[k] = v
+	}
 	var ranges string
 	for _, reg := range requests {
 		ranges += fmt.Sprintf("%d-%d,", reg.b, reg.e)
@@ -353,7 +481,11 @@ func (f *fetcher) fetch(ctx context.Context, rs []region, retry bool, opts *opti
 	req.Header.Add("Range", fmt.Sprintf("bytes=%s", ranges[:len(ranges)-1]))
 	req.Header.Add("Accept-Encoding", "identity")
 	req.Close = false
+
+	// Recording the roundtrip latency for remote registry GET operation.
+	start := time.Now()
 	res, err := tr.RoundTrip(req) // NOT DefaultClient; don't want redirects
+	commonmetrics.MeasureLatencyInMilliseconds(commonmetrics.RemoteRegistryGet, f.digest, start)
 	if err != nil {
 		return nil, err
 	}
@@ -361,66 +493,83 @@ func (f *fetcher) fetch(ctx context.Context, rs []region, retry bool, opts *opti
 		// We are getting the whole blob in one part (= status 200)
 		size, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse Content-Length")
+			return nil, fmt.Errorf("failed to parse Content-Length: %w", err)
 		}
-		return singlePartReader(region{0, size - 1}, res.Body), nil
+		return newSinglePartReader(region{0, size - 1}, res.Body), nil
 	} else if res.StatusCode == http.StatusPartialContent {
 		mediaType, params, err := mime.ParseMediaType(res.Header.Get("Content-Type"))
 		if err != nil {
-			return nil, errors.Wrapf(err, "invalid media type %q", mediaType)
+			return nil, fmt.Errorf("invalid media type %q: %w", mediaType, err)
 		}
 		if strings.HasPrefix(mediaType, "multipart/") {
 			// We are getting a set of chunks as a multipart body.
-			return multiPartReader(res.Body, params["boundary"]), nil
+			return newMultiPartReader(res.Body, params["boundary"]), nil
 		}
 
 		// We are getting single range
 		reg, _, err := parseRange(res.Header.Get("Content-Range"))
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse Content-Range")
+			return nil, fmt.Errorf("failed to parse Content-Range: %w", err)
 		}
-		return singlePartReader(reg, res.Body), nil
+		return newSinglePartReader(reg, res.Body), nil
 	} else if retry && res.StatusCode == http.StatusForbidden {
+		log.G(ctx).Infof("Received status code: %v. Refreshing URL and retrying...", res.Status)
+
 		// re-redirect and retry this once.
 		if err := f.refreshURL(ctx); err != nil {
-			return nil, errors.Wrapf(err, "failed to refresh URL on %v", res.Status)
+			return nil, fmt.Errorf("failed to refresh URL on %v: %w", res.Status, err)
 		}
-		return f.fetch(ctx, rs, false, opts)
+		return f.fetch(ctx, rs, false)
 	} else if retry && res.StatusCode == http.StatusBadRequest && !singleRangeMode {
+		log.G(ctx).Infof("Received status code: %v. Setting single range mode and retrying...", res.Status)
+
 		// gcr.io (https://storage.googleapis.com) returns 400 on multi-range request (2020 #81)
-		f.singleRangeMode()                  // fallbacks to singe range request mode
-		return f.fetch(ctx, rs, false, opts) // retries with the single range mode
+		f.singleRangeMode()            // fallbacks to singe range request mode
+		return f.fetch(ctx, rs, false) // retries with the single range mode
 	}
 
 	return nil, fmt.Errorf("unexpected status code: %v", res.Status)
 }
 
-func (f *fetcher) check() error {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
+func (f *httpFetcher) check() error {
+	ctx := context.Background()
+	if f.timeout > 0 {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, f.timeout)
+		defer cancel()
+	}
 	f.urlMu.Lock()
 	url := f.url
 	f.urlMu.Unlock()
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
-		return errors.Wrapf(err, "check failed: failed to make request")
+		return fmt.Errorf("check failed: failed to make request: %w", err)
+	}
+	req.Header = http.Header{}
+	for k, v := range f.header {
+		req.Header[k] = v
 	}
 	req.Close = false
 	req.Header.Set("Range", "bytes=0-1")
 	res, err := f.tr.RoundTrip(req)
 	if err != nil {
-		return errors.Wrapf(err, "check failed: failed to request to registry")
+		return fmt.Errorf("check failed: failed to request to registry: %w", err)
 	}
 	defer func() {
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 		res.Body.Close()
 	}()
-	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusPartialContent {
+	switch res.StatusCode {
+	case http.StatusOK, http.StatusPartialContent:
 		return nil
-	} else if res.StatusCode == http.StatusForbidden {
+	case http.StatusForbidden:
 		// Try to re-redirect this blob
-		rCtx, rCancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer rCancel()
+		rCtx := context.Background()
+		if f.timeout > 0 {
+			var rCancel context.CancelFunc
+			rCtx, rCancel = context.WithTimeout(rCtx, f.timeout)
+			defer rCancel()
+		}
 		if err := f.refreshURL(rCtx); err == nil {
 			return nil
 		}
@@ -430,36 +579,37 @@ func (f *fetcher) check() error {
 	return fmt.Errorf("unexpected status code %v", res.StatusCode)
 }
 
-func (f *fetcher) refreshURL(ctx context.Context) error {
-	newURL, err := redirect(ctx, f.blobURL, f.tr)
+func (f *httpFetcher) refreshURL(ctx context.Context) error {
+	newURL, headers, err := redirect(ctx, f.blobURL, f.tr, f.timeout, f.orgHeader)
 	if err != nil {
 		return err
 	}
 	f.urlMu.Lock()
 	f.url = newURL
+	f.header = headers
 	f.urlMu.Unlock()
 	return nil
 }
 
-func (f *fetcher) genID(reg region) string {
+func (f *httpFetcher) genID(reg region) string {
 	sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", f.blobURL, reg.b, reg.e)))
 	return fmt.Sprintf("%x", sum)
 }
 
-func (f *fetcher) singleRangeMode() {
+func (f *httpFetcher) singleRangeMode() {
 	f.singleRangeMu.Lock()
 	f.singleRange = true
 	f.singleRangeMu.Unlock()
 }
 
-func (f *fetcher) isSingleRangeMode() bool {
+func (f *httpFetcher) isSingleRangeMode() bool {
 	f.singleRangeMu.Lock()
 	r := f.singleRange
 	f.singleRangeMu.Unlock()
 	return r
 }
 
-func singlePartReader(reg region, rc io.ReadCloser) multipartReadCloser {
+func newSinglePartReader(reg region, rc io.ReadCloser) multipartReadCloser {
 	return &singlepartReader{
 		r:      rc,
 		Closer: rc,
@@ -482,7 +632,7 @@ func (sr *singlepartReader) Next() (region, io.Reader, error) {
 	return region{}, nil, io.EOF
 }
 
-func multiPartReader(rc io.ReadCloser, boundary string) multipartReadCloser {
+func newMultiPartReader(rc io.ReadCloser, boundary string) multipartReadCloser {
 	return &multipartReader{
 		m:      multipart.NewReader(rc, boundary),
 		Closer: rc,
@@ -501,7 +651,7 @@ func (sr *multipartReader) Next() (region, io.Reader, error) {
 	}
 	reg, _, err := parseRange(p.Header.Get("Content-Range"))
 	if err != nil {
-		return region{}, nil, errors.Wrapf(err, "failed to parse Content-Range")
+		return region{}, nil, fmt.Errorf("failed to parse Content-Range: %w", err)
 	}
 	return reg, p, nil
 }
@@ -513,15 +663,15 @@ func parseRange(header string) (region, int64, error) {
 	}
 	begin, err := strconv.ParseInt(submatches[1], 10, 64)
 	if err != nil {
-		return region{}, 0, errors.Wrapf(err, "failed to parse beginning offset %q", submatches[1])
+		return region{}, 0, fmt.Errorf("failed to parse beginning offset %q: %w", submatches[1], err)
 	}
 	end, err := strconv.ParseInt(submatches[2], 10, 64)
 	if err != nil {
-		return region{}, 0, errors.Wrapf(err, "failed to parse end offset %q", submatches[2])
+		return region{}, 0, fmt.Errorf("failed to parse end offset %q: %w", submatches[2], err)
 	}
 	blobSize, err := strconv.ParseInt(submatches[3], 10, 64)
 	if err != nil {
-		return region{}, 0, errors.Wrapf(err, "failed to parse blob size %q", submatches[3])
+		return region{}, 0, fmt.Errorf("failed to parse blob size %q: %w", submatches[3], err)
 	}
 
 	return region{begin, end}, blobSize, nil
@@ -531,7 +681,6 @@ type Option func(*options)
 
 type options struct {
 	ctx       context.Context
-	tr        http.RoundTripper
 	cacheOpts []cache.Option
 }
 
@@ -541,14 +690,43 @@ func WithContext(ctx context.Context) Option {
 	}
 }
 
-func WithRoundTripper(tr http.RoundTripper) Option {
-	return func(opts *options) {
-		opts.tr = tr
-	}
-}
-
 func WithCacheOpts(cacheOpts ...cache.Option) Option {
 	return func(opts *options) {
 		opts.cacheOpts = cacheOpts
 	}
 }
+
+type remoteFetcher struct {
+	r Fetcher
+}
+
+func (r *remoteFetcher) fetch(ctx context.Context, rs []region, retry bool) (multipartReadCloser, error) {
+	var s regionSet
+	for _, reg := range rs {
+		s.add(reg)
+	}
+	reg := superRegion(s.rs)
+	rc, err := r.r.Fetch(ctx, reg.b, reg.size())
+	if err != nil {
+		return nil, err
+	}
+	return newSinglePartReader(reg, rc), nil
+}
+
+func (r *remoteFetcher) check() error {
+	return r.r.Check()
+}
+
+func (r *remoteFetcher) genID(reg region) string {
+	return r.r.GenID(reg.b, reg.size())
+}
+
+type Handler interface {
+	Handle(ctx context.Context, desc ocispec.Descriptor) (fetcher Fetcher, size int64, err error)
+}
+
+type Fetcher interface {
+	Fetch(ctx context.Context, off int64, size int64) (io.ReadCloser, error)
+	Check() error
+	GenID(off int64, size int64) string
+}