action: fix smoke test for branch pattern

To match `master` and `stable/*` branches at least. Signed-off-by: Yan Song <imeoer@linux.alibaba.com>
action: upgrade golangci-lint to v1.51.2
2023-04-20 18:32:45 +08:00 · 2023-04-17 11:43:46 +08:00 · 2023-04-17 11:02:07 +08:00 · 2023-04-14 18:04:04 +08:00 · 2023-04-14 18:04:00 +08:00 · 2023-04-10 15:26:40 +08:00
36 changed files with 998 additions and 4208 deletions
--- a/.github/workflows/convert.yml
+++ b/.github/workflows/convert.yml
@ -34,7 +34,7 @@ jobs:
          ${{ runner.os }}-golang-
    - name: Build Contrib
      run: |
-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.47.3
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.51.2
        make -e DOCKER=false nydusify-release
    - name: Upload Nydusify
      uses: actions/upload-artifact@master
--- a/.github/workflows/smoke.yml
+++ b/.github/workflows/smoke.yml
@ -2,10 +2,10 @@ name: Smoke Test

 on:
  push:
-    branches: ["*"]
+    branches: ["**", "stable/**"]
    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.svg', '**/docs/**' ]
-  pull_request:
-    branches: ["*"]
+  pull_request_target:
+    branches: ["**", "stable/**"]
    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.svg', '**/docs/**' ]
  schedule:
    # Run daily sanity check at 03:00 clock UTC
@ -36,7 +36,7 @@ jobs:
          ${{ runner.os }}-golang-
    - name: Build Contrib
      run: |
-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.47.3
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.51.2
        make -e DOCKER=false nydusify-release
        make -e DOCKER=false contrib-test
    - name: Upload Nydusify
@ -125,7 +125,7 @@ jobs:
          export NYDUS_NYDUSIFY_$version_export=/usr/bin/nydus-$version/nydusify
        done

-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.47.3
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.51.2
        sudo -E make smoke-only

  nydus-unit-test:
--- a/Cargo.lock
+++ b/Cargo.lock
@ -103,6 +103,12 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"

+[[package]]
+name = "base64"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@ -376,6 +382,27 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "errno"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.7.0"
@ -805,6 +832,16 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "io-lifetimes"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
 [[package]]
 name = "io-uring"
 version = "0.5.9"
@ -890,6 +927,12 @@ dependencies = [
 "cc",
 ]

+[[package]]
+name = "linux-raw-sys"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
+
 [[package]]
 name = "lock_api"
 version = "0.4.7"
@ -989,9 +1032,9 @@ dependencies = [

 [[package]]
 name = "native-tls"
-version = "0.2.10"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
 dependencies = [
 "lazy_static",
 "libc",
@ -1146,7 +1189,7 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "assert_matches",
- "base64",
+ "base64 0.13.0",
 "bitflags",
 "blake3",
 "fuse-backend-rs",
@ -1172,7 +1215,7 @@ name = "nydus-rs"
 version = "0.0.0-git"
 dependencies = [
 "anyhow",
- "base64",
+ "base64 0.13.0",
 "clap",
 "fuse-backend-rs",
 "hex",
@ -1240,7 +1283,7 @@ name = "nydus-storage"
 version = "0.6.2"
 dependencies = [
 "arc-swap",
- "base64",
+ "base64 0.13.0",
 "bitflags",
 "fuse-backend-rs",
 "gpt",
@ -1286,6 +1329,7 @@ dependencies = [
 "lz4-sys",
 "nix",
 "nydus-error",
+ "openssl",
 "serde",
 "serde_json",
 "sha2",
@ -1312,9 +1356,9 @@ checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"

 [[package]]
 name = "openssl"
-version = "0.10.45"
+version = "0.10.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1"
+checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2"
 dependencies = [
 "bitflags",
 "cfg-if",
@ -1353,9 +1397,9 @@ dependencies = [

 [[package]]
 name = "openssl-sys"
-version = "0.9.80"
+version = "0.9.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7"
+checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b"
 dependencies = [
 "autocfg",
 "cc",
@ -1512,22 +1556,13 @@ version = "0.6.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"

-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "reqwest"
-version = "0.11.11"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92"
+checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9"
 dependencies = [
- "base64",
+ "base64 0.21.0",
 "bytes",
 "encoding_rs",
 "futures-core",
@ -1539,10 +1574,10 @@ dependencies = [
 "hyper-tls",
 "ipnet",
 "js-sys",
- "lazy_static",
 "log",
 "mime",
 "native-tls",
+ "once_cell",
 "percent-encoding",
 "pin-project-lite",
 "serde",
@ -1592,6 +1627,20 @@ version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"

+[[package]]
+name = "rustix"
+version = "0.36.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03"
+dependencies = [
+ "bitflags",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.10"
@ -1780,16 +1829,15 @@ dependencies = [

 [[package]]
 name = "tempfile"
-version = "3.3.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
 dependencies = [
 "cfg-if",
 "fastrand",
- "libc",
 "redox_syscall",
- "remove_dir_all",
- "winapi",
+ "rustix",
+ "windows-sys",
 ]

 [[package]]
--- a/Cargo.toml
+++ b/Cargo.toml
@ -53,7 +53,7 @@ vmm-sys-util = "0.10.0"
 xattr = "0.2.3"

 # Build static linked openssl library
-openssl = { version = "0.10.45", features = ["vendored"] }
+openssl = { version = "0.10.48", features = ["vendored"] }
 # pin openssl-src to bring in fix for https://rustsec.org/advisories/RUSTSEC-2022-0032
 #openssl-src = { version = "111.22" }

--- a/README.md
+++ b/README.md
@ -137,6 +137,11 @@ The containerd remote snapshotter plugin [nydus-snapshotter](https://github.com/

 In the future, `zstd::chunked` can work in this way as well.

+## Reuse Nydus Services
+
+Using the key features of nydus as native in your project without preparing and invoking `nydusd` deliberately, [nydus-service](./service/README.md) helps to reuse the core services of nyuds.
+
+
 ## Documentation

 Browse the documentation to learn more. Here are some topics you may be interested in:
--- a/contrib/ctr-remote/go.mod
+++ b/contrib/ctr-remote/go.mod
@ -3,23 +3,23 @@ module github.com/dragonflyoss/image-service/contrib/ctr-remote
 go 1.18

 require (
-	github.com/containerd/containerd v1.6.18
-	github.com/containerd/nydus-snapshotter v0.5.1
-	github.com/opencontainers/image-spec v1.1.0-rc2
+	github.com/containerd/containerd v1.6.20
+	github.com/containerd/nydus-snapshotter v0.6.1
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
 	github.com/urfave/cli v1.22.12
 )

 require (
 	github.com/Microsoft/go-winio v0.6.0 // indirect
-	github.com/Microsoft/hcsshim v0.9.6 // indirect
+	github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
 	github.com/cilium/ebpf v0.10.0 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/containerd/console v1.0.3 // indirect
 	github.com/containerd/continuity v0.3.0 // indirect
-	github.com/containerd/fifo v1.0.0 // indirect
-	github.com/containerd/go-cni v1.1.8 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/go-cni v1.1.9 // indirect
 	github.com/containerd/go-runc v1.0.0 // indirect
-	github.com/containerd/ttrpc v1.1.0 // indirect
+	github.com/containerd/ttrpc v1.2.1 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/containernetworking/cni v1.1.2 // indirect
 	github.com/containernetworking/plugins v1.2.0 // indirect
@ -27,33 +27,36 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
-	github.com/gogo/googleapis v1.4.1 // indirect
+	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
 	github.com/moby/sys/signal v0.7.0 // indirect
 	github.com/moby/sys/symlink v0.2.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.4.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/runc v1.1.4 // indirect
+	github.com/opencontainers/runc v1.1.5 // indirect
 	github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/stretchr/testify v1.8.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-	google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
-	google.golang.org/grpc v1.53.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
+	google.golang.org/genproto v0.0.0-20230330200707-38013875ee22 // indirect
+	google.golang.org/grpc v1.54.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 )
--- a/contrib/ctr-remote/go.sum
+++ b/contrib/ctr-remote/go.sum
--- a/contrib/nydus-backend-proxy/Cargo.lock
+++ b/contrib/nydus-backend-proxy/Cargo.lock
@ -278,6 +278,27 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "errno"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.7.0"
@ -564,6 +585,16 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "io-lifetimes"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3"
+dependencies = [
+ "libc",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.2"
@ -578,9 +609,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.126"
+version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"

 [[package]]
 name = "lock_api"
@ -655,7 +692,7 @@ dependencies = [
 "libc",
 "log",
 "wasi",
- "windows-sys",
+ "windows-sys 0.42.0",
 ]

 [[package]]
@ -753,7 +790,7 @@ dependencies = [
 "libc",
 "redox_syscall",
 "smallvec",
- "windows-sys",
+ "windows-sys 0.42.0",
 ]

 [[package]]
@ -929,15 +966,6 @@ version = "0.6.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"

-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "rocket"
 version = "0.5.0-rc.2"
@ -1019,6 +1047,20 @@ dependencies = [
 "uncased",
 ]

+[[package]]
+name = "rustix"
+version = "0.36.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644"
+dependencies = [
+ "bitflags",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.7"
@ -1174,16 +1216,15 @@ dependencies = [

 [[package]]
 name = "tempfile"
-version = "3.3.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
 dependencies = [
 "cfg-if",
 "fastrand",
- "libc",
 "redox_syscall",
- "remove_dir_all",
- "winapi",
+ "rustix",
+ "windows-sys 0.42.0",
 ]

 [[package]]
@ -1238,7 +1279,7 @@ dependencies = [
 "signal-hook-registry",
 "socket2",
 "tokio-macros",
- "windows-sys",
+ "windows-sys 0.42.0",
 ]

 [[package]]
@ -1498,6 +1539,30 @@ dependencies = [
 "windows_x86_64_msvc",
 ]

+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.42.1"
--- a/contrib/nydusify/cmd/nydusify.go
+++ b/contrib/nydusify/cmd/nydusify.go
@ -340,7 +340,11 @@ func main() {
 					Value:   false,
 					Usage:   "Convert Docker media types to OCI media types",
 					EnvVars: []string{"OCI"},
-					Aliases: []string{"docker-v2-format"},
+				},
+				&cli.BoolFlag{
+					Name:   "docker-v2-format",
+					Value:  false,
+					Hidden: true,
 				},
 				&cli.StringFlag{
 					Name:        "fs-version",
@ -446,6 +450,20 @@ func main() {
 					}
 				}

+				docker2OCI := false
+				if c.Bool("docker-v2-format") {
+					logrus.Warn("the option `--docker-v2-format` has been deprecated, use `--oci` instead")
+					docker2OCI = false
+				} else if c.Bool("oci") {
+					docker2OCI = true
+				}
+
+				// Forcibly enable `--oci` option when `--oci-ref` be enabled.
+				if c.Bool("oci-ref") {
+					logrus.Warn("forcibly enabled `--oci` option when `--oci-ref` be enabled")
+					docker2OCI = true
+				}
+
 				opt := converter.Opt{
 					WorkDir:        c.String("work-dir"),
 					NydusImagePath: c.String("nydus-image"),
@ -469,7 +487,7 @@ func main() {

 					PrefetchPatterns: prefetchPatterns,
 					MergePlatform:    c.Bool("merge-platform"),
-					Docker2OCI:       c.Bool("oci"),
+					Docker2OCI:       docker2OCI,
 					FsVersion:        fsVersion,
 					FsAlignChunk:     c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"),
 					Compressor:       c.String("compressor"),
--- a/contrib/nydusify/go.mod
+++ b/contrib/nydusify/go.mod
@ -3,127 +3,109 @@ module github.com/dragonflyoss/image-service/contrib/nydusify
 go 1.18

 require (
-	github.com/aliyun/aliyun-oss-go-sdk v2.1.5+incompatible
-	github.com/aws/aws-sdk-go-v2 v1.17.2
-	github.com/aws/aws-sdk-go-v2/config v1.18.4
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.4
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5
-	github.com/containerd/containerd v1.6.18
-	github.com/docker/cli v20.10.23+incompatible
+	github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible
+	github.com/aws/aws-sdk-go-v2 v1.17.6
+	github.com/aws/aws-sdk-go-v2/config v1.18.16
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.16
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
+	github.com/containerd/containerd v1.7.0
+	github.com/docker/cli v23.0.1+incompatible
 	github.com/docker/distribution v2.8.1+incompatible
-	github.com/goharbor/acceleration-service v0.2.0
+	github.com/goharbor/acceleration-service v0.2.2
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-hclog v1.3.1
 	github.com/hashicorp/go-plugin v1.4.5
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/xattr v0.4.3
 	github.com/prometheus/client_golang v1.14.0
 	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.8.1
-	github.com/urfave/cli/v2 v2.24.2
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
-	golang.org/x/sys v0.5.0
+	github.com/stretchr/testify v1.8.2
+	github.com/urfave/cli/v2 v2.25.0
+	golang.org/x/sync v0.1.0
+	golang.org/x/sys v0.6.0
 	lukechampine.com/blake3 v1.1.5
 )

 require (
-	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/Microsoft/hcsshim v0.9.6 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/astaxie/beego v1.12.2 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
+	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.22 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.25 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/containerd/continuity v0.3.0 // indirect
-	github.com/containerd/fifo v1.0.0 // indirect
-	github.com/containerd/nydus-snapshotter v0.6.1 // indirect
-	github.com/containerd/stargz-snapshotter v0.13.0 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.14.1 // indirect
-	github.com/containerd/ttrpc v1.1.0 // indirect
-	github.com/containerd/typeurl v1.0.2 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/nydus-snapshotter v0.7.3 // indirect
+	github.com/containerd/stargz-snapshotter v0.14.3 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
+	github.com/containerd/ttrpc v1.2.1 // indirect
+	github.com/containerd/typeurl/v2 v2.1.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/docker v20.10.17+incompatible // indirect
-	github.com/docker/docker-credential-helpers v0.6.4 // indirect
+	github.com/docker/docker v23.0.1+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
-	github.com/fatih/color v1.13.0 // indirect
-	github.com/felixge/httpsnoop v1.0.2 // indirect
+	github.com/fatih/color v1.14.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/gocraft/work v0.5.1 // indirect
-	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/goharbor/harbor/src v0.0.0-20211021012518-bc6a7f65a6fa // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/gomodule/redigo v2.0.0+incompatible // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/klauspost/compress v1.16.0 // indirect
 	github.com/klauspost/cpuid v1.3.1 // indirect
-	github.com/lib/pq v1.10.0 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
-	github.com/moby/sys/signal v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/signal v0.7.0 // indirect
 	github.com/oklog/run v1.1.0 // indirect
-	github.com/opencontainers/runc v1.1.2 // indirect
-	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
-	github.com/opencontainers/selinux v1.10.1 // indirect
+	github.com/opencontainers/runc v1.1.5 // indirect
+	github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
+	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/robfig/cron v1.0.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/contrib v0.22.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.22.0 // indirect
-	go.opentelemetry.io/otel v1.3.0 // indirect
-	go.opentelemetry.io/otel/exporters/jaeger v1.0.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0 // indirect
-	go.opentelemetry.io/otel/internal/metric v0.22.0 // indirect
-	go.opentelemetry.io/otel/metric v0.22.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.3.0 // indirect
-	go.opentelemetry.io/otel/trace v1.3.0 // indirect
-	go.opentelemetry.io/proto/otlp v0.11.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
-	google.golang.org/genproto v0.0.0-20220927151529-dcaddaf36704 // indirect
-	google.golang.org/grpc v1.50.1 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/otel v1.14.0 // indirect
+	go.opentelemetry.io/otel/trace v1.14.0 // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
+	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
+	google.golang.org/grpc v1.53.0 // indirect
+	google.golang.org/protobuf v1.29.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
-
-replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.2
--- a/contrib/nydusify/go.sum
+++ b/contrib/nydusify/go.sum
--- a/contrib/nydusify/pkg/converter/config.go
+++ b/contrib/nydusify/pkg/converter/config.go
@ -4,7 +4,9 @@

 package converter

-import "strconv"
+import (
+	"strconv"
+)

 func getConfig(opt Opt) map[string]string {
 	cfg := map[string]string{}
@ -17,7 +19,7 @@ func getConfig(opt Opt) map[string]string {
 	cfg["backend_force_push"] = strconv.FormatBool(opt.BackendForcePush)

 	cfg["chunk_dict_ref"] = opt.ChunkDictRef
-	cfg["docker2oci"] = strconv.FormatBool(!opt.Docker2OCI)
+	cfg["docker2oci"] = strconv.FormatBool(opt.Docker2OCI)
 	cfg["merge_manifest"] = strconv.FormatBool(opt.MergePlatform)
 	cfg["oci_ref"] = strconv.FormatBool(opt.OCIRef)

--- a/contrib/nydusify/pkg/converter/converter.go
+++ b/contrib/nydusify/pkg/converter/converter.go
@ -6,10 +6,12 @@ package converter

 import (
 	"context"
+	"os"

 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
 	"github.com/goharbor/acceleration-service/pkg/converter"
 	"github.com/goharbor/acceleration-service/pkg/platformutil"
+	"github.com/pkg/errors"
 )

 type Opt struct {
@ -48,16 +50,33 @@ type Opt struct {
 }

 func Convert(ctx context.Context, opt Opt) error {
-	pvd, err := provider.New(opt.WorkDir, hosts(opt))
-	if err != nil {
-		return err
-	}
-
 	platformMC, err := platformutil.ParsePlatforms(opt.AllPlatforms, opt.Platforms)
 	if err != nil {
 		return err
 	}

+	if _, err := os.Stat(opt.WorkDir); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			if err := os.MkdirAll(opt.WorkDir, 0755); err != nil {
+				return errors.Wrap(err, "prepare work directory")
+			}
+			// We should only clean up when the work directory not exists
+			// before, otherwise it may delete user data by mistake.
+			defer os.RemoveAll(opt.WorkDir)
+		} else {
+			return errors.Wrap(err, "stat work directory")
+		}
+	}
+	tmpDir, err := os.MkdirTemp(opt.WorkDir, "nydusify-")
+	if err != nil {
+		return errors.Wrap(err, "create temp directory")
+	}
+	pvd, err := provider.New(tmpDir, hosts(opt), platformMC)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
 	cvt, err := converter.New(
 		converter.WithProvider(pvd),
 		converter.WithDriver("nydus", getConfig(opt)),
--- a/contrib/nydusify/pkg/converter/provider/ported.go
+++ b/contrib/nydusify/pkg/converter/provider/ported.go
@ -17,6 +17,8 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
+
+	// nolint:staticcheck
 	"github.com/containerd/containerd/remotes/docker/schema1"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -44,6 +46,7 @@ func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteCont
 		limiter       *semaphore.Weighted
 	)

+	// nolint:staticcheck
 	if desc.MediaType == images.MediaTypeDockerSchema1Manifest && rCtx.ConvertSchema1 {
 		schema1Converter := schema1.NewConverter(store, fetcher)

--- a/contrib/nydusify/pkg/converter/provider/provider.go
+++ b/contrib/nydusify/pkg/converter/provider/provider.go
@ -12,6 +12,7 @@ import (
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/content/local"
 	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
 	"github.com/goharbor/acceleration-service/pkg/remote"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -23,18 +24,20 @@ type Provider struct {
 	images       map[string]*ocispec.Descriptor
 	store        content.Store
 	hosts        remote.HostFunc
+	platformMC   platforms.MatchComparer
 }

-func New(root string, hosts remote.HostFunc) (*Provider, error) {
+func New(root string, hosts remote.HostFunc, platformMC platforms.MatchComparer) (*Provider, error) {
 	store, err := local.NewLabeledStore(root, newMemoryLabelStore())
 	if err != nil {
 		return nil, err
 	}

 	return &Provider{
-		images: make(map[string]*ocispec.Descriptor),
-		store:  store,
-		hosts:  hosts,
+		images:     make(map[string]*ocispec.Descriptor),
+		store:      store,
+		hosts:      hosts,
+		platformMC: platformMC,
 	}, nil
 }

@ -56,7 +59,8 @@ func (pvd *Provider) Pull(ctx context.Context, ref string) error {
 		return err
 	}
 	rc := &containerd.RemoteContext{
-		Resolver: resolver,
+		Resolver:        resolver,
+		PlatformMatcher: pvd.platformMC,
 	}

 	img, err := fetch(ctx, pvd.store, rc, ref, 0)
@ -77,7 +81,8 @@ func (pvd *Provider) Push(ctx context.Context, desc ocispec.Descriptor, ref stri
 		return err
 	}
 	rc := &containerd.RemoteContext{
-		Resolver: resolver,
+		Resolver:        resolver,
+		PlatformMatcher: pvd.platformMC,
 	}

 	return push(ctx, pvd.store, rc, desc, ref)
--- a/contrib/nydusify/pkg/provider/remote.go
+++ b/contrib/nydusify/pkg/provider/remote.go
@ -52,10 +52,12 @@ type withCredentialFunc = func(string) (string, string, error)
 func withRemote(ref string, insecure bool, credFunc withCredentialFunc) (*remote.Remote, error) {
 	resolverFunc := func(retryWithHTTP bool) remotes.Resolver {
 		registryHosts := docker.ConfigureDefaultRegistries(
-			docker.WithAuthorizer(docker.NewAuthorizer(
-				newDefaultClient(insecure),
-				credFunc,
-			)),
+			docker.WithAuthorizer(
+				docker.NewDockerAuthorizer(
+					docker.WithAuthClient(newDefaultClient(insecure)),
+					docker.WithAuthCreds(credFunc),
+				),
+			),
 			docker.WithClient(newDefaultClient(insecure)),
 			docker.WithPlainHTTP(func(host string) (bool, error) {
 				return retryWithHTTP, nil
--- a/deny.toml
+++ b/deny.toml
@ -74,7 +74,6 @@ allow = [
    "BSD-3-Clause",
    "BSD-2-Clause",
    "CC0-1.0",
-    "ISC",
    "Unicode-DFS-2016",
 ]
 # List of explictly disallowed licenses
@ -195,6 +194,4 @@ unknown-git = "warn"
 # if not specified. If it is specified but empty, no registries are allowed.
 allow-registry = ["https://github.com/rust-lang/crates.io-index"]
 # List of URLs for allowed Git repositories
-allow-git = [
-    "https://github.com/cloud-hypervisor/micro-http.git"
-]
+#allow-git = [ ]
--- a/rafs/src/fs.rs
+++ b/rafs/src/fs.rs
@ -102,7 +102,7 @@ impl Rafs {
            initialized: false,
            digest_validate: rafs_cfg.validate,
            fs_prefetch: rafs_cfg.prefetch.enable,
-            amplify_io: rafs_cfg.prefetch.batch_size as u32,
+            amplify_io: rafs_cfg.batch_size as u32,
            prefetch_all: rafs_cfg.prefetch.prefetch_all,
            xattr_enabled: rafs_cfg.enable_xattr,

--- a/rafs/src/metadata/direct_v6.rs
+++ b/rafs/src/metadata/direct_v6.rs
@ -410,7 +410,7 @@ impl OndiskInodeWrapper {
                    Ordering::Greater => (EROFS_BLOCK_SIZE - base) as usize,
                    Ordering::Equal => {
                        if self.size() % EROFS_BLOCK_SIZE == 0 {
-                            EROFS_BLOCK_SIZE as usize
+                            (EROFS_BLOCK_SIZE - base) as usize
                        } else {
                            (self.size() % EROFS_BLOCK_SIZE - base) as usize
                        }
--- a/rafs/src/metadata/mod.rs
+++ b/rafs/src/metadata/mod.rs
@ -699,21 +699,11 @@ impl RafsSuper {
            // Old converters extracts bootstraps from data blobs with inlined bootstrap
            // use blob digest as the bootstrap file name. The last blob in the blob table from
            // the bootstrap has wrong blod id, so we need to fix it.
-            let mut fixed = false;
            let blobs = rs.superblock.get_blob_infos();
            for blob in blobs.iter() {
                // Fix blob id for new images with old converters.
                if blob.has_feature(BlobFeatures::INLINED_FS_META) {
                    blob.set_blob_id_from_meta_path(path.as_ref())?;
-                    fixed = true;
-                }
-            }
-            if !fixed && !blob_accessible && !blobs.is_empty() {
-                // Fix blob id for old images with old converters.
-                let last = blobs.len() - 1;
-                let blob = &blobs[last];
-                if !blob.has_feature(BlobFeatures::CAP_TAR_TOC) {
-                    rs.set_blob_id_from_meta_path(path.as_ref())?;
                }
            }
        }
--- a/service/README.md
+++ b/service/README.md
@ -0,0 +1,157 @@
+# nydus-service
+
+The `nydus-service` crate helps to reuse the core services of nydus, allowing you to integrate nydus services into your project elegantly and easily. It provides:
+
+* fuse service
+* virtio-fs service
+* fscache service
+* blobcache service
+
+It also supplies the nydus daemon and the daemon controller to help manage these services.
+
+## Why you need
+
+You're supposed to know that `nydusd` running as daemon to expose a [FUSE](https://www.kernel.org/doc/html/latest/filesystems/fuse.html) mountpoint, a [Virtio-FS](https://virtio-fs.gitlab.io/) mountpoint or an [EROFS](https://docs.kernel.org/filesystems/erofs.html) mountpoint inside guest for containers to access, and it provides key features include:
+
+- Container images are downloaded on demand
+- Chunk level data deduplication
+- Flatten image metadata and data to remove all intermediate layers
+- Only usable image data is saved when building a container image
+- Only usable image data is downloaded when running a container
+- End-to-end image data integrity
+- Compatible with the OCI artifacts spec and distribution spec
+- Integrated with existing CNCF project Dragonfly to support image distribution in large clusters
+- Different container image storage backends are supported
+
+If you want to use these features as native in your project without preparing and invoking `nydusd` deliberately, `nydus-service` is just born for this.
+
+## How to use
+
+For example, reuse the fuse service with `nydus-service` in three steps.
+
+**prepare the config**:
+
+```json
+{
+  "device": {
+    "backend": {
+      "type": "registry",
+      "config": {
+        "scheme": "",
+        "skip_verify": true,
+        "timeout": 5,
+        "connect_timeout": 5,
+        "retry_limit": 4,
+        "auth": "YOUR_LOGIN_AUTH="
+      }
+    },
+    "cache": {
+      "type": "blobcache",
+      "config": {
+        "work_dir": "cache"
+      }
+    }
+  },
+  "mode": "direct",
+  "digest_validate": false,
+  "iostats_files": false,
+  "enable_xattr": true,
+  "fs_prefetch": {
+    "enable": true,
+    "threads_count": 4
+  }
+}
+```
+
+**create a daemon**:
+
+```Rust
+static ref DAEMON_CONTROLLER: DaemonController = DaemonController::default()
+
+let cmd = FsBackendMountCmd {
+  fs_type: FsBackendType::Rafs,
+  // Bootstrap path
+  source: bootstrap,
+  // Backend config
+  config,
+  // Virutal mountpoint
+  mountpoint: "/".to_string(),
+  // Prefetch files
+  prefetch_files: None,
+};
+
+let daemon = {
+  create_fuse_daemon(
+    // Mountpoint for the FUSE filesystem, target for `mount.fuse`
+    mountpoint,
+    // Vfs associated with the filesystem service object
+    vfs,
+    // Supervisor
+    None,
+    // Service instance identifier
+    id,
+    // Number of working threads to serve fuse requests
+    fuse_threads,
+    // daemon controller's waker
+    waker,
+    // Path to the Nydus daemon administration API socket
+    Some("api_sock"),
+    // Start Nydus daemon in upgrade mode
+    upgrade,
+    // Mounts FUSE filesystem in rw mode
+    !writable,
+    // FUSE server failover policy
+    failvoer-policy,
+    // Request structure to mount a backend filesystem instance
+    Some(cmd),
+    BTI.to_owned(),
+  )
+  .map(|d| {
+    info!("Fuse daemon started!");
+    d
+  })
+  .map_err(|e| {
+    error!("Failed in starting daemon: {}", e);
+    e
+  })?
+};
+
+DAEMON_CONTROLLER.set_daemon(daemon);
+```
+
+**start daemon controller**:
+
+```rust
+thread::spawn(move || {
+  let daemon = DAEMON_CONTROLLER.get_daemon();
+  if let Some(fs) = daemon.get_default_fs_service() {
+    DAEMON_CONTROLLER.set_fs_service(fs);
+  }
+
+  // Run the main event loop
+  if DAEMON_CONTROLLER.is_active() {
+    DAEMON_CONTROLLER.run_loop();
+  }
+
+  // Gracefully shutdown system.
+  info!("nydusd quits");
+  DAEMON_CONTROLLER.shutdown();
+});
+```
+
+Then, you can make the most of nydus services in your project.
+
+## Support
+
+**Platforms**:
+
+- x86_64
+- aarch64
+
+**Operating Systems**:
+
+- Linux
+
+## License
+
+This code is licensed under [Apache-2.0](LICENSE-APACHE) or [BSD-3-Clause](LICENSE-BSD-3-Clause).
--- a/smoke/go.mod
+++ b/smoke/go.mod
@ -3,35 +3,37 @@ module github.com/dragonflyoss/image-service/smoke
 go 1.18

 require (
-	github.com/containerd/containerd v1.6.18
-	github.com/containerd/nydus-snapshotter v0.6.1
-	github.com/google/uuid v1.2.0
+	github.com/containerd/containerd v1.7.0-rc.1
+	github.com/containerd/nydus-snapshotter v0.7.3
+	github.com/google/uuid v1.3.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/xattr v0.4.9
-	github.com/stretchr/testify v1.8.1
-	golang.org/x/sys v0.4.0
+	github.com/stretchr/testify v1.8.2
+	golang.org/x/sys v0.6.0
 )

 require (
-	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/Microsoft/hcsshim v0.9.6 // indirect
-	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/containerd/continuity v0.3.0 // indirect
-	github.com/containerd/fifo v1.0.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/klauspost/compress v1.15.12 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/klauspost/compress v1.16.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
-	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
-	google.golang.org/grpc v1.50.1 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
+	google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
+	google.golang.org/grpc v1.53.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
--- a/smoke/go.sum
+++ b/smoke/go.sum
--- a/smoke/tests/native_layer_test.go
+++ b/smoke/tests/native_layer_test.go
@ -78,7 +78,6 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator {
 }

 func (n *NativeLayerTestSuite) testMakeLayers(ctx tool.Context, t *testing.T) {
-
 	packOption := converter.PackOption{
 		BuilderPath: ctx.Binary.Builder,
 		Compressor:  ctx.Build.Compressor,
@ -154,6 +153,60 @@ func (n *NativeLayerTestSuite) testMakeLayers(ctx tool.Context, t *testing.T) {
 	lowerLayer.Overlay(t, upperLayer)
 	ctx.Env.BootstrapPath = overlayBootstrap
 	tool.Verify(t, ctx, lowerLayer.FileTree)
+
+	// Make base layers (use as a parent bootstrap)
+	packOption.ChunkDictPath = ""
+	baseLayer1 := texture.MakeMatrixLayer(t, filepath.Join(ctx.Env.WorkDir, "source-base-1"), "1")
+	baseLayer1BlobDigest := baseLayer1.Pack(t, packOption, ctx.Env.BlobDir)
+
+	baseLayer2 := texture.MakeMatrixLayer(t, filepath.Join(ctx.Env.WorkDir, "source-base-2"), "2")
+	baseLayer2BlobDigest := baseLayer2.Pack(t, packOption, ctx.Env.BlobDir)
+
+	lowerLayer = texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "source-lower-1"))
+	lowerBlobDigest = lowerLayer.Pack(t, packOption, ctx.Env.BlobDir)
+
+	upperLayer = texture.MakeUpperLayer(t, filepath.Join(ctx.Env.WorkDir, "source-upper-1"))
+	upperBlobDigest = upperLayer.Pack(t, packOption, ctx.Env.BlobDir)
+
+	mergeOption = converter.MergeOption{
+		BuilderPath: ctx.Binary.Builder,
+	}
+	baseLayerDigests, baseBootstrap := tool.MergeLayers(t, ctx, mergeOption, []converter.Layer{
+		{
+			Digest: baseLayer1BlobDigest,
+		},
+		{
+			Digest: baseLayer2BlobDigest,
+		},
+	})
+	ctx.Env.BootstrapPath = baseBootstrap
+	require.Equal(t, []digest.Digest{baseLayer1BlobDigest, baseLayer2BlobDigest}, baseLayerDigests)
+
+	// Test merge from a parent bootstrap
+	mergeOption = converter.MergeOption{
+		ParentBootstrapPath: baseBootstrap,
+		ChunkDictPath:       baseBootstrap,
+		BuilderPath:         ctx.Binary.Builder,
+	}
+	actualDigests, overlayBootstrap = tool.MergeLayers(t, ctx, mergeOption, []converter.Layer{
+		{
+			Digest: lowerBlobDigest,
+		},
+		{
+			Digest: upperBlobDigest,
+		},
+	})
+
+	require.Equal(t, []digest.Digest{
+		baseLayer1BlobDigest,
+		baseLayer2BlobDigest,
+		lowerBlobDigest,
+		upperBlobDigest,
+	}, actualDigests)
+
+	ctx.Env.BootstrapPath = overlayBootstrap
+	baseLayer1.Overlay(t, baseLayer2).Overlay(t, lowerLayer).Overlay(t, upperLayer)
+	tool.Verify(t, ctx, baseLayer1.FileTree)
 }

 func TestNativeLayer(t *testing.T) {
--- a/smoke/tests/texture/layer.go
+++ b/smoke/tests/texture/layer.go
@ -74,3 +74,15 @@ func MakeUpperLayer(t *testing.T, workDir string) *tool.Layer {

 	return layer
 }
+
+func MakeMatrixLayer(t *testing.T, workDir, id string) *tool.Layer {
+	layer := tool.NewLayer(t, workDir)
+
+	// Create regular file
+	file1 := fmt.Sprintf("matrix-file-%s-1", id)
+	file2 := fmt.Sprintf("matrix-file-%s-2", id)
+	layer.CreateFile(t, file1, []byte(file1))
+	layer.CreateFile(t, file2, []byte(file2))
+
+	return layer
+}
--- a/smoke/tests/tool/iterator.go
+++ b/smoke/tests/tool/iterator.go
@ -51,31 +51,30 @@ func (d *DescartesItem) Str() string {
 //
 // An example is below:
 //
-//	import (
-//        "fmt"
-//        "github.com/dragonflyoss/image-service/smoke/tests/tool"
-//    )
+//		import (
+//	       "fmt"
+//	       "github.com/dragonflyoss/image-service/smoke/tests/tool"
+//	   )
 //
-//	products := tool.DescartesIterator{}
-//	products.
-//		Dimension("name", []interface{}{"foo", "imoer", "morgan"}).
-//		Dimension("age", []interface{}{"20", "30"}).
-//		Skip(func(item *tool.DescartesItem) bool {
-//    		// skip ("morgan", "30")
-//    		return item.GetString("name") == "morgan" && param.GetString("age") == "30"
-//		})
-//
-//    // output:
-//    //       age: 20, name: foo
-//    //       age: 20, name: imoer
-//    //       age: 20, name: morgan
-//    //       age: 30, name: foo
-//    //       age: 30, name: imoer
-//    for products.HasNext(){
-//        item := products.Next()
-//        fmt.Println(item.Str())
-//    }
+//		products := tool.DescartesIterator{}
+//		products.
+//			Dimension("name", []interface{}{"foo", "imoer", "morgan"}).
+//			Dimension("age", []interface{}{"20", "30"}).
+//			Skip(func(item *tool.DescartesItem) bool {
+//	   		// skip ("morgan", "30")
+//	   		return item.GetString("name") == "morgan" && param.GetString("age") == "30"
+//			})
 //
+//	   // output:
+//	   //       age: 20, name: foo
+//	   //       age: 20, name: imoer
+//	   //       age: 20, name: morgan
+//	   //       age: 30, name: foo
+//	   //       age: 30, name: imoer
+//	   for products.HasNext(){
+//	       item := products.Next()
+//	       fmt.Println(item.Str())
+//	   }
 type DescartesIterator struct {
 	cursores  []int
 	valLists  [][]interface{}
--- a/smoke/tests/tool/layer.go
+++ b/smoke/tests/tool/layer.go
@ -164,7 +164,7 @@ func (l *Layer) PackRef(t *testing.T, ctx Context, blobDir string, compress bool
 	return ociBlobDigest, rafsBlobDigest
 }

-func (l *Layer) Overlay(t *testing.T, upper *Layer) {
+func (l *Layer) Overlay(t *testing.T, upper *Layer) *Layer {
 	// Handle whiteout/opaque files
 	for upperName := range upper.FileTree {
 		name := filepath.Base(upperName)
@ -198,6 +198,8 @@ func (l *Layer) Overlay(t *testing.T, upper *Layer) {
 			}
 		}
 	}
+
+	return l
 }

 func (l *Layer) recordFileTree(t *testing.T) {
--- a/smoke/tests/tool/test/suite.go
+++ b/smoke/tests/tool/test/suite.go
@ -45,134 +45,134 @@ type Generator func() (name string, testCase Case)
 //
 // Example1: synchronized way
 //
-//        import (
-//            "fmt"
-//            "testing"
+//	import (
+//	    "fmt"
+//	    "testing"
 //
-//            "github.com/stretchr/testify/require"
-//        )
+//	    "github.com/stretchr/testify/require"
+//	)
 //
-//        type TestSuite struct{}
+//	type TestSuite struct{}
 //
-//        func (s *TestSuite) TestOk(t *testing.T) {
-//            require.Equal(t, 1, 1)
-//        }
+//	func (s *TestSuite) TestOk(t *testing.T) {
+//	    require.Equal(t, 1, 1)
+//	}
 //
-//        func (s *TestSuite) TestFail(t *testing.T) {
-//            require.Equal(t, 1, 2)
-//        }
+//	func (s *TestSuite) TestFail(t *testing.T) {
+//	    require.Equal(t, 1, 2)
+//	}
 //
-//        func (s *TestSuite) TestDynamicTest() TestGenerator {
-//            caseNum := 0
-//            return func() (name string, testCase TestCase) {
-//                if caseNum <= 5 {
-//                    testCase = func(t *testing.T) {
-//                        require.Equal(t, 1, 2)
-//                    }
-//                }
-//                caseNum++
-//                return fmt.Sprintf("dynamic_test_%v", caseNum), testCase
-//            }
-//        }
+//	func (s *TestSuite) TestDynamicTest() TestGenerator {
+//	    caseNum := 0
+//	    return func() (name string, testCase TestCase) {
+//	        if caseNum <= 5 {
+//	            testCase = func(t *testing.T) {
+//	                require.Equal(t, 1, 2)
+//	            }
+//	        }
+//	        caseNum++
+//	        return fmt.Sprintf("dynamic_test_%v", caseNum), testCase
+//	    }
+//	}
 //
-//        func Test1(t *testing.T) {
-//            Run(t, &TestSuite{}, Sync)
-//        }
+//	func Test1(t *testing.T) {
+//	    Run(t, &TestSuite{}, Sync)
+//	}
 //
 // Output:
-//        `go test -v --parallel 4`
-//            1. The cases are serialized executed.
-//            2. The dynamic tests are generated and executed.
 //
-//        === RUN   Test1
-//        === RUN   Test1/dynamic_test_1
-//        === RUN   Test1/dynamic_test_2
-//        === RUN   Test1/dynamic_test_3
-//        === RUN   Test1/dynamic_test_4
-//        === RUN   Test1/dynamic_test_5
-//        === RUN   Test1/dynamic_test_6
-//        === RUN   Test1/TestFail
-//            suite_test.go:18:
-//                        Error Trace:    suite_test.go:18
-//                        Error:          Not equal:
-//                                        expected: 1
-//                                        actual  : 2
-//                        Test:           Test1/TestFail
-//        === RUN   Test1/TestOk
-//        --- FAIL: Test1 (0.00s)
-//            --- PASS: Test1/dynamic_test_1 (0.00s)
-//            --- PASS: Test1/dynamic_test_2 (0.00s)
-//            --- PASS: Test1/dynamic_test_3 (0.00s)
-//            --- PASS: Test1/dynamic_test_4 (0.00s)
-//            --- PASS: Test1/dynamic_test_5 (0.00s)
-//            --- PASS: Test1/dynamic_test_6 (0.00s)
-//            --- FAIL: Test1/TestFail (0.00s)
-//            --- PASS: Test1/TestOk (0.00s)
+//	`go test -v --parallel 4`
+//	    1. The cases are serialized executed.
+//	    2. The dynamic tests are generated and executed.
+//
+//	=== RUN   Test1
+//	=== RUN   Test1/dynamic_test_1
+//	=== RUN   Test1/dynamic_test_2
+//	=== RUN   Test1/dynamic_test_3
+//	=== RUN   Test1/dynamic_test_4
+//	=== RUN   Test1/dynamic_test_5
+//	=== RUN   Test1/dynamic_test_6
+//	=== RUN   Test1/TestFail
+//	    suite_test.go:18:
+//	                Error Trace:    suite_test.go:18
+//	                Error:          Not equal:
+//	                                expected: 1
+//	                                actual  : 2
+//	                Test:           Test1/TestFail
+//	=== RUN   Test1/TestOk
+//	--- FAIL: Test1 (0.00s)
+//	    --- PASS: Test1/dynamic_test_1 (0.00s)
+//	    --- PASS: Test1/dynamic_test_2 (0.00s)
+//	    --- PASS: Test1/dynamic_test_3 (0.00s)
+//	    --- PASS: Test1/dynamic_test_4 (0.00s)
+//	    --- PASS: Test1/dynamic_test_5 (0.00s)
+//	    --- PASS: Test1/dynamic_test_6 (0.00s)
+//	    --- FAIL: Test1/TestFail (0.00s)
+//	    --- PASS: Test1/TestOk (0.00s)
 //
 // Example2: asynchronized way
 //
-//        import (
-//            "fmt"
-//            "testing"
-//            "time"
-//        )
+//	import (
+//	    "fmt"
+//	    "testing"
+//	    "time"
+//	)
 //
-//        type AsyncTestSuite struct{}
+//	type AsyncTestSuite struct{}
 //
-//        func (s *AsyncTestSuite) Test1(t *testing.T) {
-//            for i := 0; i < 5; i++ {
-//                time.Sleep(time.Second)
-//            }
-//        }
+//	func (s *AsyncTestSuite) Test1(t *testing.T) {
+//	    for i := 0; i < 5; i++ {
+//	        time.Sleep(time.Second)
+//	    }
+//	}
 //
-//        func (s *AsyncTestSuite) Test2(t *testing.T) {
-//            for i := 0; i < 5; i++ {
-//                time.Sleep(time.Second)
-//            }
-//        }
+//	func (s *AsyncTestSuite) Test2(t *testing.T) {
+//	    for i := 0; i < 5; i++ {
+//	        time.Sleep(time.Second)
+//	    }
+//	}
 //
-//        func (s *AsyncTestSuite) Test3(t *testing.T) {
-//            for i := 0; i < 5; i++ {
-//                time.Sleep(time.Second)
-//            }
-//        }
+//	func (s *AsyncTestSuite) Test3(t *testing.T) {
+//	    for i := 0; i < 5; i++ {
+//	        time.Sleep(time.Second)
+//	    }
+//	}
 //
-//        func (s *AsyncTestSuite) TestDynamicTest() TestGenerator {
-//            caseNum := 0
-//            return func() (name string, testCase TestCase) {
-//                if caseNum <= 5 {
-//                    testCase = func(t *testing.T) {
-//                        for i := 0; i < 5; i++ {
-//                            time.Sleep(time.Second)
-//                        }
-//                    }
-//                }
-//                caseNum++
-//                return "", testCase
-//            }
-//        }
+//	func (s *AsyncTestSuite) TestDynamicTest() TestGenerator {
+//	    caseNum := 0
+//	    return func() (name string, testCase TestCase) {
+//	        if caseNum <= 5 {
+//	            testCase = func(t *testing.T) {
+//	                for i := 0; i < 5; i++ {
+//	                    time.Sleep(time.Second)
+//	                }
+//	            }
+//	        }
+//	        caseNum++
+//	        return "", testCase
+//	    }
+//	}
 //
-//        func Test1(t *testing.T) {
-//            Run(t, &AsyncTestSuite{})
-//        }
+//	func Test1(t *testing.T) {
+//	    Run(t, &AsyncTestSuite{})
+//	}
 //
 // Output:
-//        `go test -v --parallel 4`
-//            1. The cases are parallel executed, which leads to random completion.
-//            2. The dynamic tests are named automicly in lack of customized name.
-//
-//            --- PASS: Test1 (0.00s)
-//                --- PASS: Test1/TestDynamicTest_4 (5.00s)
-//                --- PASS: Test1/Test1 (5.00s)
-//                --- PASS: Test1/TestDynamicTest_6 (5.00s)
-//                --- PASS: Test1/TestDynamicTest_5 (5.00s)
-//                --- PASS: Test1/TestDynamicTest_2 (5.00s)
-//                --- PASS: Test1/TestDynamicTest_3 (5.00s)
-//                --- PASS: Test1/TestDynamicTest_1 (5.00s)
-//                --- PASS: Test1/Test3 (5.00s)
-//                --- PASS: Test1/Test2 (5.00s)
 //
+//	`go test -v --parallel 4`
+//	    1. The cases are parallel executed, which leads to random completion.
+//	    2. The dynamic tests are named automicly in lack of customized name.
 //
+//	    --- PASS: Test1 (0.00s)
+//	        --- PASS: Test1/TestDynamicTest_4 (5.00s)
+//	        --- PASS: Test1/Test1 (5.00s)
+//	        --- PASS: Test1/TestDynamicTest_6 (5.00s)
+//	        --- PASS: Test1/TestDynamicTest_5 (5.00s)
+//	        --- PASS: Test1/TestDynamicTest_2 (5.00s)
+//	        --- PASS: Test1/TestDynamicTest_3 (5.00s)
+//	        --- PASS: Test1/TestDynamicTest_1 (5.00s)
+//	        --- PASS: Test1/Test3 (5.00s)
+//	        --- PASS: Test1/Test2 (5.00s)
 func Run(t *testing.T, suite interface{}, opts ...Option) {

 	cases := reflect.ValueOf(suite)
--- a/src/bin/nydus-image/core/blob.rs
+++ b/src/bin/nydus-image/core/blob.rs
@ -95,7 +95,9 @@ impl Blob {
        blob_mgr: &mut BlobManager,
        blob_writer: &mut ArtifactWriter,
    ) -> Result<()> {
-        if ctx.blob_inline_meta || ctx.features.is_enabled(Feature::BlobToc) {
+        if !ctx.blob_features.contains(BlobFeatures::SEPARATE)
+            && (ctx.blob_inline_meta || ctx.features.is_enabled(Feature::BlobToc))
+        {
            if let Some((_, blob_ctx)) = blob_mgr.get_current_blob() {
                blob_ctx.write_tar_header(
                    blob_writer,
--- a/src/bin/nydus-image/core/node.rs
+++ b/src/bin/nydus-image/core/node.rs
@ -942,19 +942,44 @@ impl Node {

    pub fn v6_dir_d_size(&self, tree: &Tree) -> Result<u64> {
        ensure!(self.is_dir(), "{} is not a directory", self);
-        // Use length in byte, instead of length in character.
-        let mut d_size: u64 = (".".as_bytes().len()
-            + size_of::<RafsV6Dirent>()
-            + "..".as_bytes().len()
-            + size_of::<RafsV6Dirent>()) as u64;
+        let mut d_size = 0;

-        for child in tree.children.iter() {
-            let len = child.node.name().as_bytes().len() + size_of::<RafsV6Dirent>();
-            // erofs disk format requires dirent to be aligned with 4096.
-            if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE {
-                d_size = div_round_up(d_size as u64, EROFS_BLOCK_SIZE) * EROFS_BLOCK_SIZE;
+        // Sort all children if "." and ".." are not at the head after sorting.
+        if !tree.children.is_empty() && tree.children[0].node.name() < ".." {
+            let mut children = Vec::with_capacity(tree.children.len() + 2);
+            let dot = OsString::from(".");
+            let dotdot = OsString::from("..");
+            children.push(dot.as_os_str());
+            children.push(dotdot.as_os_str());
+            for child in tree.children.iter() {
+                children.push(child.node.name());
+            }
+            children.sort_unstable();
+
+            for c in children {
+                // Use length in byte, instead of length in character.
+                let len = c.as_bytes().len() + size_of::<RafsV6Dirent>();
+                // erofs disk format requires dirent to be aligned to block size.
+                if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE {
+                    d_size = round_up(d_size as u64, EROFS_BLOCK_SIZE);
+                }
+                d_size += len as u64;
+            }
+        } else {
+            // Avoid sorting again if "." and ".." are at the head after sorting due to that
+            // `tree.children` has already been sorted.
+            d_size = (".".as_bytes().len()
+                + size_of::<RafsV6Dirent>()
+                + "..".as_bytes().len()
+                + size_of::<RafsV6Dirent>()) as u64;
+            for child in tree.children.iter() {
+                let len = child.node.name().as_bytes().len() + size_of::<RafsV6Dirent>();
+                // erofs disk format requires dirent to be aligned to block size.
+                if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE {
+                    d_size = round_up(d_size as u64, EROFS_BLOCK_SIZE);
+                }
+                d_size += len as u64;
            }
-            d_size += len as u64;
        }

        Ok(d_size)
--- a/src/bin/nydus-image/main.rs
+++ b/src/bin/nydus-image/main.rs
@ -327,11 +327,17 @@ fn prepare_cmd_args(bti_string: &'static str) -> App {
        .subcommand(
            App::new("merge")
                .about("Merge multiple bootstraps into a overlaid bootstrap")
+                .arg(
+                    Arg::new("parent-bootstrap")
+                        .long("parent-bootstrap")
+                        .help("File path of the parent/referenced RAFS metadata blob (optional)")
+                        .required(false),
+                )
                .arg(
                    Arg::new("bootstrap")
                        .long("bootstrap")
                        .short('B')
-                        .help("output path of nydus overlaid bootstrap"),
+                        .help("Output path of nydus overlaid bootstrap"),
                )
                .arg(
                    Arg::new("blob-dir")
@ -930,8 +936,11 @@ impl Command {
        };
        ctx.configuration = config.clone();

+        let parent_bootstrap_path = Self::get_parent_bootstrap(matches)?;
+
        let output = Merger::merge(
            &mut ctx,
+            parent_bootstrap_path,
            source_bootstrap_paths,
            blob_digests,
            blob_sizes,
--- a/src/bin/nydus-image/merge.rs
+++ b/src/bin/nydus-image/merge.rs
@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0

+use std::collections::HashMap;
 use std::collections::HashSet;
 use std::convert::TryFrom;
 use std::ops::Deref;
@ -61,6 +62,7 @@ impl Merger {
    #[allow(clippy::too_many_arguments)]
    pub fn merge(
        ctx: &mut BuildContext,
+        parent_bootstrap_path: Option<String>,
        sources: Vec<PathBuf>,
        blob_digests: Option<Vec<String>>,
        blob_sizes: Option<Vec<u64>>,
@ -106,6 +108,26 @@ impl Merger {
            );
        }

+        let mut tree: Option<Tree> = None;
+        let mut blob_mgr = BlobManager::new(ctx.digester);
+
+        // Load parent bootstrap
+        let mut blob_idx_map = HashMap::new();
+        let mut parent_layers = 0;
+        if let Some(parent_bootstrap_path) = &parent_bootstrap_path {
+            let (rs, _) =
+                RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false, false)
+                    .context(format!("load parent bootstrap {:?}", parent_bootstrap_path))?;
+            tree = Some(Tree::from_bootstrap(&rs, &mut ())?);
+            let blobs = rs.superblock.get_blob_infos();
+            for blob in &blobs {
+                let blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
+                blob_idx_map.insert(blob_ctx.blob_id.clone(), blob_mgr.len());
+                blob_mgr.add(blob_ctx);
+            }
+            parent_layers = blobs.len();
+        }
+
        // Get the blobs come from chunk dict bootstrap.
        let mut chunk_dict_blobs = HashSet::new();
        let mut config = None;
@ -121,8 +143,6 @@ impl Merger {

        let mut fs_version = RafsVersion::V6;
        let mut chunk_size = None;
-        let mut tree: Option<Tree> = None;
-        let mut blob_mgr = BlobManager::new(ctx.digester);

        for (layer_idx, bootstrap_path) in sources.iter().enumerate() {
            let (rs, _) = RafsSuper::load_from_file(bootstrap_path, config_v2.clone(), true, false)
@ -136,9 +156,9 @@ impl Merger {
            ctx.digester = rs.meta.get_digester();
            ctx.explicit_uidgid = rs.meta.explicit_uidgid();

-            let mut blob_idx_map = Vec::new();
            let mut parent_blob_added = false;
-            for blob in rs.superblock.get_blob_infos() {
+            let blobs = &rs.superblock.get_blob_infos();
+            for blob in blobs {
                let mut blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
                if let Some(chunk_size) = chunk_size {
                    ensure!(
@ -191,15 +211,8 @@ impl Merger {
                    }
                }

-                let mut found = false;
-                for (idx, blob) in blob_mgr.get_blobs().iter().enumerate() {
-                    if blob.blob_id == blob_ctx.blob_id {
-                        blob_idx_map.push(idx as u32);
-                        found = true;
-                    }
-                }
-                if !found {
-                    blob_idx_map.push(blob_mgr.len() as u32);
+                if !blob_idx_map.contains_key(&blob.blob_id()) {
+                    blob_idx_map.insert(blob.blob_id().clone(), blob_mgr.len());
                    blob_mgr.add(blob_ctx);
                }
            }
@ -218,8 +231,11 @@ impl Merger {
                                ))?;
                        for chunk in &mut node.chunks {
                            let origin_blob_index = chunk.inner.blob_index() as usize;
-                            // Set the blob index of chunk to real index in blob table of final bootstrap.
-                            chunk.inner.set_blob_index(blob_idx_map[origin_blob_index]);
+                            let blob_ctx = blobs[origin_blob_index].as_ref();
+                            if let Some(blob_index) = blob_idx_map.get(&blob_ctx.blob_id()) {
+                                // Set the blob index of chunk to real index in blob table of final bootstrap.
+                                chunk.inner.set_blob_index(*blob_index as u32);
+                            }
                        }
                        // Set node's layer index to distinguish same inode number (from bootstrap)
                        // between different layers.
@ -227,7 +243,7 @@ impl Merger {
                            "too many layers {}, limited to {}",
                            layer_idx,
                            u16::MAX
-                        ))?;
+                        ))? + parent_layers as u16;
                        node.overlay = Overlay::UpperAddition;
                        match node.whiteout_type(WhiteoutSpec::Oci) {
                            // Insert whiteouts at the head, so they will be handled first when
--- a/src/lib.rs
+++ b/src/lib.rs
@ -40,7 +40,8 @@ impl<'a> ServiceArgs for SubCmdArgs<'a> {
    }

    fn is_present(&self, key: &str) -> bool {
-        self.subargs.get_flag(key) || self.args.get_flag(key)
+        matches!(self.subargs.try_get_one::<bool>(key), Ok(Some(true)))
+            || matches!(self.args.try_get_one::<bool>(key), Ok(Some(true)))
    }
 }

--- a/storage/Cargo.toml
+++ b/storage/Cargo.toml
@ -23,7 +23,7 @@ leaky-bucket = "0.12.1"
 libc = "0.2"
 log = "0.4.8"
 nix = "0.24"
-reqwest = { version = "0.11.11", features = ["blocking", "json"], optional = true }
+reqwest = { version = "0.11.14", features = ["blocking", "json"], optional = true }
 serde = { version = "1.0.110", features = ["serde_derive", "rc"] }
 serde_json = "1.0.53"
 sha2 = { version = "0.10.2", optional = true }
--- a/tests/bats/compile_nydus_snapshotter.bats
+++ b/tests/bats/compile_nydus_snapshotter.bats
@ -9,7 +9,7 @@ setup() {
 }

@test "compile nydus snapshotter" {
-        docker run --rm -v /tmp/nydus-snapshotter:/nydus-snapshotter $compile_image bash -c 'cd /nydus-snapshotter && make clear && make'
+        docker run --rm -v /tmp/nydus-snapshotter:/nydus-snapshotter $compile_image bash -c 'cd /nydus-snapshotter && make clean && make'
        if [ -f "/tmp/nydus-snapshotter/bin/containerd-nydus-grpc" ]; then
                /usr/bin/cp -f /tmp/nydus-snapshotter/bin/containerd-nydus-grpc /usr/local/bin/
                echo "nydus-snapshotter version"
--- a/utils/Cargo.toml
+++ b/utils/Cargo.toml
@ -15,6 +15,7 @@ libc = "0.2"
 log = "0.4"
 lz4-sys = "1.9.4"
 lz4 = "1.24.0"
+openssl = { version = "0.10.48", features = ["vendored"], optional = true }
 serde = { version = ">=1.0.27", features = ["serde_derive", "rc"] }
 serde_json = ">=1.0.9"
 sha2 = "0.10.0"
Author	SHA1	Message	Date
Yan Song	04fb92c5aa	action: fix smoke test for branch pattern To match `master` and `stable/*` branches at least. Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-04-20 18:32:45 +08:00
Yan Song	8c9054264c	action: upgrade golangci-lint to v1.51.2 To resolve the panic when run golangci-lint: ``` panic: load embedded ruleguard rules: rules/rules.go:13: can't load fmt ``` Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-04-17 11:43:46 +08:00
imeoer	154bbbf4c7	Merge pull request #1215 from jiangliu/v2.2-backport Backports two bugfixes from master into stable/v2.2	2023-04-17 11:02:07 +08:00
Jiang Liu	8482792dab	rafs: fix a regression caused by commit `2616fb2c05` Fix a regression caused by commit `2616fb2c05`. Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>	2023-04-14 18:04:04 +08:00
Jiang Liu	27fd2b4925	rafs: fix a possible bug in v6_dirent_size() Function Node::v6_dirent_size() may return wrong result when "." and ".." are not at the first and second entries in the sorted dirent array. Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>	2023-04-14 18:04:00 +08:00
imeoer	72da69cb3d	Merge pull request #1195 from jiangliu/is_present nydus: fix a possible panic caused by SubCmdArgs::is_present()	2023-04-10 15:26:40 +08:00
imeoer	460454a635	Merge pull request #1199 from taoohong/mushu/stable/v2.2 service: add README for nydus-service	2023-04-10 10:15:04 +08:00
taohong	c0293263ec	service: add README for nydus-service Signed-off-by: taohong <taoohong@linux.alibaba.com>	2023-04-07 16:51:37 +08:00
Jiang Liu	5153260d7a	nydus: fix a possible panic caused by SubCmdArgs::is_present() Fix a possible panic caused by SubCmdArgs::is_present(). Fixes: https://github.com/dragonflyoss/image-service/issues/1194 Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>	2023-04-05 10:54:41 +08:00
Jiang Liu	41a8e11c80	Merge pull request #1191 from adamqqqplay/v2.2-backport [backport] contrib: upgrade runc to v1.1.5	2023-03-31 16:32:22 +08:00
Qinqi Qu	5ac2a5b666	contrib: upgrade runc to v1.1.5 Runc v1.1.5 fixes three CVEs, we should upgrade it. Signed-off-by: Qinqi Qu <quqinqi@linux.alibaba.com>	2023-03-31 15:01:29 +08:00
Jiang Liu	4bcccd7ccd	deny: fix cargo deny warnings related to openssl Fix cargo deny warnings related to openssl. https://github.com/dragonflyoss/image-service/actions/runs/4522515576/jobs/7965040490 Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>	2023-03-31 15:01:01 +08:00
Jiang Liu	d2bbd82149	Merge pull request #1171 from ccx1024cc/morgan/backport backport fix/feature to stable 2.2	2023-03-24 22:33:24 +08:00
Qinqi Qu	3031f7573a	deps: bump tempfile version to 3.4.0 Update tempfile related crates to fix https://github.com/advisories/GHSA-mc8h-8q98-g5hr Signed-off-by: Qinqi Qu <quqinqi@linux.alibaba.com>	2023-03-23 18:04:25 +08:00
Yiqun Leng	3c4ceb6118	ci test: fix bug of compiling nydus-snapshotter Since developers changed "make clear" to "make clean" in the Makefile in nydus-snapshotter, it also needs to be updated in ci test. Signed-off-by: Yiqun Leng <yqleng@linux.alibaba.com>	2023-03-23 18:04:20 +08:00
泰友	6973d9db3e	fix: ci: actions are not triggered for stable/v2.2 Signed-off-by: 泰友 <cuichengxu.ccx@antgroup.com>	2023-03-23 15:27:50 +08:00
Yan Song	d885d1a25b	nydusify: cleanup work directory when conversion finish Remove the work directory to clean up the temporary image blob data after the conversion is finished. We should only clean up when the work directory not exists before, otherwise it may delete user data by mistake. Fix: https://github.com/dragonflyoss/image-service/issues/1162 Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:26:53 +08:00
Yan Song	009443b91e	nydusify: fix oci media type handle Bump nydus snapshotter v0.7.3 and bring some fixups: 1. If the original image is already an OCI type, we should forcibly set the bootstrap layer to the OCI type. 2. We need to append history item for bootstrap layer, to ensure the history consistency, see: `e5d5810851/manifest/schema1/config_builder.go (L136)` Related PR: https://github.com/containerd/nydus-snapshotter/pull/427, https://github.com/goharbor/acceleration-service/pull/119 Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:26:25 +08:00
泰友	62d213e6fa	rafs: fix amplify can not be skipped ``` json { "device":{ "backend":{ "type":"registry", "config":{ "readahead":false, "host":"dockerhub.kubekey.local", "repo":"dfns/alpine", "auth":"YWRtaw46SGFyYm9VMTIZNDU=", "scheme":"https", "skip_verify":true, "proxy":{ "fallback":false } } }, "cache":{ "type":"", "config":{ "work_dir":"/var/lib/containerd-nydus/cache", "disable_indexed_map":false } } }, "mode":"direct", "digest_validate":false, "jostats_files":true, "enable_xattr":true, "access_pattern":true, "latest_read_files":true, "batch_size":0, "amplify_io":0, "fs_prefetch":{ "enable":false, "prefetch_all":false, "threads_count":10, "merging_size":131072, "bandwidth_rate":1048576, "batch_size":0, "amplify_io":0 } } ``` `{.fs_prefetch.merging_size}` is used, instead of `{.amplify_io}` Signed-off-by: 泰友 <cuichengxu.ccx@antgroup.com>	2023-03-23 15:25:58 +08:00
Yan Song	3fb31b91c2	nydusify: forcibly enabled `--oci` option when `--oci-ref` be enabled We need to forcibly enable `--oci` option for allowing to append related annotation for zran image, otherwise an error be thrown: ``` merge nydus layers: invalid label containerd.io/snapshot/nydus-ref=: invalid checksum digest format ``` Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:25:36 +08:00
Yan Song	37e382c72d	nydusify: fix unnecessary golang-lint error ``` golangci-lint run Error: pkg/converter/provider/ported.go:47:64: SA1019: rCtx.ConvertSchema1 is deprecated: use Schema 2 or OCI images. (staticcheck) if desc.MediaType == images.MediaTypeDockerSchema1Manifest && rCtx.ConvertSchema1 { ^ Error: pkg/converter/provider/ported.go:20:2: SA1019: "github.com/containerd/containerd/remotes/docker/schema1" is deprecated: use images formatted in Docker Image Manifest v2, Schema 2, or OCI Image Spec v1. (staticcheck) "github.com/containerd/containerd/remotes/docker/schema1" ^ ``` Disabled the check, it's unnecessary to check the ported codes. Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:25:36 +08:00
Yan Song	b60e92ae6a	nydusify: fix `--oci` option for convert subcommand The `--oci` option is not working, we make it reverse before, this patch fix it and keep compatibility with the old option `--docker-v2-format`. Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:25:36 +08:00
Yan Song	da8083c550	nydusify: fix pulling all platforms of source image We should only handle specific platform for pulling by `platforms.MatchComparer`, otherwise nydusify will pull the layer data of all platforms for an source image. Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:25:36 +08:00
Yan Song	b0f5edbbc7	rafs: do not fix blob id for old bootstrap In fact, there is no way to tell if a separate old bootstrap file was inline to the blob, for example, for an old merged bootstrap, we can't set the blob id it references to as the filename, otherwise it will break blob table on loading rafs. Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:20:06 +08:00
Yan Song	a2ad16d4d2	smoke: add `--parent-bootstrap` for merge test Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:20:06 +08:00
Yan Song	7e6502711f	builder: support `--parent-bootstrap` for merge This option allows merging multiple bootstraps of upper layer with the bootstrap of a parent image, so that we can implement container commit operation for nydus image. Signed-off-by: Yan Song <imeoer@linux.alibaba.com>	2023-03-23 15:20:03 +08:00
Jiang Liu	115525298f	Merge pull request #1133 from jiangliu/v2.2-fix-get-compressed-size nydus-image: fix a underflow issue in get_compressed_size()	2023-03-03 11:20:46 +08:00
Jiang Liu	6e0f69b673	nydus-image: fix a underflow issue in get_compressed_size() Fix a underflow issue in get_compressed_size() by skipping generating useless Tar/Toc headers. Fixes: https://github.com/dragonflyoss/image-service/issues/1129 Signed-off-by: Jiang Liu <gerry@linux.alibaba.com>	2023-03-03 10:22:19 +08:00