Enable to distribute images on IPFS
Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>
This commit is contained in:
Kohei Tokunaga
parent
f316402913
commit
2ae16dea47
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
|
||||
test-integration:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
containerd: [1.4.5, 1.5.7, 1.6.0-beta.2]
|
||||
|
|
@ -77,7 +77,7 @@ jobs:
|
|||
|
||||
test-integration-rootless:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
containerd: [1.4.5, 1.5.7, 1.6.0-beta.2]
|
||||
|
|
@ -100,7 +100,7 @@ jobs:
|
|||
|
||||
cross:
|
||||
runs-on: ubuntu-20.04
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
|
|
@ -129,7 +129,7 @@ jobs:
|
|||
test-integration-cgroup2:
|
||||
# nested virtualization is only available on macOS hosts
|
||||
runs-on: macos-10.15
|
||||
timeout-minutes: 40
|
||||
timeout-minutes: 50
|
||||
env:
|
||||
VAGRANT_VAGRANTFILE: hack/Vagrantfile.fedora
|
||||
steps:
|
||||
|
|
|
|||
21
Dockerfile
21
Dockerfile
|
|
@ -34,6 +34,8 @@ ARG SLIRP4NETNS_VERSION=1.1.12
|
|||
# Extra deps: FUSE-OverlayFS
|
||||
ARG FUSE_OVERLAYFS_VERSION=1.7.1
|
||||
ARG CONTAINERD_FUSE_OVERLAYFS_VERSION=1.0.3
|
||||
# Extra deps: IPFS
|
||||
ARG IPFS_VERSION=0.10.0
|
||||
|
||||
# Test deps
|
||||
ARG GO_VERSION=1.17
|
||||
|
|
@ -161,6 +163,15 @@ RUN fname="containerd-fuse-overlayfs-${CONTAINERD_FUSE_OVERLAYFS_VERSION}-${TARG
|
|||
tar xzf "${fname}" -C /out/bin && \
|
||||
rm -f "${fname}" && \
|
||||
echo "- containerd-fuse-overlayfs: v${CONTAINERD_FUSE_OVERLAYFS_VERSION}" >> /out/share/doc/nerdctl-full/README.md
|
||||
ARG IPFS_VERSION
|
||||
RUN fname="go-ipfs_v${IPFS_VERSION}_${TARGETOS:-linux}-${TARGETARCH:-amd64}.tar.gz" && \
|
||||
curl -o "${fname}" -fSL "https://github.com/ipfs/go-ipfs/releases/download/v${IPFS_VERSION}/${fname}" && \
|
||||
grep "${fname}" "/SHA256SUMS.d/go-ipfs-${IPFS_VERSION}" | sha512sum -c && \
|
||||
tmpout=$(mktemp -d) && \
|
||||
tar -C ${tmpout} -xzf "${fname}" go-ipfs/ipfs && \
|
||||
mv ${tmpout}/go-ipfs/ipfs /out/bin/ && \
|
||||
echo "- IPFS: v${IPFS_VERSION}" >> /out/share/doc/nerdctl-full/README.md
|
||||
|
||||
RUN echo "" >> /out/share/doc/nerdctl-full/README.md && \
|
||||
echo "## License" >> /out/share/doc/nerdctl-full/README.md && \
|
||||
echo "- bin/slirp4netns: [GNU GENERAL PUBLIC LICENSE, Version 2](https://github.com/rootless-containers/slirp4netns/blob/v${SLIRP4NETNS_VERSION}/COPYING)" >> /out/share/doc/nerdctl-full/README.md && \
|
||||
|
|
@ -213,6 +224,14 @@ COPY . /go/src/github.com/containerd/nerdctl
|
|||
WORKDIR /go/src/github.com/containerd/nerdctl
|
||||
VOLUME /tmp
|
||||
ENV CGO_ENABLED=0
|
||||
# enable offline ipfs for integration test
|
||||
COPY ./Dockerfile.d/test-integration-etc_containerd-stargz-grpc_config.toml /etc/containerd-stargz-grpc/config.toml
|
||||
COPY ./Dockerfile.d/test-integration-ipfs-offline.service /usr/local/lib/systemd/system/
|
||||
# install ipfs service. avoid using 5001(api)/8080(gateway) which are reserved by tests.
|
||||
RUN systemctl enable test-integration-ipfs-offline && \
|
||||
ipfs init && \
|
||||
ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5888" && \
|
||||
ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/5889"
|
||||
CMD ["go", "test", "-v", "./cmd/nerdctl/..."]
|
||||
|
||||
FROM test-integration AS test-integration-rootless
|
||||
|
|
@ -231,6 +250,8 @@ RUN ssh-keygen -q -t rsa -f /root/.ssh/id_rsa -N '' && \
|
|||
cp -a /root/.ssh/id_rsa.pub /home/rootless/.ssh/authorized_keys && \
|
||||
mkdir -p /home/rootless/.local/share && \
|
||||
chown -R rootless:rootless /home/rootless
|
||||
# ipfs daemon for rootless containerd will be enabled in /test-integration-rootless.sh
|
||||
RUN systemctl disable test-integration-ipfs-offline
|
||||
VOLUME /home/rootless/.local/share
|
||||
RUN go test -o /usr/local/bin/nerdctl.test -c ./cmd/nerdctl
|
||||
COPY ./Dockerfile.d/test-integration-rootless.sh /
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
# From https://github.com/ipfs/go-ipfs/releases
|
||||
519d912be9367b8d5de75866aab1989954018ee1d414733ce0283735f806decb4fd11cffcb42f1beb638348a05cae95e3271c5121132e9e8e6b91ad0bfb1fda4 go-ipfs_v0.10.0_linux-amd64.tar.gz
|
||||
82a9b7a3e8701b982daa9ffbb9565ee725b50dc367a4358ab2a67e97a3b8b29b48d6c607e7edb54608181171dd9b4eded5b88bc4018b4dc348bac64de0edd26a go-ipfs_v0.10.0_linux-arm64.tar.gz
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
# Enable IPFS
|
||||
ipfs = true
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[Unit]
|
||||
Description=ipfs daemon for integration test (offline)
|
||||
|
||||
[Service]
|
||||
ExecStart=ipfs daemon --init --offline
|
||||
|
||||
[Install]
|
||||
WantedBy=docker-entrypoint.target
|
||||
|
|
@ -21,5 +21,17 @@ if [[ "$(id -u)" = "0" ]]; then
|
|||
else
|
||||
containerd-rootless-setuptool.sh install
|
||||
containerd-rootless-setuptool.sh install-buildkit
|
||||
containerd-rootless-setuptool.sh install-stargz
|
||||
cat <<EOF >> /home/rootless/.config/containerd/config.toml
|
||||
[proxy_plugins]
|
||||
[proxy_plugins."stargz"]
|
||||
type = "snapshot"
|
||||
address = "/run/user/1000/containerd-stargz-grpc/containerd-stargz-grpc.sock"
|
||||
EOF
|
||||
systemctl --user restart containerd.service
|
||||
containerd-rootless-setuptool.sh -- install-ipfs --init --offline # offline ipfs daemon for testing
|
||||
echo "ipfs = true" >> /home/rootless/.config/containerd-stargz-grpc/config.toml
|
||||
systemctl --user restart stargz-snapshotter.service
|
||||
export IPFS_PATH="/home/rootless/.local/share/ipfs"
|
||||
exec "$@"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -281,6 +281,8 @@ Run a command in a new container.
|
|||
|
||||
Usage: `nerdctl run [OPTIONS] IMAGE [COMMAND] [ARG...]`
|
||||
|
||||
:nerd_face: `ipfs://` prefix can be used for `IMAGE` to pull it from IFPS. See [`/docs/ipfs.md`](./docs/ipfs.md) for details.
|
||||
|
||||
Basic flags:
|
||||
- :whale: :window: `-i, --interactive`: Keep STDIN open even if not attached"
|
||||
- :whale: :window: `-t, --tty`: Allocate a pseudo-TTY
|
||||
|
|
@ -667,6 +669,8 @@ Pull an image from a registry.
|
|||
|
||||
Usage: `nerdctl pull [OPTIONS] NAME[:TAG|@DIGEST]`
|
||||
|
||||
:nerd_face: `ipfs://` prefix can be used for `IMAGE` to pull it from IFPS. See [`/docs/ipfs.md`](./docs/ipfs.md) for details.
|
||||
|
||||
Flags:
|
||||
- :whale: `--platform=(amd64|arm64|...)`: Pull content for a specific platform
|
||||
- :nerd_face: Unlike Docker, this flag can be specified multiple times (`--platform=amd64 --platform=arm64`)
|
||||
|
|
@ -680,6 +684,8 @@ Push an image to a registry.
|
|||
|
||||
Usage: `nerdctl push [OPTIONS] NAME[:TAG]`
|
||||
|
||||
:nerd_face: `ipfs://` prefix can be used for `IMAGE` to push it to IFPS. See [`/docs/ipfs.md`](./docs/ipfs.md) for details.
|
||||
|
||||
Flags:
|
||||
- :nerd_face: `--platform=(amd64|arm64|...)`: Push content for a specific platform
|
||||
- :nerd_face: `--all-platforms`: Push content for all platforms
|
||||
|
|
@ -1110,3 +1116,4 @@ Others:
|
|||
- [`./docs/freebsd.md`](./docs/freebsd.md): Running FreeBSD jails
|
||||
- [`./docs/multi-platform.md`](./docs/multi-platform.md): Multi-platform mode
|
||||
- [`./docs/experimental.md`](./docs/experimental.md): Experimental features
|
||||
- [`./docs/ipfs.md`](./docs/ipfs.md): Distributing images on IPFS
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/nerdctl/pkg/idutil/containerwalker"
|
||||
"github.com/containerd/nerdctl/pkg/imgutil/commit"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -85,7 +85,7 @@ func commitAction(cmd *cobra.Command, args []string) error {
|
|||
func newCommitOpts(cmd *cobra.Command, args []string) (*commit.Opts, error) {
|
||||
rawRef := args[1]
|
||||
|
||||
named, err := refdocker.ParseDockerRef(rawRef)
|
||||
named, err := referenceutil.ParseDockerRef(rawRef)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,10 +24,12 @@ import (
|
|||
"github.com/containerd/containerd"
|
||||
"github.com/containerd/containerd/errdefs"
|
||||
"github.com/containerd/containerd/platforms"
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/nerdctl/pkg/composer"
|
||||
"github.com/containerd/nerdctl/pkg/imgutil"
|
||||
"github.com/containerd/nerdctl/pkg/ipfs"
|
||||
"github.com/containerd/nerdctl/pkg/netutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
httpapi "github.com/ipfs/go-ipfs-http-client"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
|
@ -145,11 +147,11 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
|
|||
}
|
||||
|
||||
o.ImageExists = func(ctx context.Context, rawRef string) (bool, error) {
|
||||
named, err := refdocker.ParseDockerRef(rawRef)
|
||||
refNamed, err := referenceutil.ParseAny(rawRef)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ref := named.String()
|
||||
ref := refNamed.String()
|
||||
if _, err := client.ImageService().Get(ctx, ref); err != nil {
|
||||
if errors.Is(err, errdefs.ErrNotFound) {
|
||||
return false, nil
|
||||
|
|
@ -168,8 +170,18 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
|
|||
}
|
||||
ocispecPlatforms = []ocispec.Platform{parsed} // no append
|
||||
}
|
||||
_, imgErr := imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
|
||||
pullMode, insecure, ocispecPlatforms, nil)
|
||||
var imgErr error
|
||||
if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(imageName); err == nil {
|
||||
ipfsClient, err := httpapi.NewLocalApi()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, imgErr = ipfs.EnsureImage(ctx, client, ipfsClient, cmd.OutOrStdout(), snapshotter, scheme, ref,
|
||||
pullMode, ocispecPlatforms, nil)
|
||||
} else {
|
||||
_, imgErr = imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
|
||||
pullMode, insecure, ocispecPlatforms, nil)
|
||||
}
|
||||
return imgErr
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package main
|
|||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -30,8 +31,7 @@ import (
|
|||
|
||||
func TestComposeUp(t *testing.T) {
|
||||
base := testutil.NewBase(t)
|
||||
|
||||
var dockerComposeYAML = fmt.Sprintf(`
|
||||
testComposeUp(t, base, fmt.Sprintf(`
|
||||
version: '3.1'
|
||||
|
||||
services:
|
||||
|
|
@ -63,7 +63,82 @@ services:
|
|||
volumes:
|
||||
wordpress:
|
||||
db:
|
||||
`, testutil.WordpressImage, testutil.MariaDBImage)
|
||||
`, testutil.WordpressImage, testutil.MariaDBImage))
|
||||
}
|
||||
|
||||
func TestIPFSComposeUp(t *testing.T) {
|
||||
requiresIPFS(t)
|
||||
testutil.DockerIncompatible(t)
|
||||
tests := []struct {
|
||||
name string
|
||||
snapshotter string
|
||||
pushOptions []string
|
||||
requiresStargz bool
|
||||
}{
|
||||
{
|
||||
name: "overlayfs",
|
||||
snapshotter: "overlayfs",
|
||||
},
|
||||
{
|
||||
name: "stargz",
|
||||
snapshotter: "stargz",
|
||||
pushOptions: []string{"--estargz"},
|
||||
requiresStargz: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
base := testutil.NewBase(t)
|
||||
if tt.requiresStargz {
|
||||
requiresStargz(base)
|
||||
}
|
||||
ipfsImgs := make([]string, 2)
|
||||
for i, img := range []string{testutil.WordpressImage, testutil.MariaDBImage} {
|
||||
ipfsImgs[i] = pushImageToIPFS(t, base, img, tt.pushOptions...)
|
||||
}
|
||||
base.Env = append(os.Environ(), "CONTAINERD_SNAPSHOTTER="+tt.snapshotter)
|
||||
testComposeUp(t, base, fmt.Sprintf(`
|
||||
version: '3.1'
|
||||
|
||||
services:
|
||||
|
||||
wordpress:
|
||||
image: %s
|
||||
restart: always
|
||||
ports:
|
||||
- 8080:80
|
||||
environment:
|
||||
WORDPRESS_DB_HOST: db
|
||||
WORDPRESS_DB_USER: exampleuser
|
||||
WORDPRESS_DB_PASSWORD: examplepass
|
||||
WORDPRESS_DB_NAME: exampledb
|
||||
volumes:
|
||||
# workaround for https://github.com/containerd/stargz-snapshotter/issues/444
|
||||
- "/run"
|
||||
- wordpress:/var/www/html
|
||||
|
||||
db:
|
||||
image: %s
|
||||
restart: always
|
||||
environment:
|
||||
MYSQL_DATABASE: exampledb
|
||||
MYSQL_USER: exampleuser
|
||||
MYSQL_PASSWORD: examplepass
|
||||
MYSQL_RANDOM_ROOT_PASSWORD: '1'
|
||||
volumes:
|
||||
# workaround for https://github.com/containerd/stargz-snapshotter/issues/444
|
||||
- "/run"
|
||||
- db:/var/lib/mysql
|
||||
|
||||
volumes:
|
||||
wordpress:
|
||||
db:
|
||||
`, ipfsImgs[0], ipfsImgs[1]))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testComposeUp(t *testing.T, base *testutil.Base, dockerComposeYAML string) {
|
||||
comp := testutil.NewComposeDir(t, dockerComposeYAML)
|
||||
defer comp.CleanUp()
|
||||
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ import (
|
|||
|
||||
"github.com/containerd/containerd/images/converter"
|
||||
"github.com/containerd/containerd/images/converter/uncompress"
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/nerdctl/pkg/platformutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
"github.com/containerd/stargz-snapshotter/estargz"
|
||||
estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
|
||||
"github.com/containerd/stargz-snapshotter/recorder"
|
||||
|
|
@ -90,13 +90,13 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
|
|||
return errors.New("src and target image need to be specified")
|
||||
}
|
||||
|
||||
srcNamed, err := refdocker.ParseDockerRef(srcRawRef)
|
||||
srcNamed, err := referenceutil.ParseAny(srcRawRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
srcRef := srcNamed.String()
|
||||
|
||||
targetNamed, err := refdocker.ParseDockerRef(targetRawRef)
|
||||
targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,10 +23,10 @@ import (
|
|||
|
||||
"github.com/containerd/containerd/content"
|
||||
"github.com/containerd/containerd/images/converter"
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/imgcrypt/images/encryption"
|
||||
"github.com/containerd/imgcrypt/images/encryption/parsehelpers"
|
||||
"github.com/containerd/nerdctl/pkg/platformutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -105,13 +105,13 @@ func getImgcryptAction(encrypt bool) func(cmd *cobra.Command, args []string) err
|
|||
return errors.New("src and target image need to be specified")
|
||||
}
|
||||
|
||||
srcNamed, err := refdocker.ParseDockerRef(srcRawRef)
|
||||
srcNamed, err := referenceutil.ParseAny(srcRawRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
srcRef := srcNamed.String()
|
||||
|
||||
targetNamed, err := refdocker.ParseDockerRef(targetRawRef)
|
||||
targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,10 +34,10 @@ import (
|
|||
"github.com/containerd/containerd/images"
|
||||
"github.com/containerd/containerd/pkg/progress"
|
||||
"github.com/containerd/containerd/platforms"
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/containerd/snapshots"
|
||||
"github.com/containerd/nerdctl/pkg/formatter"
|
||||
"github.com/containerd/nerdctl/pkg/imgutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
"github.com/opencontainers/image-spec/identity"
|
||||
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
|
@ -79,7 +79,7 @@ func imagesAction(cmd *cobra.Command, args []string) error {
|
|||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
canonicalRef, err := refdocker.ParseDockerRef(args[0])
|
||||
canonicalRef, err := referenceutil.ParseAny(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,11 @@ import (
|
|||
"os"
|
||||
|
||||
"github.com/containerd/nerdctl/pkg/imgutil"
|
||||
"github.com/containerd/nerdctl/pkg/ipfs"
|
||||
"github.com/containerd/nerdctl/pkg/platformutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
"github.com/containerd/nerdctl/pkg/strutil"
|
||||
httpapi "github.com/ipfs/go-ipfs-http-client"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -30,7 +33,7 @@ import (
|
|||
func newPullCommand() *cobra.Command {
|
||||
var pullCommand = &cobra.Command{
|
||||
Use: "pull",
|
||||
Short: "Pull an image from a registry",
|
||||
Short: "Pull an image from a registry. Optionally specify \"ipfs://\" or \"ipns://\" scheme to pull image from IPFS.",
|
||||
RunE: pullAction,
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
|
|
@ -88,6 +91,17 @@ func pullAction(cmd *cobra.Command, args []string) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(args[0]); err == nil {
|
||||
ipfsClient, err := httpapi.NewLocalApi()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = ipfs.EnsureImage(ctx, client, ipfsClient, os.Stdout, snapshotter, scheme, ref,
|
||||
"always", ocispecPlatforms, unpack)
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = imgutil.EnsureImage(ctx, client, os.Stdout, snapshotter, args[0],
|
||||
"always", insecure, ocispecPlatforms, unpack)
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -17,16 +17,27 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/containerd/containerd/content"
|
||||
"github.com/containerd/containerd/images/converter"
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/containerd/remotes"
|
||||
"github.com/containerd/nerdctl/pkg/imgutil"
|
||||
"github.com/containerd/nerdctl/pkg/imgutil/dockerconfigresolver"
|
||||
"github.com/containerd/nerdctl/pkg/imgutil/push"
|
||||
"github.com/containerd/nerdctl/pkg/ipfs"
|
||||
"github.com/containerd/nerdctl/pkg/platformutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
"github.com/containerd/stargz-snapshotter/estargz"
|
||||
"github.com/containerd/stargz-snapshotter/estargz/zstdchunked"
|
||||
estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
|
||||
httpapi "github.com/ipfs/go-ipfs-http-client"
|
||||
digest "github.com/opencontainers/go-digest"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -34,7 +45,7 @@ import (
|
|||
func newPushCommand() *cobra.Command {
|
||||
var pushCommand = &cobra.Command{
|
||||
Use: "push NAME[:TAG]",
|
||||
Short: "Push an image or a repository to a registry",
|
||||
Short: "Push an image or a repository to a registry. Optionally specify \"ipfs://\" or \"ipns://\" scheme to push image to IPFS.",
|
||||
RunE: pushAction,
|
||||
ValidArgsFunction: pushShellComplete,
|
||||
SilenceUsage: true,
|
||||
|
|
@ -47,6 +58,9 @@ func newPushCommand() *cobra.Command {
|
|||
pushCommand.Flags().Bool("all-platforms", false, "Push content for all platforms")
|
||||
// #endregion
|
||||
|
||||
pushCommand.PersistentFlags().Bool("estargz", false, "Convert the image into eStargz")
|
||||
pushCommand.PersistentFlags().Bool("ipfs-ensure-image", true, "Ensure the entire contents of the image is locally available before push")
|
||||
|
||||
return pushCommand
|
||||
}
|
||||
|
||||
|
|
@ -55,17 +69,7 @@ func pushAction(cmd *cobra.Command, args []string) error {
|
|||
return errors.New("image name needs to be specified")
|
||||
}
|
||||
rawRef := args[0]
|
||||
named, err := refdocker.ParseDockerRef(rawRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ref := named.String()
|
||||
refDomain := refdocker.Domain(named)
|
||||
|
||||
insecure, err := cmd.Flags().GetBool("insecure-registry")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client, ctx, cancel, err := newClient(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -80,28 +84,80 @@ func pushAction(cmd *cobra.Command, args []string) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
convertEStargz, err := cmd.Flags().GetBool("estargz")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(rawRef); err == nil {
|
||||
if scheme != "ipfs" {
|
||||
return fmt.Errorf("ipfs scheme is only supported but got %q", scheme)
|
||||
}
|
||||
ensureImage, err := cmd.Flags().GetBool("ipfs-ensure-image")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logrus.Infof("pushing image %q to IPFS", ref)
|
||||
ipfsClient, err := httpapi.NewLocalApi()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var layerConvert converter.ConvertFunc
|
||||
if convertEStargz {
|
||||
layerConvert = eStargzConvertFunc()
|
||||
}
|
||||
c, err := ipfs.Push(ctx, client, ipfsClient, ref, layerConvert, allPlatforms, platform, ensureImage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout(), c.String())
|
||||
return err
|
||||
}
|
||||
|
||||
named, err := refdocker.ParseDockerRef(rawRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ref := named.String()
|
||||
refDomain := refdocker.Domain(named)
|
||||
|
||||
insecure, err := cmd.Flags().GetBool("insecure-registry")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
platMC, err := platformutil.NewMatchComparer(allPlatforms, platform)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
platRef := ref
|
||||
pushRef := ref
|
||||
if !allPlatforms {
|
||||
platRef = ref + "-tmp-reduced-platform"
|
||||
pushRef = ref + "-tmp-reduced-platform"
|
||||
// Push fails with "400 Bad Request" when the manifest is multi-platform but we do not locally have multi-platform blobs.
|
||||
// So we create a tmp reduced-platform image to avoid the error.
|
||||
platImg, err := converter.Convert(ctx, client, platRef, ref, converter.WithPlatform(platMC))
|
||||
platImg, err := converter.Convert(ctx, client, pushRef, ref, converter.WithPlatform(platMC))
|
||||
if err != nil {
|
||||
if len(platform) == 0 {
|
||||
return fmt.Errorf("failed to create a tmp single-platform image %q: %w", platRef, err)
|
||||
return fmt.Errorf("failed to create a tmp single-platform image %q: %w", pushRef, err)
|
||||
}
|
||||
return fmt.Errorf("failed to create a tmp reduced-platform image %q (platform=%v): %w", platRef, platform, err)
|
||||
return fmt.Errorf("failed to create a tmp reduced-platform image %q (platform=%v): %w", pushRef, platform, err)
|
||||
}
|
||||
defer client.ImageService().Delete(ctx, platImg.Name)
|
||||
logrus.Infof("pushing as a reduced-platform image (%s, %s)", platImg.Target.MediaType, platImg.Target.Digest)
|
||||
}
|
||||
|
||||
if convertEStargz {
|
||||
pushRef = ref + "-tmp-esgz"
|
||||
esgzImg, err := converter.Convert(ctx, client, pushRef, ref, converter.WithPlatform(platMC), converter.WithLayerConvertFunc(eStargzConvertFunc()))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to convert to eStargz: %v", err)
|
||||
}
|
||||
defer client.ImageService().Delete(ctx, esgzImg.Name)
|
||||
logrus.Infof("pushing as an eStargz image (%s, %s)", esgzImg.Target.MediaType, esgzImg.Target.Digest)
|
||||
}
|
||||
|
||||
pushFunc := func(r remotes.Resolver) error {
|
||||
return push.Push(ctx, client, r, cmd.OutOrStdout(), platRef, ref, platMC)
|
||||
return push.Push(ctx, client, r, cmd.OutOrStdout(), pushRef, ref, platMC)
|
||||
}
|
||||
|
||||
var dOpts []dockerconfigresolver.Opt
|
||||
|
|
@ -138,3 +194,44 @@ func pushShellComplete(cmd *cobra.Command, args []string, toComplete string) ([]
|
|||
// show image names
|
||||
return shellCompleteImageNames(cmd)
|
||||
}
|
||||
|
||||
func eStargzConvertFunc() converter.ConvertFunc {
|
||||
convertToESGZ := estargzconvert.LayerConvertFunc()
|
||||
return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
|
||||
if isReusableESGZ(ctx, cs, desc) {
|
||||
logrus.Infof("reusing estargz %s without conversion", desc.Digest)
|
||||
return nil, nil
|
||||
}
|
||||
newDesc, err := convertToESGZ(ctx, cs, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logrus.Infof("converted %q to %s", desc.MediaType, newDesc.Digest)
|
||||
return newDesc, err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func isReusableESGZ(ctx context.Context, cs content.Store, desc ocispec.Descriptor) bool {
|
||||
dgstStr, ok := desc.Annotations[estargz.TOCJSONDigestAnnotation]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
tocdgst, err := digest.Parse(dgstStr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ra, err := cs.ReaderAt(ctx, desc)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer ra.Close()
|
||||
r, err := estargz.Open(io.NewSectionReader(ra, 0, desc.Size), estargz.WithDecompressors(new(zstdchunked.Decompressor)))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if _, err := r.VerifyTOC(tocdgst); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ func newTestInsecureRegistry(base *testutil.Base, name, user, pass string) *test
|
|||
// listen on 0.0.0.0 to enable 127.0.0.1
|
||||
listenIP := net.ParseIP("0.0.0.0")
|
||||
const listenPort = 5000 // TODO: choose random empty port
|
||||
const authPort = 5001 // TODO: choose random empty port
|
||||
const authPort = 5100 // TODO: choose random empty port
|
||||
base.T.Logf("hostIP=%q, listenIP=%q, listenPort=%d, authPort=%d", hostIP, listenIP, listenPort, authPort)
|
||||
|
||||
registryCert, registryKey, registryClose := generateTestCert(base, hostIP.String())
|
||||
|
|
@ -116,7 +116,7 @@ func newTestInsecureRegistry(base *testutil.Base, name, user, pass string) *test
|
|||
authConfigFileName := authConfigFile.Name()
|
||||
_, err = authConfigFile.Write([]byte(fmt.Sprintf(`
|
||||
server:
|
||||
addr: ":5001"
|
||||
addr: ":5100"
|
||||
certificate: "/auth/domain.crt"
|
||||
key: "/auth/domain.key"
|
||||
token:
|
||||
|
|
@ -135,7 +135,7 @@ acl:
|
|||
authContainerName := "auth-" + name
|
||||
cmd := base.Cmd("run",
|
||||
"-d",
|
||||
"-p", fmt.Sprintf("%s:%d:5001", listenIP, authPort),
|
||||
"-p", fmt.Sprintf("%s:%d:5100", listenIP, authPort),
|
||||
"--name", authContainerName,
|
||||
"-v", authCert+":/auth/domain.crt",
|
||||
"-v", authKey+":/auth/domain.key",
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ import (
|
|||
"github.com/containerd/nerdctl/pkg/dnsutil/hostsstore"
|
||||
"github.com/containerd/nerdctl/pkg/idgen"
|
||||
"github.com/containerd/nerdctl/pkg/imgutil"
|
||||
"github.com/containerd/nerdctl/pkg/ipfs"
|
||||
"github.com/containerd/nerdctl/pkg/labels"
|
||||
"github.com/containerd/nerdctl/pkg/logging"
|
||||
"github.com/containerd/nerdctl/pkg/mountutil"
|
||||
|
|
@ -52,11 +53,13 @@ import (
|
|||
"github.com/containerd/nerdctl/pkg/netutil/nettype"
|
||||
"github.com/containerd/nerdctl/pkg/platformutil"
|
||||
"github.com/containerd/nerdctl/pkg/portutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
"github.com/containerd/nerdctl/pkg/resolvconf"
|
||||
"github.com/containerd/nerdctl/pkg/rootlessutil"
|
||||
"github.com/containerd/nerdctl/pkg/strutil"
|
||||
"github.com/containerd/nerdctl/pkg/taskutil"
|
||||
"github.com/docker/cli/opts"
|
||||
httpapi "github.com/ipfs/go-ipfs-http-client"
|
||||
"github.com/opencontainers/runtime-spec/specs-go"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
|
@ -64,7 +67,7 @@ import (
|
|||
)
|
||||
|
||||
func newRunCommand() *cobra.Command {
|
||||
shortHelp := "Run a command in a new container"
|
||||
shortHelp := "Run a command in a new container. Optionally specify \"ipfs://\" or \"ipns://\" scheme to pull image from IPFS."
|
||||
longHelp := shortHelp
|
||||
switch runtime.GOOS {
|
||||
case "windows":
|
||||
|
|
@ -711,10 +714,21 @@ func generateRootfsOpts(ctx context.Context, client *containerd.Client, platform
|
|||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
ensured, err = imgutil.EnsureImage(ctx, client, os.Stdout, snapshotter, args[0],
|
||||
pull, insecureRegistry, ocispecPlatforms, nil)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(args[0]); err == nil {
|
||||
ipfsClient, err := httpapi.NewLocalApi()
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
ensured, err = ipfs.EnsureImage(ctx, client, ipfsClient, os.Stdout, snapshotter, scheme, ref, pull, ocispecPlatforms, nil)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
} else {
|
||||
ensured, err = imgutil.EnsureImage(ctx, client, os.Stdout, snapshotter, args[0],
|
||||
pull, insecureRegistry, ocispecPlatforms, nil)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -28,10 +28,126 @@ import (
|
|||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/containerd/nerdctl/pkg/infoutil"
|
||||
"github.com/containerd/nerdctl/pkg/rootlessutil"
|
||||
"github.com/containerd/nerdctl/pkg/testutil"
|
||||
"github.com/ipfs/go-cid"
|
||||
httpapi "github.com/ipfs/go-ipfs-http-client"
|
||||
|
||||
"gotest.tools/v3/assert"
|
||||
)
|
||||
|
||||
func TestIPFS(t *testing.T) {
|
||||
requiresIPFS(t)
|
||||
testutil.DockerIncompatible(t)
|
||||
base := testutil.NewBase(t)
|
||||
ipfsCID := pushImageToIPFS(t, base, testutil.AlpineImage)
|
||||
base.Env = append(os.Environ(), "CONTAINERD_SNAPSHOTTER=overlayfs")
|
||||
base.Cmd("pull", ipfsCID).AssertOK()
|
||||
base.Cmd("run", "--rm", ipfsCID, "echo", "hello").AssertOK()
|
||||
|
||||
// encryption
|
||||
keyPair := newJWEKeyPair(t)
|
||||
defer keyPair.cleanup()
|
||||
encryptImageRef := "newimg:enc"
|
||||
layersNum := 1
|
||||
base.Cmd("image", "encrypt", "--recipient=jwe:"+keyPair.pub, ipfsCID, encryptImageRef).AssertOK()
|
||||
base.Cmd("image", "inspect", "--mode=native", "--format={{len .Manifest.Layers}}", encryptImageRef).AssertOutExactly(fmt.Sprintf("%d\n", layersNum))
|
||||
for i := 0; i < layersNum; i++ {
|
||||
base.Cmd("image", "inspect", "--mode=native", fmt.Sprintf("--format={{json (index .Manifest.Layers %d) }}", i), encryptImageRef).AssertOutContains("org.opencontainers.image.enc.keys.jwe")
|
||||
}
|
||||
ipfsCIDEnc := cidOf(t, base.Cmd("push", "ipfs://"+encryptImageRef).OutLines())
|
||||
rmiAll(base)
|
||||
|
||||
decryptImageRef := "newimg:dec"
|
||||
base.Cmd("pull", "--unpack=false", ipfsCIDEnc).AssertOK()
|
||||
base.Cmd("image", "decrypt", "--key="+keyPair.pub, ipfsCIDEnc, decryptImageRef).AssertFail() // decryption needs prv key, not pub key
|
||||
base.Cmd("image", "decrypt", "--key="+keyPair.prv, ipfsCIDEnc, decryptImageRef).AssertOK()
|
||||
base.Cmd("run", "--rm", decryptImageRef, "/bin/sh", "-c", "echo hello").AssertOK()
|
||||
}
|
||||
|
||||
func TestIPFSCommit(t *testing.T) {
|
||||
requiresIPFS(t)
|
||||
// cgroup is required for nerdctl commit
|
||||
if rootlessutil.IsRootless() && infoutil.CgroupsVersion() == "1" {
|
||||
t.Skip("test skipped for rootless containers on cgroup v1")
|
||||
}
|
||||
testutil.DockerIncompatible(t)
|
||||
base := testutil.NewBase(t)
|
||||
ipfsCID := pushImageToIPFS(t, base, testutil.AlpineImage)
|
||||
|
||||
base.Env = append(os.Environ(), "CONTAINERD_SNAPSHOTTER=overlayfs")
|
||||
base.Cmd("pull", ipfsCID).AssertOK()
|
||||
base.Cmd("run", "--rm", ipfsCID, "echo", "hello").AssertOK()
|
||||
newContainer, newImg := "hello", "helloimg:v1"
|
||||
base.Cmd("run", "--name", "hello", "-d", ipfsCID, "/bin/sh", "-c", "echo hello > /hello ; sleep 10000").AssertOK()
|
||||
base.Cmd("commit", newContainer, newImg).AssertOK()
|
||||
base.Cmd("stop", newContainer).AssertOK()
|
||||
base.Cmd("rm", newContainer).AssertOK()
|
||||
ipfsCID2 := cidOf(t, base.Cmd("push", "ipfs://"+newImg).OutLines())
|
||||
rmiAll(base)
|
||||
base.Cmd("pull", ipfsCID2).AssertOK()
|
||||
base.Cmd("run", "--rm", ipfsCID2, "/bin/sh", "-c", "cat /hello").AssertOK()
|
||||
}
|
||||
|
||||
func TestIPFSWithLazyPulling(t *testing.T) {
|
||||
requiresIPFS(t)
|
||||
testutil.DockerIncompatible(t)
|
||||
base := testutil.NewBase(t)
|
||||
ipfsCID := pushImageToIPFS(t, base, testutil.AlpineImage, "--estargz")
|
||||
|
||||
base.Env = append(os.Environ(), "CONTAINERD_SNAPSHOTTER=stargz")
|
||||
base.Cmd("pull", ipfsCID).AssertOK()
|
||||
base.Cmd("run", "--rm", ipfsCID, "ls", "/.stargz-snapshotter").AssertOK()
|
||||
}
|
||||
|
||||
func TestIPFSWithLazyPullingCommit(t *testing.T) {
|
||||
requiresIPFS(t)
|
||||
// cgroup is required for nerdctl commit
|
||||
if rootlessutil.IsRootless() && infoutil.CgroupsVersion() == "1" {
|
||||
t.Skip("test skipped for rootless containers on cgroup v1")
|
||||
}
|
||||
testutil.DockerIncompatible(t)
|
||||
base := testutil.NewBase(t)
|
||||
ipfsCID := pushImageToIPFS(t, base, testutil.AlpineImage, "--estargz")
|
||||
|
||||
base.Env = append(os.Environ(), "CONTAINERD_SNAPSHOTTER=stargz")
|
||||
base.Cmd("pull", ipfsCID).AssertOK()
|
||||
base.Cmd("run", "--rm", ipfsCID, "ls", "/.stargz-snapshotter").AssertOK()
|
||||
newContainer, newImg := "hello", "helloimg:v1"
|
||||
base.Cmd("run", "--name", "hello", "-d", ipfsCID, "/bin/sh", "-c", "echo hello > /hello ; sleep 10000").AssertOK()
|
||||
base.Cmd("commit", newContainer, newImg).AssertOK()
|
||||
base.Cmd("stop", newContainer).AssertOK()
|
||||
base.Cmd("rm", newContainer).AssertOK()
|
||||
ipfsCID2 := cidOf(t, base.Cmd("push", "--estargz", "ipfs://"+newImg).OutLines())
|
||||
rmiAll(base)
|
||||
|
||||
base.Cmd("pull", ipfsCID2).AssertOK()
|
||||
base.Cmd("run", "--rm", ipfsCID2, "/bin/sh", "-c", "ls /.stargz-snapshotter && cat /hello").AssertOK()
|
||||
base.Cmd("image", "rm", ipfsCID2).AssertOK()
|
||||
}
|
||||
|
||||
func pushImageToIPFS(t *testing.T, base *testutil.Base, name string, opts ...string) string {
|
||||
base.Cmd("pull", name).AssertOK()
|
||||
ipfsCID := cidOf(t, base.Cmd(append([]string{"push"}, append(opts, "ipfs://"+name)...)...).OutLines())
|
||||
base.Cmd("rmi", name).AssertOK()
|
||||
return ipfsCID
|
||||
}
|
||||
|
||||
func cidOf(t *testing.T, lines []string) string {
|
||||
assert.Equal(t, len(lines) >= 2, true)
|
||||
c, err := cid.Decode(lines[len(lines)-2])
|
||||
assert.NilError(t, err)
|
||||
return "ipfs://" + c.String()
|
||||
}
|
||||
|
||||
func requiresIPFS(t *testing.T) {
|
||||
if _, err := httpapi.NewLocalApi(); err != nil {
|
||||
t.Skipf("test requires ipfs daemon, but got: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func TestRunEntrypointWithBuild(t *testing.T) {
|
||||
testutil.RequiresBuild(t)
|
||||
base := testutil.NewBase(t)
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import (
|
|||
"os"
|
||||
|
||||
"github.com/containerd/containerd/images/archive"
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/nerdctl/pkg/platformutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
"github.com/mattn/go-isatty"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
|
@ -110,7 +110,7 @@ func saveImage(images []string, out io.Writer, saveOpts []archive.ExportOpt, cmd
|
|||
|
||||
imageStore := client.ImageService()
|
||||
for _, img := range images {
|
||||
named, err := refdocker.ParseDockerRef(img)
|
||||
named, err := referenceutil.ParseAny(img)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/containerd/containerd/errdefs"
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/nerdctl/pkg/idutil/imagewalker"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -70,7 +70,7 @@ func tagAction(cmd *cobra.Command, args []string) error {
|
|||
return fmt.Errorf("%s: not found", args[0])
|
||||
}
|
||||
|
||||
target, err := refdocker.ParseDockerRef(args[1])
|
||||
target, err := referenceutil.ParseDockerRef(args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,3 +6,4 @@ The following features are experimental and subject to change:
|
|||
- [FreeBSD containers](./freebsd.md)
|
||||
- Importing an external eStargz record JSON file with `nerdctl image convert --estargz-record-in=FILE` .
|
||||
eStargz itself is out of experimental.
|
||||
- [Image Distribution on IPFS](./ipfs.md)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,170 @@
|
|||
# Distribute Container Images on IPFS (Experimental)
|
||||
|
||||
You can distribute container images without registries, using IPFS.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
To use this feature, make sure `ipfs daemon` is running on your host.
|
||||
For example, you can run an IPFS daemon using the following command.
|
||||
|
||||
```
|
||||
ipfs daemon
|
||||
```
|
||||
|
||||
In rootless mode, you need to install ipfs daemon using `containerd-rootless-setuptool.sh`.
|
||||
|
||||
```
|
||||
containerd-rootless-setuptool.sh -- install-ipfs --init
|
||||
```
|
||||
|
||||
:information_source: If you don't want IPFS to communicate with nodes on the internet, you can run IPFS daemon in offline mode using `--offline` flag or you can create a private IPFS network as described [here](https://github.com/containerd/stargz-snapshotter/blob/main/docs/ipfs.md#appendix-1-creating-ipfs-private-network).
|
||||
|
||||
## IPFS-enabled image and OCI Compatibility
|
||||
|
||||
Image distribution on IPFS is achieved by OCI-compatible *IPFS-enabled image format*.
|
||||
nerdctl automatically converts an image to IPFS-enabled when necessary.
|
||||
For example, when nerdctl pushes an image to IPFS, if that image isn't an IPFS-enabled one, it converts that image to the IPFS-enabled one.
|
||||
|
||||
Please see [the doc in stargz-snapshotter project](https://github.com/containerd/stargz-snapshotter/blob/v0.10.0/docs/ipfs.md) for details about IPFS-enabled image format.
|
||||
|
||||
## Using nerdctl with IPFS
|
||||
|
||||
nerdctl supports an image name prefix `ipfs://` to handle images on IPFS.
|
||||
|
||||
### `nerdctl push ipfs://<image-name>`
|
||||
|
||||
For `nerdctl push`, you can specify `ipfs://` prefix for arbitrary image names stored in containerd.
|
||||
When this prefix is specified, nerdctl pushes that image to IPFS.
|
||||
|
||||
```console
|
||||
> nerdctl push ipfs://ubuntu:20.04
|
||||
INFO[0000] pushing image "ubuntu:20.04" to IPFS
|
||||
INFO[0000] ensuring image contents
|
||||
bafkreicq4dg6nkef5ju422ptedcwfz6kcvpvvhuqeykfrwq5krazf3muze
|
||||
```
|
||||
|
||||
At last line of the output, the IPFS CID of the pushed image is printed.
|
||||
You can use this CID to pull this image from IPFS.
|
||||
|
||||
You can also specify `--estargz` option to enable [eStargz-based lazy pulling](https://github.com/containerd/stargz-snapshotter/blob/v0.10.0/docs/ipfs.md) on IPFS.
|
||||
Please see the later section for details.
|
||||
|
||||
```console
|
||||
> nerdctl push --estargz ipfs://fedora:36
|
||||
INFO[0000] pushing image "fedora:36" to IPFS
|
||||
INFO[0000] ensuring image contents
|
||||
INFO[0011] converted "application/vnd.docker.image.rootfs.diff.tar.gzip" to sha256:cd4be969f12ef45dee7270f3643f796364045edf94cfa9ef6744d91d5cdf2208
|
||||
bafkreibp2ncujcia663uum25ustwvmyoguxqyzjnxnlhebhsgk2zowscye
|
||||
```
|
||||
|
||||
### `nerdctl pull ipfs://<CID>` and `nerdctl run ipfs://<CID>`
|
||||
|
||||
You can pull an image from IPFS by specifying `ipfs://<CID>` where `CID` is the CID of the image.
|
||||
|
||||
```console
|
||||
> nerdctl pull ipfs://bafkreicq4dg6nkef5ju422ptedcwfz6kcvpvvhuqeykfrwq5krazf3muze
|
||||
bafkreicq4dg6nkef5ju422ptedcwfz6kcvpvvhuqeykfrwq5krazf3muze: resolved |++++++++++++++++++++++++++++++++++++++|
|
||||
index-sha256:28bfa1fc6d491d3bee91bab451cab29c747e72917efacb0adc4e73faffe1f51c: done |++++++++++++++++++++++++++++++++++++++|
|
||||
manifest-sha256:f6eed19a2880f1000be1d46fb5d114d094a59e350f9d025580f7297c8d9527d5: done |++++++++++++++++++++++++++++++++++++++|
|
||||
config-sha256:ba6acccedd2923aee4c2acc6a23780b14ed4b8a5fa4e14e252a23b846df9b6c1: done |++++++++++++++++++++++++++++++++++++++|
|
||||
layer-sha256:7b1a6ab2e44dbac178598dabe7cff59bd67233dba0b27e4fbd1f9d4b3c877a54: done |++++++++++++++++++++++++++++++++++++++|
|
||||
elapsed: 1.2 s total: 27.2 M (22.7 MiB/s)
|
||||
```
|
||||
|
||||
`nerdctl run` also supports the same image name syntax.
|
||||
When specified, this command pulls the image from IPFS.
|
||||
|
||||
```console
|
||||
> nerdctl run --rm -it ipfs://bafkreicq4dg6nkef5ju422ptedcwfz6kcvpvvhuqeykfrwq5krazf3muze echo hello
|
||||
hello
|
||||
```
|
||||
|
||||
You can also push that image to the container registry.
|
||||
|
||||
```
|
||||
nerdctl tag ipfs://bafkreicq4dg6nkef5ju422ptedcwfz6kcvpvvhuqeykfrwq5krazf3muze ghcr.io/ktock/ubuntu:20.04-ipfs
|
||||
nerdctl push ghcr.io/ktock/ubuntu:20.04-ipfs
|
||||
```
|
||||
|
||||
The pushed image can run on other (IPFS-agnostic) runtimes.
|
||||
|
||||
```console
|
||||
> docker run --rm -it ghcr.io/ktock/ubuntu:20.04-ipfs echo hello
|
||||
hello
|
||||
```
|
||||
|
||||
:information_source: Note that though the IPFS-enabled image is OCI compatible, some runtimes including [containerd](https://github.com/containerd/containerd/pull/6221) and [podman](https://github.com/containers/image/pull/1403) had bugs and failed to pull that image. Containerd has already fixed this.
|
||||
|
||||
### compose on IPFS
|
||||
|
||||
`nerdctl compose` supports same image name syntax to pull images from IPFS.
|
||||
|
||||
```yaml
|
||||
version: "3.8"
|
||||
services:
|
||||
ubuntu:
|
||||
image: ipfs://bafkreicq4dg6nkef5ju422ptedcwfz6kcvpvvhuqeykfrwq5krazf3muze
|
||||
command: echo hello
|
||||
```
|
||||
|
||||
### encryption
|
||||
|
||||
You can distribute [encrypted images](./ocicrypt.md) on IPFS using OCIcrypt.
|
||||
Please see [`/docs/ocycrypt.md`](./ocicrypt.md) for details about how to ecrypt and decrypt an image.
|
||||
|
||||
Same as normal images, the encrypted image can be pushed to IPFS using `ipfs://` prefix.
|
||||
|
||||
```console
|
||||
> nerdctl image encrypt --recipient=jwe:mypubkey.pem ubuntu:20.04 ubuntu:20.04-encrypted
|
||||
sha256:a5c57411f3d11bb058b584934def0710c6c5b5a4a2d7e9b78f5480ecfc450740
|
||||
> nerdctl push ipfs://ubuntu:20.04-encrypted
|
||||
INFO[0000] pushing image "ubuntu:20.04-encrypted" to IPFS
|
||||
INFO[0000] ensuring image contents
|
||||
bafkreifajsysbvhtgd7fdgrfesszexdq6v5zbj5y2jnjfwxdjyqws2s3s4
|
||||
```
|
||||
|
||||
You can pull the encrypted image from IPFS using `ipfs://` prefix and can decrypt it in the same way as described in [`/docs/ocycrypt.md`](./ocicrypt.md).
|
||||
|
||||
```console
|
||||
> nerdctl pull --unpack=false ipfs://bafkreifajsysbvhtgd7fdgrfesszexdq6v5zbj5y2jnjfwxdjyqws2s3s4
|
||||
bafkreifajsysbvhtgd7fdgrfesszexdq6v5zbj5y2jnjfwxdjyqws2s3s4: resolved |++++++++++++++++++++++++++++++++++++++|
|
||||
index-sha256:73334fee83139d1d8dbf488b28ad100767c38428b2a62504c758905c475c1d6c: done |++++++++++++++++++++++++++++++++++++++|
|
||||
manifest-sha256:8855ae825902045ea2b27940634673ba410b61885f91b9f038f6b3303f48727c: done |++++++++++++++++++++++++++++++++++++++|
|
||||
config-sha256:ba6acccedd2923aee4c2acc6a23780b14ed4b8a5fa4e14e252a23b846df9b6c1: done |++++++++++++++++++++++++++++++++++++++|
|
||||
layer-sha256:e74a9a7749e808e4ad1e90d5a81ce3146ce270de0fbdf22429cd465df8f10a13: done |++++++++++++++++++++++++++++++++++++++|
|
||||
elapsed: 0.3 s total: 22.0 M (73.2 MiB/s)
|
||||
> nerdctl image decrypt --key=mykey.pem ipfs://bafkreifajsysbvhtgd7fdgrfesszexdq6v5zbj5y2jnjfwxdjyqws2s3s4 ubuntu:20.04-decrypted
|
||||
sha256:b0ccaddb7e7e4e702420de126468eab263eb0f3c25abf0b957ce8adcd1e82105
|
||||
> nerdctl run --rm -it ubuntu:20.04-decrypted echo hello
|
||||
hello
|
||||
```
|
||||
|
||||
## Running containers on IPFS with eStargz-based lazy pulling
|
||||
|
||||
nerdctl supports running eStargz images on IPFS with lazy pulling using Stargz Snapshotter.
|
||||
|
||||
In this configuration, Stargz Snapshotter mounts the eStargz image from IPFS to the container's rootfs using FUSE with lazy pulling support.
|
||||
Thus the container can startup without waiting for the entire image contents to be locally available.
|
||||
You can see faster container cold-start.
|
||||
|
||||
To use this feature, you need to enable Stargz Snapshotter following [`/docs/stargz.md`](./stargz.md).
|
||||
You also need to add the following configuration to `config.toml` of Stargz Snapsohtter (typically located at `/etc/containerd-stargz-grpc/config.toml`).
|
||||
|
||||
```toml
|
||||
ipfs = true
|
||||
```
|
||||
|
||||
You can push an arbitrary image to IPFS with converting it to eStargz using `--estargz` option.
|
||||
|
||||
```
|
||||
nerdctl push --estargz ipfs://fedora:36
|
||||
```
|
||||
|
||||
You can pull and run that eStargz image with lazy pulling.
|
||||
|
||||
```
|
||||
nerdctl run --rm -it ipfs://bafkreibp2ncujcia663uum25ustwvmyoguxqyzjnxnlhebhsgk2zowscye echo hello
|
||||
```
|
||||
|
||||
- See [the doc in stargz-snapshotter project](https://github.com/containerd/stargz-snapshotter/blob/v0.10.0/docs/ipfs.md) for details about lazy pulling on IPFS.
|
||||
- See [`/docs/stargz.md`](./stargz.md) for details about the configuration of nerdctl for Stargz Snapshotter.
|
||||
|
|
@ -49,6 +49,7 @@ SYSTEMD_CONTAINERD_UNIT="containerd.service"
|
|||
SYSTEMD_BUILDKIT_UNIT="buildkit.service"
|
||||
SYSTEMD_FUSE_OVERLAYFS_UNIT="containerd-fuse-overlayfs.service"
|
||||
SYSTEMD_STARGZ_UNIT="stargz-snapshotter.service"
|
||||
SYSTEMD_IPFS_UNIT="ipfs-daemon.service"
|
||||
|
||||
# global vars
|
||||
ARG0="$0"
|
||||
|
|
@ -336,6 +337,7 @@ cmd_entrypoint_install_stargz() {
|
|||
|
||||
[Service]
|
||||
Environment=PATH=$BIN:/sbin:/usr/sbin:$PATH
|
||||
Environment=IPFS_PATH=${XDG_DATA_HOME}/ipfs
|
||||
ExecStart="$REALPATH0" nsenter -- containerd-stargz-grpc -address "${XDG_RUNTIME_DIR}/containerd-stargz-grpc/containerd-stargz-grpc.sock" -root "${XDG_DATA_HOME}/containerd-stargz-grpc" -config "${XDG_CONFIG_HOME}/containerd-stargz-grpc/config.toml"
|
||||
ExecReload=/bin/kill -s HUP \$MAINPID
|
||||
RestartSec=2
|
||||
|
|
@ -358,6 +360,56 @@ cmd_entrypoint_install_stargz() {
|
|||
INFO "Set \`export CONTAINERD_SNAPSHOTTER=\"stargz\"\` to use the stargz snapshotter."
|
||||
}
|
||||
|
||||
# CLI subcommand: "install-ipfs"
|
||||
cmd_entrypoint_install_ipfs() {
|
||||
init
|
||||
if ! command -v "ipfs" >/dev/null 2>&1; then
|
||||
ERROR "ipfs needs to be present under \$PATH"
|
||||
exit 1
|
||||
fi
|
||||
if ! systemctl --user --no-pager status "${SYSTEMD_CONTAINERD_UNIT}" >/dev/null 2>&1; then
|
||||
ERROR "Install containerd first (\`$ARG0 install\`)"
|
||||
exit 1
|
||||
fi
|
||||
IPFS_PATH="${XDG_DATA_HOME}/ipfs"
|
||||
mkdir -p "${IPFS_PATH}"
|
||||
cat <<-EOT | install_systemd_unit "${SYSTEMD_IPFS_UNIT}"
|
||||
[Unit]
|
||||
Description=ipfs daemon for rootless nerdctl
|
||||
PartOf=${SYSTEMD_CONTAINERD_UNIT}
|
||||
|
||||
[Service]
|
||||
Environment=PATH=$BIN:/sbin:/usr/sbin:$PATH
|
||||
Environment=IPFS_PATH=${IPFS_PATH}
|
||||
ExecStart="$REALPATH0" nsenter -- ipfs daemon $@
|
||||
ExecReload=/bin/kill -s HUP \$MAINPID
|
||||
RestartSec=2
|
||||
Restart=always
|
||||
Type=simple
|
||||
KillMode=mixed
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
EOT
|
||||
|
||||
# Aavoid using 5001(api)/8080(gateway) which are reserved by tests.
|
||||
# TODO: support unix socket
|
||||
systemctl --user stop "${SYSTEMD_IPFS_UNIT}"
|
||||
sleep 3
|
||||
IPFS_PATH=${IPFS_PATH} ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5888"
|
||||
IPFS_PATH=${IPFS_PATH} ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/5889"
|
||||
systemctl --user restart "${SYSTEMD_IPFS_UNIT}"
|
||||
sleep 3
|
||||
|
||||
INFO "If you use stargz-snapshotter, add the following line to \"${XDG_CONFIG_HOME}/containerd-stargz-grpc/config.toml\" manually, and then run \`systemctl --user restart ${SYSTEMD_STARGZ_UNIT}\`:"
|
||||
cat <<-EOT
|
||||
### BEGIN ###
|
||||
ipfs = true
|
||||
### END ###
|
||||
EOT
|
||||
INFO "Set \`export IPFS_PATH=\"${IPFS_PATH}\"\` to use ipfs."
|
||||
}
|
||||
|
||||
# CLI subcommand: "uninstall"
|
||||
cmd_entrypoint_uninstall() {
|
||||
init
|
||||
|
|
@ -393,6 +445,14 @@ cmd_entrypoint_uninstall_stargz() {
|
|||
INFO "To remove data, run: \`$BIN/rootlesskit rm -rf ${XDG_DATA_HOME}/containerd-stargz-grpc"
|
||||
}
|
||||
|
||||
# CLI subcommand: "uninstall-ipfs"
|
||||
cmd_entrypoint_uninstall_ipfs() {
|
||||
init
|
||||
uninstall_systemd_unit "${SYSTEMD_IPFS_UNIT}"
|
||||
INFO "This uninstallation tool does NOT remove data."
|
||||
INFO "To remove data, run: \`$BIN/rootlesskit rm -rf ${XDG_DATA_HOME}/ipfs"
|
||||
}
|
||||
|
||||
# text for --help
|
||||
usage() {
|
||||
echo "Usage: ${ARG0} [OPTIONS] COMMAND"
|
||||
|
|
@ -416,6 +476,10 @@ usage() {
|
|||
echo "Add-on commands (stargz):"
|
||||
echo " install-stargz Install the systemd unit for stargz snapshotter"
|
||||
echo " uninstall-stargz Uninstall the systemd unit for stargz snapshotter"
|
||||
echo
|
||||
echo "Add-on commands (ipfs):"
|
||||
echo " install-ipfs [ipfs-daemon-flags...] Install the systemd unit for ipfs daemon. Specify \"--offline\" if run the daemon in offline mode"
|
||||
echo " uninstall-ipfs Uninstall the systemd unit for ipfs daemon"
|
||||
}
|
||||
|
||||
# parse CLI args
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -14,6 +14,7 @@ require (
|
|||
github.com/containerd/imgcrypt v1.1.2
|
||||
github.com/containerd/stargz-snapshotter v0.10.0
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.10.0
|
||||
github.com/containerd/stargz-snapshotter/ipfs v0.10.0
|
||||
github.com/containerd/typeurl v1.0.2
|
||||
github.com/containernetworking/cni v1.0.1
|
||||
github.com/containernetworking/plugins v1.0.1
|
||||
|
|
@ -25,6 +26,10 @@ require (
|
|||
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
|
||||
github.com/fatih/color v1.13.0
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/ipfs/go-cid v0.1.0
|
||||
github.com/ipfs/go-ipfs-files v0.0.9
|
||||
github.com/ipfs/go-ipfs-http-client v0.1.0
|
||||
github.com/ipfs/interface-go-ipfs-core v0.5.2
|
||||
github.com/mattn/go-isatty v0.0.14
|
||||
github.com/moby/sys/mount v0.2.0
|
||||
github.com/opencontainers/go-digest v1.0.0
|
||||
|
|
@ -43,6 +48,8 @@ require (
|
|||
)
|
||||
|
||||
replace (
|
||||
// Temporary fork for avoiding importing patent-protected code: https://github.com/hashicorp/golang-lru/issues/73
|
||||
github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c
|
||||
github.com/spf13/cobra => github.com/robberphex/cobra v1.2.2-0.20211012081327-8e3ac9400ac4 // https://github.com/spf13/cobra/pull/1503
|
||||
github.com/spf13/pflag => github.com/robberphex/pflag v1.0.6-0.20211014094653-9df3e45100fd // https://github.com/spf13/pflag/pull/333
|
||||
)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import (
|
|||
|
||||
"github.com/containerd/containerd"
|
||||
"github.com/containerd/containerd/images"
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
)
|
||||
|
||||
type Found struct {
|
||||
|
|
@ -45,7 +45,7 @@ type ImageWalker struct {
|
|||
// Returns the number of the found entries.
|
||||
func (w *ImageWalker) Walk(ctx context.Context, req string) (int, error) {
|
||||
var filters []string
|
||||
if canonicalRef, err := refdocker.ParseDockerRef(req); err == nil {
|
||||
if canonicalRef, err := referenceutil.ParseAny(req); err == nil {
|
||||
filters = append(filters, fmt.Sprintf("name==%s", canonicalRef.String()))
|
||||
}
|
||||
filters = append(filters,
|
||||
|
|
|
|||
|
|
@ -53,8 +53,8 @@ type EnsuredImage struct {
|
|||
// PullMode is either one of "always", "missing", "never"
|
||||
type PullMode = string
|
||||
|
||||
// getExistingImage may return errdefs.NotFound()
|
||||
func getExistingImage(ctx context.Context, client *containerd.Client, snapshotter, rawRef string, platform ocispec.Platform) (*EnsuredImage, error) {
|
||||
// GetExistingImage returns the specified image if exists in containerd. May return errdefs.NotFound() if not exists.
|
||||
func GetExistingImage(ctx context.Context, client *containerd.Client, snapshotter, rawRef string, platform ocispec.Platform) (*EnsuredImage, error) {
|
||||
var res *EnsuredImage
|
||||
imagewalker := &imagewalker.ImageWalker{
|
||||
Client: client,
|
||||
|
|
@ -109,7 +109,7 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout io.Write
|
|||
}
|
||||
|
||||
if mode != "always" && len(ocispecPlatforms) == 1 {
|
||||
res, err := getExistingImage(ctx, client, snapshotter, rawRef, ocispecPlatforms[0])
|
||||
res, err := GetExistingImage(ctx, client, snapshotter, rawRef, ocispecPlatforms[0])
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
|
|
@ -139,7 +139,7 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout io.Write
|
|||
return nil, err
|
||||
}
|
||||
|
||||
img, err := pullImage(ctx, client, stdout, snapshotter, resolver, ref, ocispecPlatforms, unpack)
|
||||
img, err := PullImage(ctx, client, stdout, snapshotter, resolver, ref, ocispecPlatforms, unpack)
|
||||
if err != nil {
|
||||
if !IsErrHTTPResponseToHTTPSClient(err) {
|
||||
return nil, err
|
||||
|
|
@ -151,7 +151,7 @@ func EnsureImage(ctx context.Context, client *containerd.Client, stdout io.Write
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pullImage(ctx, client, stdout, snapshotter, resolver, ref, ocispecPlatforms, unpack)
|
||||
return PullImage(ctx, client, stdout, snapshotter, resolver, ref, ocispecPlatforms, unpack)
|
||||
} else {
|
||||
logrus.WithError(err).Errorf("server %q does not seem to support HTTPS", refDomain)
|
||||
logrus.Info("Hint: you may want to try --insecure-registry to allow plain HTTP (if you are in a trusted network)")
|
||||
|
|
@ -170,7 +170,8 @@ func IsErrHTTPResponseToHTTPSClient(err error) bool {
|
|||
return strings.Contains(err.Error(), unexposed)
|
||||
}
|
||||
|
||||
func pullImage(ctx context.Context, client *containerd.Client, stdout io.Writer, snapshotter string, resolver remotes.Resolver, ref string, ocispecPlatforms []ocispec.Platform, unpack *bool) (*EnsuredImage, error) {
|
||||
// PullImage pulls an image using the specified resolver.
|
||||
func PullImage(ctx context.Context, client *containerd.Client, stdout io.Writer, snapshotter string, resolver remotes.Resolver, ref string, ocispecPlatforms []ocispec.Platform, unpack *bool) (*EnsuredImage, error) {
|
||||
ctx, done, err := client.WithLease(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -0,0 +1,151 @@
|
|||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package ipfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/containerd/containerd"
|
||||
"github.com/containerd/containerd/images"
|
||||
"github.com/containerd/containerd/images/converter"
|
||||
"github.com/containerd/containerd/remotes"
|
||||
"github.com/containerd/nerdctl/pkg/idutil/imagewalker"
|
||||
"github.com/containerd/nerdctl/pkg/imgutil"
|
||||
"github.com/containerd/nerdctl/pkg/platformutil"
|
||||
"github.com/containerd/nerdctl/pkg/referenceutil"
|
||||
"github.com/containerd/stargz-snapshotter/ipfs"
|
||||
"github.com/docker/docker/errdefs"
|
||||
"github.com/ipfs/go-cid"
|
||||
files "github.com/ipfs/go-ipfs-files"
|
||||
iface "github.com/ipfs/interface-go-ipfs-core"
|
||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// EnsureImage pull the specified image from IPFS.
|
||||
func EnsureImage(ctx context.Context, client *containerd.Client, ipfsClient iface.CoreAPI, stdout io.Writer, snapshotter string, scheme string, ref string, mode imgutil.PullMode, ocispecPlatforms []ocispec.Platform, unpack *bool) (*imgutil.EnsuredImage, error) {
|
||||
switch mode {
|
||||
case "always", "missing", "never":
|
||||
// NOP
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected pull mode: %q", mode)
|
||||
}
|
||||
switch scheme {
|
||||
case "ipfs", "ipns":
|
||||
// NOP
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected scheme: %q", scheme)
|
||||
}
|
||||
|
||||
if mode != "always" && len(ocispecPlatforms) == 1 {
|
||||
res, err := imgutil.GetExistingImage(ctx, client, snapshotter, ref, ocispecPlatforms[0])
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
if !errdefs.IsNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if mode == "never" {
|
||||
return nil, fmt.Errorf("image %q is not available", ref)
|
||||
}
|
||||
r, err := ipfs.NewResolver(ipfsClient, ipfs.ResolverOptions{
|
||||
Scheme: scheme,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return imgutil.PullImage(ctx, client, os.Stdout, snapshotter, r, ref, ocispecPlatforms, unpack)
|
||||
}
|
||||
|
||||
// Push pushes the specified image to IPFS.
|
||||
func Push(ctx context.Context, client *containerd.Client, ipfsClient iface.CoreAPI, rawRef string, layerConvert converter.ConvertFunc, allPlatforms bool, platform []string, ensureImage bool) (cid.Cid, error) {
|
||||
platMC, err := platformutil.NewMatchComparer(allPlatforms, platform)
|
||||
if err != nil {
|
||||
return cid.Cid{}, err
|
||||
}
|
||||
if ensureImage {
|
||||
// Ensure image contents are fully downloaded
|
||||
logrus.Infof("ensuring image contents")
|
||||
if err := ensureContentsOfIPFSImage(ctx, client, ipfsClient, rawRef, allPlatforms, platform); err != nil {
|
||||
logrus.WithError(err).Warnf("failed to ensure the existence of image %q", rawRef)
|
||||
}
|
||||
}
|
||||
ref, err := referenceutil.ParseAny(rawRef)
|
||||
if err != nil {
|
||||
return cid.Cid{}, err
|
||||
}
|
||||
p, err := ipfs.Push(ctx, client, ipfsClient, ref.String(), layerConvert, platMC)
|
||||
if err != nil {
|
||||
return cid.Cid{}, err
|
||||
}
|
||||
return p.Cid(), nil
|
||||
}
|
||||
|
||||
// ensureContentsOfIPFSImage ensures that the entire contents of an exisiting IPFS image are fully downloaded to containerd.
|
||||
func ensureContentsOfIPFSImage(ctx context.Context, client *containerd.Client, ipfsClient iface.CoreAPI, ref string, allPlatforms bool, platform []string) error {
|
||||
platMC, err := platformutil.NewMatchComparer(allPlatforms, platform)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var img images.Image
|
||||
walker := &imagewalker.ImageWalker{
|
||||
Client: client,
|
||||
OnFound: func(ctx context.Context, found imagewalker.Found) error {
|
||||
img = found.Image
|
||||
return nil
|
||||
},
|
||||
}
|
||||
n, err := walker.Walk(ctx, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if n == 0 {
|
||||
return fmt.Errorf("image does not exist: %q", ref)
|
||||
} else if n > 1 {
|
||||
return fmt.Errorf("ambigious reference %q matched %d objects", ref, n)
|
||||
}
|
||||
cs := client.ContentStore()
|
||||
childrenHandler := images.ChildrenHandler(cs)
|
||||
childrenHandler = images.SetChildrenLabels(cs, childrenHandler)
|
||||
childrenHandler = images.FilterPlatforms(childrenHandler, platMC)
|
||||
return images.Dispatch(ctx, images.Handlers(
|
||||
remotes.FetchHandler(cs, &fetcher{ipfsClient}),
|
||||
childrenHandler,
|
||||
), nil, img.Target)
|
||||
}
|
||||
|
||||
// fetcher fetches a file from IPFS
|
||||
// TODO: fix github.com/containerd/stargz-snapshotter/ipfs to export this and we should import that
|
||||
type fetcher struct {
|
||||
api iface.CoreAPI
|
||||
}
|
||||
|
||||
func (f *fetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
|
||||
p, err := ipfs.GetPath(desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n, err := f.api.Unixfs().Get(ctx, p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file %q: %v", p.String(), err)
|
||||
}
|
||||
return files.ToFile(n), nil
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
Copyright The containerd Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package referenceutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
refdocker "github.com/containerd/containerd/reference/docker"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
// Reference is a reference to an image.
|
||||
type Reference interface {
|
||||
|
||||
// String returns the full reference which can be understood by containerd.
|
||||
String() string
|
||||
}
|
||||
|
||||
// ParseAny parses the passed reference with allowing it to be non-docker reference.
|
||||
// If the ref has IPFS scheme or can be parsed as CID, it's parsed as an IPFS reference.
|
||||
// Otherwise it's parsed as a docker reference.
|
||||
func ParseAny(rawRef string) (Reference, error) {
|
||||
if _, ref, err := ParseIPFSRefWithScheme(rawRef); err == nil {
|
||||
return stringRef{ref}, nil
|
||||
}
|
||||
if c, err := cid.Decode(rawRef); err == nil {
|
||||
return c, nil
|
||||
}
|
||||
return ParseDockerRef(rawRef)
|
||||
}
|
||||
|
||||
// ParseDockerRef parses the passed reference with assuming it's a docker reference.
|
||||
func ParseDockerRef(rawRef string) (refdocker.Named, error) {
|
||||
return refdocker.ParseDockerRef(rawRef)
|
||||
}
|
||||
|
||||
// ParseIPFSSRefWithScheme parses the passed reference with assuming it's an IPFS reference with scheme prefix.
|
||||
func ParseIPFSRefWithScheme(name string) (scheme, ref string, err error) {
|
||||
if strings.HasPrefix(name, "ipfs://") || strings.HasPrefix(name, "ipns://") {
|
||||
return name[:4], name[7:], nil
|
||||
}
|
||||
return "", "", fmt.Errorf("reference is not an IPFS reference")
|
||||
}
|
||||
|
||||
type stringRef struct {
|
||||
s string
|
||||
}
|
||||
|
||||
func (s stringRef) String() string {
|
||||
return s.s
|
||||
}
|
||||
Loading…
Reference in New Issue