automation-tests/common/libimage/runtime.go

package libimage

import (
	"context"
	"fmt"
	"os"
	"strings"

	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/image/v5/pkg/shortnames"
	storageTransport "github.com/containers/image/v5/storage"
	"github.com/containers/image/v5/transports/alltransports"
	"github.com/containers/image/v5/types"
	"github.com/containers/storage"
	deepcopy "github.com/jinzhu/copier"
	jsoniter "github.com/json-iterator/go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// Faster than the standard library, see https://github.com/json-iterator/go.
var json = jsoniter.ConfigCompatibleWithStandardLibrary

// tmpdir returns a path to a temporary directory.
func tmpdir() string {
	tmpdir := os.Getenv("TMPDIR")
	if tmpdir == "" {
		tmpdir = "/var/tmp"
	}

	return tmpdir
}

// RuntimeOptions allow for creating a customized Runtime.
type RuntimeOptions struct {
	// The base system context of the runtime which will be used throughout
	// the entire lifespan of the Runtime.  Certain options in some
	// functions may override specific fields.
	SystemContext *types.SystemContext
}

// setRegistriesConfPath sets the registries.conf path for the specified context.
func setRegistriesConfPath(systemContext *types.SystemContext) {
	if systemContext.SystemRegistriesConfPath != "" {
		return
	}
	if envOverride, ok := os.LookupEnv("CONTAINERS_REGISTRIES_CONF"); ok {
		systemContext.SystemRegistriesConfPath = envOverride
		return
	}
	if envOverride, ok := os.LookupEnv("REGISTRIES_CONFIG_PATH"); ok {
		systemContext.SystemRegistriesConfPath = envOverride
		return
	}
}

// Runtime is responsible for image management and storing them in a containers
// storage.
type Runtime struct {
	// Use to send events out to users.
	eventChannel chan *Event
	// Underlying storage store.
	store storage.Store
	// Global system context.  No pointer to simplify copying and modifying
	// it.
	systemContext types.SystemContext
}

// Returns a copy of the runtime's system context.
func (r *Runtime) SystemContext() *types.SystemContext {
	return r.systemContextCopy()
}

// Returns a copy of the runtime's system context.
func (r *Runtime) systemContextCopy() *types.SystemContext {
	var sys types.SystemContext
	deepcopy.Copy(&sys, &r.systemContext)
	return &sys
}

// EventChannel creates a buffered channel for events that the Runtime will use
// to write events to.  Callers are expected to read from the channel in a
// timely manner.
// Can be called once for a given Runtime.
func (r *Runtime) EventChannel() chan *Event {
	if r.eventChannel != nil {
		return r.eventChannel
	}
	r.eventChannel = make(chan *Event, 100)
	return r.eventChannel
}

// RuntimeFromStore returns a Runtime for the specified store.
func RuntimeFromStore(store storage.Store, options *RuntimeOptions) (*Runtime, error) {
	if options == nil {
		options = &RuntimeOptions{}
	}

	var systemContext types.SystemContext
	if options.SystemContext != nil {
		systemContext = *options.SystemContext
	} else {
		systemContext = types.SystemContext{}
	}
	if systemContext.BigFilesTemporaryDir == "" {
		systemContext.BigFilesTemporaryDir = tmpdir()
	}

	setRegistriesConfPath(&systemContext)

	return &Runtime{
		store:         store,
		systemContext: systemContext,
	}, nil
}

// RuntimeFromStoreOptions returns a return for the specified store options.
func RuntimeFromStoreOptions(runtimeOptions *RuntimeOptions, storeOptions *storage.StoreOptions) (*Runtime, error) {
	if storeOptions == nil {
		storeOptions = &storage.StoreOptions{}
	}
	store, err := storage.GetStore(*storeOptions)
	if err != nil {
		return nil, err
	}
	storageTransport.Transport.SetStore(store)
	return RuntimeFromStore(store, runtimeOptions)
}

// Shutdown attempts to free any kernel resources which are being used by the
// underlying driver.  If "force" is true, any mounted (i.e., in use) layers
// are unmounted beforehand.  If "force" is not true, then layers being in use
// is considered to be an error condition.
func (r *Runtime) Shutdown(force bool) error {
	_, err := r.store.Shutdown(force)
	if r.eventChannel != nil {
		close(r.eventChannel)
	}
	return err
}

// storageToImage transforms a storage.Image to an Image.
func (r *Runtime) storageToImage(storageImage *storage.Image, ref types.ImageReference) *Image {
	return &Image{
		runtime:          r,
		storageImage:     storageImage,
		storageReference: ref,
	}
}

// Exists returns true if the specicifed image exists in the local containers
// storage.  Note that it may return false if an image corrupted.
func (r *Runtime) Exists(name string) (bool, error) {
	image, _, err := r.LookupImage(name, nil)
	if err != nil && errors.Cause(err) != storage.ErrImageUnknown {
		return false, err
	}
	if image == nil {
		return false, nil
	}
	if err := image.isCorrupted(name); err != nil {
		logrus.Error(err)
		return false, nil
	}
	return true, nil
}

// LookupImageOptions allow for customizing local image lookups.
type LookupImageOptions struct {
	// Lookup an image matching the specified architecture.
	Architecture string
	// Lookup an image matching the specified OS.
	OS string
	// Lookup an image matching the specified variant.
	Variant string

	// If set, do not look for items/instances in the manifest list that
	// match the current platform but return the manifest list as is.
	// only check for manifest list, return ErrNotAManifestList if not found.
	lookupManifest bool

	// If matching images resolves to a manifest list, return manifest list
	// instead of resolving to image instance, if manifest list is not found
	// try resolving image.
	ManifestList bool

	// If the image resolves to a manifest list, we usually lookup a
	// matching instance and error if none could be found.  In this case,
	// just return the manifest list.  Required for image removal.
	returnManifestIfNoInstance bool
}

// Lookup Image looks up `name` in the local container storage.  Returns the
// image and the name it has been found with.  Note that name may also use the
// `containers-storage:` prefix used to refer to the containers-storage
// transport.  Returns storage.ErrImageUnknown if the image could not be found.
//
// Unless specified via the options, the image will be looked up by name only
// without matching the architecture, os or variant.  An exception is if the
// image resolves to a manifest list, where an instance of the manifest list
// matching the local or specified platform (via options.{Architecture,OS,Variant})
// is returned.
//
// If the specified name uses the `containers-storage` transport, the resolved
// name is empty.
func (r *Runtime) LookupImage(name string, options *LookupImageOptions) (*Image, string, error) {
	logrus.Debugf("Looking up image %q in local containers storage", name)

	if options == nil {
		options = &LookupImageOptions{}
	}

	// If needed extract the name sans transport.
	storageRef, err := alltransports.ParseImageName(name)
	if err == nil {
		if storageRef.Transport().Name() != storageTransport.Transport.Name() {
			return nil, "", errors.Errorf("unsupported transport %q for looking up local images", storageRef.Transport().Name())
		}
		img, err := storageTransport.Transport.GetStoreImage(r.store, storageRef)
		if err != nil {
			return nil, "", err
		}
		logrus.Debugf("Found image %q in local containers storage (%s)", name, storageRef.StringWithinTransport())
		return r.storageToImage(img, storageRef), "", nil
	} else {
		// Docker compat: strip off the tag iff name is tagged and digested
		// (e.g., fedora:latest@sha256...).  In that case, the tag is stripped
		// off and entirely ignored.  The digest is the sole source of truth.
		normalizedName, err := normalizeTaggedDigestedString(name)
		if err != nil {
			return nil, "", err
		}
		name = normalizedName
	}

	originalName := name
	idByDigest := false
	if strings.HasPrefix(name, "sha256:") {
		// Strip off the sha256 prefix so it can be parsed later on.
		idByDigest = true
		name = strings.TrimPrefix(name, "sha256:")
	}

	// Unless specified, set the platform specified in the system context
	// for later platform matching.  Builder likes to set these things via
	// the system context at runtime creation.
	if options.Architecture == "" {
		options.Architecture = r.systemContext.ArchitectureChoice
	}
	if options.OS == "" {
		options.OS = r.systemContext.OSChoice
	}
	if options.Variant == "" {
		options.Variant = r.systemContext.VariantChoice
	}
	// Normalize platform to be OCI compatible (e.g., "aarch64" -> "arm64").
	options.OS, options.Architecture, options.Variant = NormalizePlatform(options.OS, options.Architecture, options.Variant)

	// First, check if we have an exact match in the storage. Maybe an ID
	// or a fully-qualified image name.
	img, err := r.lookupImageInLocalStorage(name, name, options)
	if err != nil {
		return nil, "", err
	}
	if img != nil {
		return img, originalName, nil
	}

	// If the name clearly referred to a local image, there's nothing we can
	// do anymore.
	if storageRef != nil || idByDigest {
		return nil, "", errors.Wrap(storage.ErrImageUnknown, originalName)
	}

	// Second, try out the candidates as resolved by shortnames. This takes
	// "localhost/" prefixed images into account as well.
	candidates, err := shortnames.ResolveLocally(&r.systemContext, name)
	if err != nil {
		return nil, "", errors.Wrap(storage.ErrImageUnknown, originalName)
	}
	// Backwards compat: normalize to docker.io as some users may very well
	// rely on that.
	if dockerNamed, err := reference.ParseDockerRef(name); err == nil {
		candidates = append(candidates, dockerNamed)
	}

	for _, candidate := range candidates {
		img, err := r.lookupImageInLocalStorage(name, candidate.String(), options)
		if err != nil {
			return nil, "", err
		}
		if img != nil {
			return img, candidate.String(), err
		}
	}

	return r.lookupImageInDigestsAndRepoTags(originalName, options)
}

// lookupImageInLocalStorage looks up the specified candidate for name in the
// storage and checks whether it's matching the system context.
func (r *Runtime) lookupImageInLocalStorage(name, candidate string, options *LookupImageOptions) (*Image, error) {
	logrus.Debugf("Trying %q ...", candidate)
	img, err := r.store.Image(candidate)
	if err != nil && errors.Cause(err) != storage.ErrImageUnknown {
		return nil, err
	}
	if img == nil {
		return nil, nil
	}
	ref, err := storageTransport.Transport.ParseStoreReference(r.store, img.ID)
	if err != nil {
		return nil, err
	}

	image := r.storageToImage(img, ref)
	logrus.Debugf("Found image %q as %q in local containers storage", name, candidate)

	// If we referenced a manifest list, we need to check whether we can
	// find a matching instance in the local containers storage.
	isManifestList, err := image.IsManifestList(context.Background())
	if err != nil {
		if errors.Cause(err) == os.ErrNotExist {
			// We must be tolerant toward corrupted images.
			// See containers/podman commit fd9dd7065d44.
			logrus.Warnf("Failed to determine if an image is a manifest list: %v, ignoring the error", err)
			return image, nil
		}
		return nil, err
	}
	if options.lookupManifest || options.ManifestList {
		if isManifestList {
			return image, nil
		}
		// return ErrNotAManifestList if lookupManifest is set otherwise try resolving image.
		if options.lookupManifest {
			return nil, errors.Wrapf(ErrNotAManifestList, candidate)
		}
	}

	if isManifestList {
		logrus.Debugf("Candidate %q is a manifest list, looking up matching instance", candidate)
		manifestList, err := image.ToManifestList()
		if err != nil {
			return nil, err
		}
		instance, err := manifestList.LookupInstance(context.Background(), options.Architecture, options.OS, options.Variant)
		if err != nil {
			if options.returnManifestIfNoInstance {
				logrus.Debug("No matching instance was found: returning manifest list instead")
				return image, nil
			}
			return nil, errors.Wrap(storage.ErrImageUnknown, err.Error())
		}
		ref, err = storageTransport.Transport.ParseStoreReference(r.store, "@"+instance.ID())
		if err != nil {
			return nil, err
		}
		image = instance
	}

	matches, err := r.imageReferenceMatchesContext(ref, options)
	if err != nil {
		return nil, err
	}

	// NOTE: if the user referenced by ID we must optimistically assume
	// that they know what they're doing.  Given, we already did the
	// manifest limbo above, we may already have resolved it.
	if !matches && !strings.HasPrefix(image.ID(), candidate) {
		return nil, nil
	}
	// Also print the string within the storage transport.  That may aid in
	// debugging when using additional stores since we see explicitly where
	// the store is and which driver (options) are used.
	logrus.Debugf("Found image %q as %q in local containers storage (%s)", name, candidate, ref.StringWithinTransport())
	return image, nil
}

// lookupImageInDigestsAndRepoTags attempts to match name against any image in
// the local containers storage.  If name is digested, it will be compared
// against image digests.  Otherwise, it will be looked up in the repo tags.
func (r *Runtime) lookupImageInDigestsAndRepoTags(name string, options *LookupImageOptions) (*Image, string, error) {
	// Until now, we've tried very hard to find an image but now it is time
	// for limbo.  If the image includes a digest that we couldn't detect
	// verbatim in the storage, we must have a look at all digests of all
	// images.  Those may change over time (e.g., via manifest lists).
	// Both Podman and Buildah want us to do that dance.
	allImages, err := r.ListImages(context.Background(), nil, nil)
	if err != nil {
		return nil, "", err
	}

	ref, err := reference.Parse(name) // Warning! This is not ParseNormalizedNamed
	if err != nil {
		return nil, "", err
	}
	named, isNamed := ref.(reference.Named)
	if !isNamed {
		return nil, "", errors.Wrap(storage.ErrImageUnknown, name)
	}

	digested, isDigested := named.(reference.Digested)
	if isDigested {
		logrus.Debug("Looking for image with matching recorded digests")
		digest := digested.Digest()
		for _, image := range allImages {
			for _, d := range image.Digests() {
				if d == digest {
					return image, name, nil
				}
			}
		}
		return nil, "", errors.Wrap(storage.ErrImageUnknown, name)
	}

	if !shortnames.IsShortName(name) {
		return nil, "", errors.Wrap(storage.ErrImageUnknown, name)
	}

	named = reference.TagNameOnly(named) // Make sure to add ":latest" if needed
	namedTagged, isNammedTagged := named.(reference.NamedTagged)
	if !isNammedTagged {
		// NOTE: this should never happen since we already know it's
		// not a digested reference.
		return nil, "", fmt.Errorf("%s: %w (could not cast to tagged)", name, storage.ErrImageUnknown)
	}

	for _, image := range allImages {
		named, err := image.inRepoTags(namedTagged)
		if err != nil {
			return nil, "", err
		}
		if named == nil {
			continue
		}
		img, err := r.lookupImageInLocalStorage(name, named.String(), options)
		if err != nil {
			return nil, "", err
		}
		if img != nil {
			return img, named.String(), err
		}
	}

	return nil, "", errors.Wrap(storage.ErrImageUnknown, name)
}

// ResolveName resolves the specified name.  If the name resolves to a local
// image, the fully resolved name will be returned.  Otherwise, the name will
// be properly normalized.
//
// Note that an empty string is returned as is.
func (r *Runtime) ResolveName(name string) (string, error) {
	if name == "" {
		return "", nil
	}
	image, resolvedName, err := r.LookupImage(name, nil)
	if err != nil && errors.Cause(err) != storage.ErrImageUnknown {
		return "", err
	}

	if image != nil && !strings.HasPrefix(image.ID(), resolvedName) {
		return resolvedName, err
	}

	normalized, err := NormalizeName(name)
	if err != nil {
		return "", err
	}

	return normalized.String(), nil
}

// imageReferenceMatchesContext return true if the specified reference matches
// the platform (os, arch, variant) as specified by the lookup options.
func (r *Runtime) imageReferenceMatchesContext(ref types.ImageReference, options *LookupImageOptions) (bool, error) {
	if options.Architecture+options.OS+options.Variant == "" {
		return true, nil
	}

	ctx := context.Background()
	img, err := ref.NewImage(ctx, &r.systemContext)
	if err != nil {
		return false, err
	}
	defer img.Close()
	data, err := img.Inspect(ctx)
	if err != nil {
		return false, err
	}

	if options.Architecture != "" && options.Architecture != data.Architecture {
		logrus.Debugf("architecture %q does not match architecture %q of image %s", options.Architecture, data.Architecture, ref)
		return false, nil
	}
	if options.OS != "" && options.OS != data.Os {
		logrus.Debugf("OS %q does not match OS %q of image %s", options.OS, data.Os, ref)
		return false, nil
	}
	if options.Variant != "" && options.Variant != data.Variant {
		logrus.Debugf("variant %q does not match variant %q of image %s", options.Variant, data.Variant, ref)
		return false, nil
	}

	return true, nil
}

// IsExternalContainerFunc allows for checking whether the specified container
// is an external one.  The definition of an external container can be set by
// callers.
type IsExternalContainerFunc func(containerID string) (bool, error)

// ListImagesOptions allow for customizing listing images.
type ListImagesOptions struct {
	// Filters to filter the listed images.  Supported filters are
	// * after,before,since=image
	// * containers=true,false,external
	// * dangling=true,false
	// * intermediate=true,false (useful for pruning images)
	// * id=id
	// * label=key[=value]
	// * readonly=true,false
	// * reference=name[:tag] (wildcards allowed)
	Filters []string
	// IsExternalContainerFunc allows for checking whether the specified
	// container is an external one (when containers=external filter is
	// used).  The definition of an external container can be set by
	// callers.
	IsExternalContainerFunc IsExternalContainerFunc
}

// ListImages lists images in the local container storage.  If names are
// specified, only images with the specified names are looked up and filtered.
func (r *Runtime) ListImages(ctx context.Context, names []string, options *ListImagesOptions) ([]*Image, error) {
	if options == nil {
		options = &ListImagesOptions{}
	}

	var images []*Image
	if len(names) > 0 {
		for _, name := range names {
			image, _, err := r.LookupImage(name, nil)
			if err != nil {
				return nil, err
			}
			images = append(images, image)
		}
	} else {
		storageImages, err := r.store.Images()
		if err != nil {
			return nil, err
		}
		for i := range storageImages {
			images = append(images, r.storageToImage(&storageImages[i], nil))
		}
	}

	return r.filterImages(ctx, images, options)
}

// RemoveImagesOptions allow for customizing image removal.
type RemoveImagesOptions struct {
	// Force will remove all containers from the local storage that are
	// using a removed image.  Use RemoveContainerFunc for a custom logic.
	// If set, all child images will be removed as well.
	Force bool
	// LookupManifest will expect all specified names to be manifest lists (no instance look up).
	// This allows for removing manifest lists.
	// By default, RemoveImages will attempt to resolve to a manifest instance matching
	// the local platform (i.e., os, architecture, variant).
	LookupManifest bool
	// RemoveContainerFunc allows for a custom logic for removing
	// containers using a specific image.  By default, all containers in
	// the local containers storage will be removed (if Force is set).
	RemoveContainerFunc RemoveContainerFunc
	// IsExternalContainerFunc allows for checking whether the specified
	// container is an external one (when containers=external filter is
	// used).  The definition of an external container can be set by
	// callers.
	IsExternalContainerFunc IsExternalContainerFunc
	// Remove external containers even when Force is false.  Requires
	// IsExternalContainerFunc to be specified.
	ExternalContainers bool
	// Filters to filter the removed images.  Supported filters are
	// * after,before,since=image
	// * containers=true,false,external
	// * dangling=true,false
	// * intermediate=true,false (useful for pruning images)
	// * id=id
	// * label=key[=value]
	// * readonly=true,false
	// * reference=name[:tag] (wildcards allowed)
	Filters []string
	// The RemoveImagesReport will include the size of the removed image.
	// This information may be useful when pruning images to figure out how
	// much space was freed. However, computing the size of an image is
	// comparatively expensive, so it is made optional.
	WithSize bool
}

// RemoveImages removes images specified by names.  If no names are specified,
// remove images as specified via the options' filters.  All images are
// expected to exist in the local containers storage.
//
// If an image has more names than one name, the image will be untagged with
// the specified name.  RemoveImages returns a slice of untagged and removed
// images.
//
// Note that most errors are non-fatal and collected into `rmErrors` return
// value.
func (r *Runtime) RemoveImages(ctx context.Context, names []string, options *RemoveImagesOptions) (reports []*RemoveImageReport, rmErrors []error) {
	if options == nil {
		options = &RemoveImagesOptions{}
	}

	if options.ExternalContainers && options.IsExternalContainerFunc == nil {
		return nil, []error{fmt.Errorf("libimage error: cannot remove external containers without callback")}
	}

	// The logic here may require some explanation.  Image removal is
	// surprisingly complex since it is recursive (intermediate parents are
	// removed) and since multiple items in `names` may resolve to the
	// *same* image.  On top, the data in the containers storage is shared,
	// so we need to be careful and the code must be robust.  That is why
	// users can only remove images via this function; the logic may be
	// complex but the execution path is clear.

	// Bundle an image with a possible empty slice of names to untag.  That
	// allows for a decent untagging logic and to bundle multiple
	// references to the same *Image (and circumvent consistency issues).
	type deleteMe struct {
		image        *Image
		referencedBy []string
	}

	appendError := func(err error) {
		rmErrors = append(rmErrors, err)
	}

	deleteMap := make(map[string]*deleteMe) // ID -> deleteMe
	toDelete := []string{}
	// Look up images in the local containers storage and fill out
	// toDelete and the deleteMap.

	switch {
	case len(names) > 0:
		// prepare lookupOptions
		var lookupOptions *LookupImageOptions
		if options.LookupManifest {
			// LookupManifest configured as true make sure we only remove manifests and no referenced images.
			lookupOptions = &LookupImageOptions{lookupManifest: true}
		} else {
			lookupOptions = &LookupImageOptions{returnManifestIfNoInstance: true}
		}
		// Look up the images one-by-one.  That allows for removing
		// images that have been looked up successfully while reporting
		// lookup errors at the end.
		for _, name := range names {
			img, resolvedName, err := r.LookupImage(name, lookupOptions)
			if err != nil {
				appendError(err)
				continue
			}
			dm, exists := deleteMap[img.ID()]
			if !exists {
				toDelete = append(toDelete, img.ID())
				dm = &deleteMe{image: img}
				deleteMap[img.ID()] = dm
			}
			dm.referencedBy = append(dm.referencedBy, resolvedName)
		}

	default:
		options := &ListImagesOptions{
			IsExternalContainerFunc: options.IsExternalContainerFunc,
			Filters:                 options.Filters,
		}
		filteredImages, err := r.ListImages(ctx, nil, options)
		if err != nil {
			appendError(err)
			return nil, rmErrors
		}
		for _, img := range filteredImages {
			toDelete = append(toDelete, img.ID())
			deleteMap[img.ID()] = &deleteMe{image: img}
		}
	}

	// Return early if there's no image to delete.
	if len(deleteMap) == 0 {
		return nil, rmErrors
	}

	// Now remove the images in the given order.
	rmMap := make(map[string]*RemoveImageReport)
	orderedIDs := []string{}
	visitedIDs := make(map[string]bool)
	for _, id := range toDelete {
		del, exists := deleteMap[id]
		if !exists {
			appendError(errors.Errorf("internal error: ID %s not in found in image-deletion map", id))
			continue
		}
		if len(del.referencedBy) == 0 {
			del.referencedBy = []string{""}
		}
		for _, ref := range del.referencedBy {
			processedIDs, err := del.image.remove(ctx, rmMap, ref, options)
			if err != nil {
				appendError(err)
			}
			// NOTE: make sure to add given ID only once to orderedIDs.
			for _, id := range processedIDs {
				if visited := visitedIDs[id]; visited {
					continue
				}
				orderedIDs = append(orderedIDs, id)
				visitedIDs[id] = true
			}
		}
	}

	// Finally, we can assemble the reports slice.
	for _, id := range orderedIDs {
		report, exists := rmMap[id]
		if exists {
			reports = append(reports, report)
		}
	}

	return reports, rmErrors
}