vendor: update c/common + libhvee to latest main

Signed-off-by: Paul Holzinger <pholzing@redhat.com>
This commit is contained in:
Paul Holzinger 2024-02-05 17:10:24 +01:00
parent f439f4e9da
commit 5de4bd5d13
No known key found for this signature in database
GPG Key ID: EB145DD938A3CAF2
10 changed files with 731 additions and 69 deletions

4
go.mod
View File

@ -11,11 +11,11 @@ require (
github.com/checkpoint-restore/go-criu/v7 v7.0.0
github.com/containernetworking/plugins v1.4.0
github.com/containers/buildah v1.34.1-0.20240201124221-b850c711ff5c
github.com/containers/common v0.57.1-0.20240205132223-de5cb00e891c
github.com/containers/common v0.57.1-0.20240206153655-323e410f34bf
github.com/containers/conmon v2.0.20+incompatible
github.com/containers/gvisor-tap-vsock v0.7.2
github.com/containers/image/v5 v5.29.2-0.20240130233108-e66a1ade2efc
github.com/containers/libhvee v0.6.0
github.com/containers/libhvee v0.6.1-0.20240205152934-3a16bce3e4be
github.com/containers/ocicrypt v1.1.9
github.com/containers/psgo v1.8.0
github.com/containers/storage v1.52.1-0.20240202181245-1419a5980565

8
go.sum
View File

@ -257,16 +257,16 @@ github.com/containernetworking/plugins v1.4.0 h1:+w22VPYgk7nQHw7KT92lsRmuToHvb7w
github.com/containernetworking/plugins v1.4.0/go.mod h1:UYhcOyjefnrQvKvmmyEKsUA+M9Nfn7tqULPpH0Pkcj0=
github.com/containers/buildah v1.34.1-0.20240201124221-b850c711ff5c h1:r+1vFyTAoXptJrsPsnOMI3G0jm4+BCfXAcIyuA33lzo=
github.com/containers/buildah v1.34.1-0.20240201124221-b850c711ff5c/go.mod h1:Hw4qo2URFpWvZ2tjLstoQMpNC6+gR4PtxQefvV/UKaA=
github.com/containers/common v0.57.1-0.20240205132223-de5cb00e891c h1:Xzo9t4eIalkeilcmYTz0YEgL7hMrGQ12GK6UlSHrEsU=
github.com/containers/common v0.57.1-0.20240205132223-de5cb00e891c/go.mod h1:s1gEyucR3ryIex1aDMo1KzbfpvRl0CaGER6s5jqXRkI=
github.com/containers/common v0.57.1-0.20240206153655-323e410f34bf h1:n/MU6nLwLt+YcMKcb7ClwtgnCDzipWdbvN5zxHY9rmg=
github.com/containers/common v0.57.1-0.20240206153655-323e410f34bf/go.mod h1:s1gEyucR3ryIex1aDMo1KzbfpvRl0CaGER6s5jqXRkI=
github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
github.com/containers/gvisor-tap-vsock v0.7.2 h1:6CyU5D85C0/DciRRd7W0bPljK4FAS+DPrrHEQMHfZKY=
github.com/containers/gvisor-tap-vsock v0.7.2/go.mod h1:6NiTxh2GCVxZQLPzfuEB78/Osp2Usd9uf6nLdd6PiUY=
github.com/containers/image/v5 v5.29.2-0.20240130233108-e66a1ade2efc h1:3I5+mrrG7Fuv4aA13t1hAMQcjN3rTAQInfbxa5P+XH4=
github.com/containers/image/v5 v5.29.2-0.20240130233108-e66a1ade2efc/go.mod h1:oMMRA6avp1Na54lVPCj/OvcfXDMLlzfy3H7xeRiWmmI=
github.com/containers/libhvee v0.6.0 h1:tUzwSz8R0GjR6IctgDnkTMjdtCk5Mxhpai4Vyv6UeF4=
github.com/containers/libhvee v0.6.0/go.mod h1:f/q1wCdQqOLiK3IZqqBfOD7exMZYBU5pDYsrMa/pSFg=
github.com/containers/libhvee v0.6.1-0.20240205152934-3a16bce3e4be h1:M0lI66eh3tYtvfcxy78dMbhKuYVP8aE0oLDoS5nDPq0=
github.com/containers/libhvee v0.6.1-0.20240205152934-3a16bce3e4be/go.mod h1:IMG6nPEIBqC3FvxV//mCTRKo12gvY0NqSjRIKQoMaKY=
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
github.com/containers/luksy v0.0.0-20240129181507-b62d551ce6d8 h1:0p58QJRICjkRVCDix1nsnyrtJ3Qj4CWcGd1bOEY9sVY=

View File

@ -0,0 +1,29 @@
package internal
import (
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
// DeepCopyDescriptor copies a Descriptor, deeply copying its contents
func DeepCopyDescriptor(original *v1.Descriptor) *v1.Descriptor {
tmp := *original
if original.URLs != nil {
tmp.URLs = slices.Clone(original.URLs)
}
if original.Annotations != nil {
tmp.Annotations = maps.Clone(original.Annotations)
}
if original.Data != nil {
tmp.Data = slices.Clone(original.Data)
}
if original.Platform != nil {
tmpPlatform := *original.Platform
if original.Platform.OSFeatures != nil {
tmpPlatform.OSFeatures = slices.Clone(original.Platform.OSFeatures)
}
tmp.Platform = &tmpPlatform
}
return &tmp
}

View File

@ -4,24 +4,28 @@ import (
"github.com/containers/image/v5/manifest"
)
// ManifestListDescriptor references a platform-specific manifest.
// Contains exclusive field like `annotations` which is only present in
// OCI spec and not in docker image spec.
// ManifestListDescriptor describes a manifest that is mentioned in an
// image index or manifest list.
// Contains a subset of the fields which are present in both the OCI spec and
// the Docker spec, along with some which are unique to one or the other.
type ManifestListDescriptor struct {
manifest.Schema2Descriptor
Platform manifest.Schema2PlatformSpec `json:"platform"`
// Annotations contains arbitrary metadata for the image index.
Annotations map[string]string `json:"annotations,omitempty"`
Platform manifest.Schema2PlatformSpec `json:"platform,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
ArtifactType string `json:"artifactType,omitempty"`
Data []byte `json:"data,omitempty"`
Files []string `json:"files,omitempty"`
}
// ManifestListData is a list of platform-specific manifests, specifically used to
// generate output struct for `podman manifest inspect`. Reason for maintaining and
// having this type is to ensure we can have a common type which contains exclusive
// having this type is to ensure we can have a single type which contains exclusive
// fields from both Docker manifest format and OCI manifest format.
type ManifestListData struct {
SchemaVersion int `json:"schemaVersion"`
MediaType string `json:"mediaType"`
ArtifactType string `json:"artifactType,omitempty"`
Manifests []ManifestListDescriptor `json:"manifests"`
// Annotations contains arbitrary metadata for the image index.
Annotations map[string]string `json:"annotations,omitempty"`
Subject *ManifestListDescriptor `json:"subject,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
}

View File

@ -18,6 +18,8 @@ import (
"github.com/containers/storage"
structcopier "github.com/jinzhu/copier"
"github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/exp/slices"
)
// NOTE: the abstractions and APIs here are a first step to further merge
@ -221,17 +223,73 @@ func (i *Image) IsManifestList(ctx context.Context) (bool, error) {
// Inspect returns a dockerized version of the manifest list.
func (m *ManifestList) Inspect() (*define.ManifestListData, error) {
inspectList := define.ManifestListData{}
// Copy the fields from the Docker-format version of the list.
dockerFormat := m.list.Docker()
err := structcopier.Copy(&inspectList, &dockerFormat)
if err != nil {
return &inspectList, err
}
// Get missing annotation field from OCIv1 Spec
// and populate inspect data.
// Get OCI-specific fields from the OCIv1-format version of the list
// and copy them to the inspect data.
ociFormat := m.list.OCIv1()
inspectList.ArtifactType = ociFormat.ArtifactType
inspectList.Annotations = ociFormat.Annotations
for i, manifest := range ociFormat.Manifests {
inspectList.Manifests[i].Annotations = manifest.Annotations
inspectList.Manifests[i].ArtifactType = manifest.ArtifactType
if manifest.URLs != nil {
inspectList.Manifests[i].URLs = slices.Clone(manifest.URLs)
}
inspectList.Manifests[i].Data = manifest.Data
inspectList.Manifests[i].Files, err = m.list.Files(manifest.Digest)
if err != nil {
return &inspectList, err
}
}
if ociFormat.Subject != nil {
platform := ociFormat.Subject.Platform
if platform == nil {
platform = &imgspecv1.Platform{}
}
var osFeatures []string
if platform.OSFeatures != nil {
osFeatures = slices.Clone(platform.OSFeatures)
}
inspectList.Subject = &define.ManifestListDescriptor{
Platform: manifest.Schema2PlatformSpec{
OS: platform.OS,
Architecture: platform.Architecture,
OSVersion: platform.OSVersion,
Variant: platform.Variant,
OSFeatures: osFeatures,
},
Schema2Descriptor: manifest.Schema2Descriptor{
MediaType: ociFormat.Subject.MediaType,
Digest: ociFormat.Subject.Digest,
Size: ociFormat.Subject.Size,
URLs: ociFormat.Subject.URLs,
},
Annotations: ociFormat.Subject.Annotations,
ArtifactType: ociFormat.Subject.ArtifactType,
Data: ociFormat.Subject.Data,
}
}
// Set MediaType to mirror the value we'd use when saving the list
// using defaults, instead of forcing it to one or the other by
// using the value from one version or the other that we explicitly
// requested above.
serialized, err := m.list.Serialize("")
if err != nil {
return &inspectList, err
}
var typed struct {
MediaType string `json:"mediaType,omitempty"`
}
if err := json.Unmarshal(serialized, &typed); err != nil {
return &inspectList, err
}
if typed.MediaType != "" {
inspectList.MediaType = typed.MediaType
}
return &inspectList, nil
}

View File

@ -1,13 +1,20 @@
package manifests
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/containers/common/internal"
"github.com/containers/common/pkg/manifests"
"github.com/containers/common/pkg/retry"
"github.com/containers/common/pkg/supplemented"
@ -15,6 +22,7 @@ import (
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/manifest"
ocilayout "github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/pkg/compression"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/signature/signer"
@ -23,17 +31,25 @@ import (
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
"github.com/containers/storage/pkg/ioutils"
"github.com/containers/storage/pkg/lockfile"
digest "github.com/opencontainers/go-digest"
imgspec "github.com/opencontainers/image-spec/specs-go"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
const (
defaultMaxRetries = 3
)
const instancesData = "instances.json"
const (
instancesData = "instances.json"
artifactsData = "artifacts.json"
pushingArtifactsSubdirectory = "referenced-artifacts"
)
// LookupReferenceFunc return an image reference based on the specified one.
// The returned reference can return custom ImageSource or ImageDestination
@ -45,9 +61,19 @@ type LookupReferenceFunc func(ref types.ImageReference) (types.ImageReference, e
// for a List that has not yet been saved to an image.
var ErrListImageUnknown = errors.New("unable to determine which image holds the manifest list")
type artifactsDetails struct {
Manifests map[digest.Digest]string `json:"manifests,omitempty"` // artifact (and other?) manifest digests → manifest contents
Files map[digest.Digest][]string `json:"files,omitempty"` // artifact (and other?) manifest digests → file paths (mainly for display)
Configs map[digest.Digest]digest.Digest `json:"config,omitempty"` // artifact (and other?) manifest digests → referenced config digests
Layers map[digest.Digest][]digest.Digest `json:"layers,omitempty"` // artifact (and other?) manifest digests → referenced layer digests
Detached map[digest.Digest]string `json:"detached,omitempty"` // "config" and "layer" (and other?) digests in (usually artifact) manifests → file paths
Blobs map[digest.Digest][]byte `json:"blobs,omitempty"` // "config" and "layer" (and other?) manifest digests → inlined blob contents
}
type list struct {
manifests.List
instances map[digest.Digest]string
instances map[digest.Digest]string // instance manifest digests → image references
artifacts artifactsDetails
}
// List is a manifest list or image index, either created using Create(), or
@ -58,6 +84,9 @@ type List interface {
Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error)
Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error)
Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error)
AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error)
InstanceByFile(file string) (digest.Digest, error)
Files(instanceDigest digest.Digest) ([]string, error)
}
// PushOptions includes various settings which are needed for pushing the
@ -93,6 +122,14 @@ func Create() List {
return &list{
List: manifests.Create(),
instances: make(map[digest.Digest]string),
artifacts: artifactsDetails{
Manifests: make(map[digest.Digest]string),
Files: make(map[digest.Digest][]string),
Configs: make(map[digest.Digest]digest.Digest),
Layers: make(map[digest.Digest][]digest.Digest),
Detached: make(map[digest.Digest]string),
Blobs: make(map[digest.Digest][]byte),
},
}
}
@ -115,6 +152,14 @@ func LoadFromImage(store storage.Store, image string) (string, List, error) {
list := &list{
List: manifestList,
instances: make(map[digest.Digest]string),
artifacts: artifactsDetails{
Manifests: make(map[digest.Digest]string),
Files: make(map[digest.Digest][]string),
Configs: make(map[digest.Digest]digest.Digest),
Layers: make(map[digest.Digest][]digest.Digest),
Detached: make(map[digest.Digest]string),
Blobs: make(map[digest.Digest][]byte),
},
}
instancesBytes, err := store.ImageBigData(img.ID, instancesData)
if err != nil {
@ -123,8 +168,18 @@ func LoadFromImage(store storage.Store, image string) (string, List, error) {
if err := json.Unmarshal(instancesBytes, &list.instances); err != nil {
return "", nil, fmt.Errorf("decoding instance list for image %q: %w", image, err)
}
artifactsBytes, err := store.ImageBigData(img.ID, artifactsData)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return "", nil, fmt.Errorf("locating image %q for loading instance list: %w", image, err)
}
artifactsBytes = []byte("{}")
}
if err := json.Unmarshal(artifactsBytes, &list.artifacts); err != nil {
return "", nil, fmt.Errorf("decoding artifact list for image %q: %w", image, err)
}
list.instances[""] = img.ID
return img.ID, list, err
return img.ID, list, nil
}
// SaveToImage saves the manifest list or image index as the manifest of an
@ -140,41 +195,79 @@ func (l *list) SaveToImage(store storage.Store, imageID string, names []string,
if err != nil {
return "", err
}
img, err := store.CreateImage(imageID, names, "", "", &storage.ImageOptions{})
if err == nil || errors.Is(err, storage.ErrDuplicateID) {
created := (err == nil)
if created {
imageID = img.ID
l.instances[""] = img.ID
}
err := store.SetImageBigData(imageID, storage.ImageDigestManifestBigDataNamePrefix, manifestBytes, manifest.Digest)
if err != nil {
if created {
if _, err2 := store.DeleteImage(img.ID, true); err2 != nil {
logrus.Errorf("Deleting image %q after failing to save manifest for it", img.ID)
}
}
return "", fmt.Errorf("saving manifest list to image %q: %w", imageID, err)
}
err = store.SetImageBigData(imageID, instancesData, instancesBytes, nil)
if err != nil {
if created {
if _, err2 := store.DeleteImage(img.ID, true); err2 != nil {
logrus.Errorf("Deleting image %q after failing to save instance locations for it", img.ID)
}
}
return "", fmt.Errorf("saving instance list to image %q: %w", imageID, err)
}
return imageID, nil
artifactsBytes, err := json.Marshal(&l.artifacts)
if err != nil {
return "", err
}
return "", fmt.Errorf("creating image to hold manifest list: %w", err)
manifestDigest, err := manifest.Digest(manifestBytes)
if err != nil {
return "", err
}
imageOptions := &storage.ImageOptions{
BigData: []storage.ImageBigDataOption{
{Key: storage.ImageDigestManifestBigDataNamePrefix, Data: manifestBytes, Digest: manifestDigest},
{Key: instancesData, Data: instancesBytes},
{Key: artifactsData, Data: artifactsBytes},
},
}
img, err := store.CreateImage(imageID, names, "", "", imageOptions)
if err != nil {
if imageID != "" && errors.Is(err, storage.ErrDuplicateID) {
for _, bd := range imageOptions.BigData {
digester := manifest.Digest
if !strings.HasPrefix(bd.Key, storage.ImageDigestManifestBigDataNamePrefix) {
digester = nil
}
err := store.SetImageBigData(imageID, bd.Key, bd.Data, digester)
if err != nil {
return "", fmt.Errorf("saving manifest list to image %q: %w", imageID, err)
}
}
return imageID, nil
}
return "", err
}
l.instances[""] = img.ID
return img.ID, nil
}
// Files returns the list of files associated with a particular artifact
// instance in the image index, primarily for display purposes.
func (l *list) Files(instanceDigest digest.Digest) ([]string, error) {
filesList, ok := l.artifacts.Files[instanceDigest]
if ok {
return slices.Clone(filesList), nil
}
return nil, nil
}
// instanceByFile returns the instanceDigest of the first manifest in the index
// which refers to the named file. The name will be passed to filepath.Abs()
// before searching for an instance which references it.
func (l *list) InstanceByFile(file string) (digest.Digest, error) {
if parsedDigest, err := digest.Parse(file); err == nil {
// nice try, but that's already a digest!
return parsedDigest, nil
}
abs, err := filepath.Abs(file)
if err != nil {
return "", err
}
for instanceDigest, files := range l.artifacts.Files {
for _, file := range files {
if file == abs {
return instanceDigest, nil
}
}
}
return "", os.ErrNotExist
}
// Reference returns an image reference for the composite image being built
// in the list, or an error if the list has never been saved to a local image.
func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) {
if l.instances[""] == "" {
return nil, fmt.Errorf("building reference to list: %w", ErrListImageUnknown)
return nil, fmt.Errorf("building reference to list, appears to have not been saved first: %w", ErrListImageUnknown)
}
s, err := is.Transport.ParseStoreReference(store, l.instances[""])
if err != nil {
@ -198,6 +291,94 @@ func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, in
}
}
}
if len(l.artifacts.Manifests) > 0 {
img, err := is.Transport.GetImage(s)
if err != nil {
return nil, fmt.Errorf("locating image %s: %w", transports.ImageName(s), err)
}
imgDirectory, err := store.ImageDirectory(img.ID)
if err != nil {
return nil, fmt.Errorf("locating per-image directory for %s: %w", img.ID, err)
}
tmp, err := os.MkdirTemp(imgDirectory, pushingArtifactsSubdirectory)
if err != nil {
return nil, err
}
for artifactManifestDigest, contents := range l.artifacts.Manifests {
// create the blobs directory
blobsDir := filepath.Join(tmp, "blobs", artifactManifestDigest.Algorithm().String())
if err := os.MkdirAll(blobsDir, 0o700); err != nil {
return nil, fmt.Errorf("creating directory for blobs: %w", err)
}
// write the artifact manifest
if err := os.WriteFile(filepath.Join(blobsDir, artifactManifestDigest.Encoded()), []byte(contents), 0o644); err != nil {
return nil, fmt.Errorf("writing artifact manifest as blob: %w", err)
}
// symlink all of the referenced files and write the inlined blobs into the blobs directory
var referencedBlobDigests []digest.Digest
var symlinkedFiles []string
if referencedConfigDigest, ok := l.artifacts.Configs[artifactManifestDigest]; ok {
referencedBlobDigests = append(referencedBlobDigests, referencedConfigDigest)
}
referencedBlobDigests = append(referencedBlobDigests, l.artifacts.Layers[artifactManifestDigest]...)
for _, referencedBlobDigest := range referencedBlobDigests {
referencedFile, knownFile := l.artifacts.Detached[referencedBlobDigest]
referencedBlob, knownBlob := l.artifacts.Blobs[referencedBlobDigest]
if !knownFile && !knownBlob {
return nil, fmt.Errorf(`internal error: no file or blob with artifact "config" or "layer" digest %q recorded`, referencedBlobDigest)
}
expectedLayerBlobPath := filepath.Join(blobsDir, referencedBlobDigest.Encoded())
if _, err := os.Lstat(expectedLayerBlobPath); err == nil {
// did this one already
continue
} else if knownFile {
if err := os.Symlink(referencedFile, expectedLayerBlobPath); err != nil {
return nil, err
}
symlinkedFiles = append(symlinkedFiles, referencedFile)
} else if knownBlob {
if err := os.WriteFile(expectedLayerBlobPath, referencedBlob, 0o600); err != nil {
return nil, err
}
}
}
// write the index that refers to this one artifact image
tag := "latest"
indexFile := filepath.Join(tmp, "index.json")
index := v1.Index{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
MediaType: v1.MediaTypeImageIndex,
Manifests: []v1.Descriptor{{
MediaType: v1.MediaTypeImageManifest,
Digest: artifactManifestDigest,
Size: int64(len(contents)),
Annotations: map[string]string{
v1.AnnotationRefName: tag,
},
}},
}
indexBytes, err := json.Marshal(&index)
if err != nil {
return nil, fmt.Errorf("encoding image index for OCI layout: %w", err)
}
if err := os.WriteFile(indexFile, indexBytes, 0o644); err != nil {
return nil, fmt.Errorf("writing image index for OCI layout: %w", err)
}
// write the layout file
layoutFile := filepath.Join(tmp, "oci-layout")
if err := os.WriteFile(layoutFile, []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644); err != nil {
return nil, fmt.Errorf("writing oci-layout file: %w", err)
}
// build the reference to this artifact image's oci layout
ref, err := ocilayout.NewReference(tmp, tag)
if err != nil {
return nil, fmt.Errorf("creating ImageReference for artifact with files %q: %w", symlinkedFiles, err)
}
references = append(references, ref)
}
}
for _, instance := range whichInstances {
imageName := l.instances[instance]
ref, err := alltransports.ParseImageName(imageName)
@ -331,6 +512,8 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag
OS, Architecture, OSVersion, Variant string
Features, OSFeatures, Annotations []string
Size int64
ConfigInfo types.BlobInfo
ArtifactType string
}
var instanceInfos []instanceInfo
var manifestDigest digest.Digest
@ -361,6 +544,7 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag
Features: append([]string{}, lists.Docker().Manifests[i].Platform.Features...),
OSFeatures: append([]string{}, platform.OSFeatures...),
Size: instance.Size,
ArtifactType: instance.ArtifactType,
}
instanceInfos = append(instanceInfos, instanceInfo)
}
@ -391,6 +575,7 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag
Features: append([]string{}, lists.Docker().Manifests[i].Platform.Features...),
OSFeatures: append([]string{}, platform.OSFeatures...),
Size: instance.Size,
ArtifactType: instance.ArtifactType,
}
instanceInfos = append(instanceInfos, instanceInfo)
added = true
@ -406,11 +591,28 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag
instanceInfo := instanceInfo{
instanceDigest: nil,
}
if primaryManifestType == v1.MediaTypeImageManifest {
if m, err := manifest.OCI1FromManifest(primaryManifestBytes); err == nil {
instanceInfo.ArtifactType = m.ArtifactType
}
}
instanceInfos = append(instanceInfos, instanceInfo)
}
knownConfigTypes := []string{manifest.DockerV2Schema2ConfigMediaType, v1.MediaTypeImageConfig}
for _, instanceInfo := range instanceInfos {
if instanceInfo.OS == "" || instanceInfo.Architecture == "" {
manifestBytes, manifestType, err := src.GetManifest(ctx, instanceInfo.instanceDigest)
if err != nil {
return "", fmt.Errorf("reading manifest from %q, instance %q: %w", transports.ImageName(ref), instanceInfo.instanceDigest, err)
}
instanceManifest, err := manifest.FromBlob(manifestBytes, manifestType)
if err != nil {
return "", fmt.Errorf("parsing manifest from %q, instance %q: %w", transports.ImageName(ref), instanceInfo.instanceDigest, err)
}
instanceInfo.ConfigInfo = instanceManifest.ConfigInfo()
hasPlatformConfig := instanceInfo.ArtifactType == "" && slices.Contains(knownConfigTypes, instanceInfo.ConfigInfo.MediaType)
needToParsePlatformConfig := (instanceInfo.OS == "" || instanceInfo.Architecture == "")
if hasPlatformConfig && needToParsePlatformConfig {
img, err := image.FromUnparsedImage(ctx, sys, image.UnparsedInstance(src, instanceInfo.instanceDigest))
if err != nil {
return "", fmt.Errorf("reading configuration blob from %q: %w", transports.ImageName(ref), err)
@ -422,17 +624,15 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag
if instanceInfo.OS == "" {
instanceInfo.OS = config.OS
instanceInfo.OSVersion = config.OSVersion
instanceInfo.OSFeatures = config.OSFeatures
if config.OSFeatures != nil {
instanceInfo.OSFeatures = slices.Clone(config.OSFeatures)
}
}
if instanceInfo.Architecture == "" {
instanceInfo.Architecture = config.Architecture
instanceInfo.Variant = config.Variant
}
}
manifestBytes, manifestType, err := src.GetManifest(ctx, instanceInfo.instanceDigest)
if err != nil {
return "", fmt.Errorf("reading manifest from %q, instance %q: %w", transports.ImageName(ref), instanceInfo.instanceDigest, err)
}
if instanceInfo.instanceDigest == nil {
manifestDigest, err = manifest.Digest(manifestBytes)
if err != nil {
@ -455,6 +655,267 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag
return manifestDigest, nil
}
// AddArtifactOptions contains options which control the contents of the
// artifact manifest that AddArtifact will create and add to the image index.
// This should provide for all of the ways to construct a manifest outlined in
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage
// * no blobs → set ManifestArtifactType
// * blobs, no configuration → set ManifestArtifactType and possibly LayerMediaType, and provide file names
// * blobs and configuration → set ManifestArtifactType, possibly LayerMediaType, and ConfigDescriptor, and provide file names
//
// The older style of describing artifacts:
// * leave ManifestArtifactType blank
// * specify a zero-length application/vnd.oci.image.config.v1+json config blob
// * set LayerMediaType to a custom type
//
// When reading data produced elsewhere, note that newer tooling will produce
// manifests with ArtifactType set. If the manifest's ArtifactType is not set,
// consumers should consult the config descriptor's MediaType.
type AddArtifactOptions struct {
ManifestArtifactType *string // overall type of the artifact manifest. default: "application/vnd.unknown.artifact.v1"
Platform v1.Platform // default: add to the index without platform information
ConfigDescriptor *v1.Descriptor // default: a descriptor for an explicitly empty config blob
ConfigFile string // path to config contents, recorded if ConfigDescriptor.Size != 0 and ConfigDescriptor.Data is not set
LayerMediaType *string // default: mime.TypeByExtension() if basename contains ".", else http.DetectContentType()
Annotations map[string]string // optional, default is none
SubjectReference types.ImageReference // optional
ExcludeTitles bool // don't add "org.opencontainers.image.title" annotations set to file base names
}
// AddArtifact creates an artifact manifest describing the specified file or
// files, then adds them to the specified image index. Returns the
// instanceDigest for the artifact manifest.
// The caller could craft the manifest themselves and use Add() to add it to
// the image index and get the same end-result, but this should save them some
// work.
func (l *list) AddArtifact(ctx context.Context, sys *types.SystemContext, options AddArtifactOptions, files ...string) (digest.Digest, error) {
// If we were given a subject, build a descriptor for it first, since
// it might be remote, and anything else we do before looking at it
// might have to get thrown away if we can't get to it for whatever
// reason.
var subject *v1.Descriptor
if options.SubjectReference != nil {
subjectReference, err := options.SubjectReference.NewImageSource(ctx, sys)
if err != nil {
return "", fmt.Errorf("setting up to read manifest and configuration from subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
defer subjectReference.Close()
subjectManifestBytes, subjectManifestType, err := subjectReference.GetManifest(ctx, nil)
if err != nil {
return "", fmt.Errorf("reading manifest from subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
subjectManifestDigest, err := manifest.Digest(subjectManifestBytes)
if err != nil {
return "", fmt.Errorf("digesting manifest of subject %q: %w", transports.ImageName(options.SubjectReference), err)
}
subject = &v1.Descriptor{
MediaType: subjectManifestType,
Digest: subjectManifestDigest,
Size: int64(len(subjectManifestBytes)),
}
}
// Build up the layers list piece by piece.
var layers []v1.Descriptor
fileDigests := make(map[string]digest.Digest)
if len(files) == 0 {
// https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage
// says that we should have at least one layer listed, even if it's just a placeholder
layers = append(layers, v1.DescriptorEmptyJSON)
}
for _, file := range files {
if err := func() error {
// Open the file so that we can digest it.
absFile, err := filepath.Abs(file)
if err != nil {
return fmt.Errorf("converting %q to an absolute path: %w", file, err)
}
f, err := os.Open(absFile)
if err != nil {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
defer f.Close()
// Hang on to a copy of the first 512 bytes, but digest the whole thing.
digester := digest.Canonical.Digester()
writeCounter := ioutils.NewWriteCounter(digester.Hash())
var detectableData bytes.Buffer
_, err = io.CopyN(writeCounter, io.TeeReader(f, &detectableData), 512)
if err != nil && !errors.Is(err, io.EOF) {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
if err == nil {
if _, err := io.Copy(writeCounter, f); err != nil {
return fmt.Errorf("reading %q to determine its digest: %w", file, err)
}
}
fileDigests[absFile] = digester.Digest()
// If one wasn't specified, figure out what the MediaType should be.
title := filepath.Base(absFile)
layerMediaType := options.LayerMediaType
if layerMediaType == nil {
if index := strings.LastIndex(title, "."); index != -1 {
// File's basename has an extension, try to use a shortcut.
tmp := mime.TypeByExtension(title[index:])
if tmp != "" {
layerMediaType = &tmp
}
}
if layerMediaType == nil {
// File's basename has no extension or didn't map to a type, look at the contents we saved.
tmp := http.DetectContentType(detectableData.Bytes())
layerMediaType = &tmp
}
if layerMediaType != nil {
// Strip off any parameters, since we only want the type name.
if parsedMediaType, _, err := mime.ParseMediaType(*layerMediaType); err == nil {
layerMediaType = &parsedMediaType
}
}
}
// Build the descriptor for the layer.
descriptor := v1.Descriptor{
MediaType: *layerMediaType,
Digest: fileDigests[absFile],
Size: writeCounter.Count,
}
// OCI annotations are usually applied at the image manifest as a whole,
// but tools like oras (https://oras.land/) also apply them to blob
// descriptors. AnnotationTitle is used as a suggestion for the name
// to give to a blob if it's being stored as a file, and we default
// to adding one based on its original name.
if !options.ExcludeTitles {
descriptor.Annotations = map[string]string{
v1.AnnotationTitle: title,
}
}
layers = append(layers, descriptor)
return nil
}(); err != nil {
return "", err
}
}
// Unless we were told what this is, use the default that ORAS uses.
artifactType := "application/vnd.unknown.artifact.v1"
if options.ManifestArtifactType != nil {
artifactType = *options.ManifestArtifactType
}
// Unless we were explicitly told otherwise, default to an empty config blob.
configDescriptor := internal.DeepCopyDescriptor(&v1.DescriptorEmptyJSON)
if options.ConfigDescriptor != nil {
configDescriptor = internal.DeepCopyDescriptor(options.ConfigDescriptor)
} else if options.ConfigFile != "" {
configDescriptor = &v1.Descriptor{
MediaType: v1.MediaTypeImageConfig,
Digest: "", // to be figured out below
Size: -1, // to be figured out below
}
}
configFilePath := ""
if configDescriptor.Size != 0 {
if len(configDescriptor.Data) == 0 {
if options.ConfigFile == "" {
return "", fmt.Errorf("needed config data file, but none was provided")
}
filePath, err := filepath.Abs(options.ConfigFile)
if err != nil {
return "", fmt.Errorf("recording artifact config data file %q: %w", options.ConfigFile, err)
}
digester := digest.Canonical.Digester()
counter := ioutils.NewWriteCounter(digester.Hash())
if err := func() error {
f, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("reading artifact config data file %q: %w", options.ConfigFile, err)
}
defer f.Close()
if _, err := io.Copy(counter, f); err != nil {
return fmt.Errorf("digesting artifact config data file %q: %w", options.ConfigFile, err)
}
return nil
}(); err != nil {
return "", err
}
configDescriptor.Data = nil
configDescriptor.Size = counter.Count
configDescriptor.Digest = digester.Digest()
configFilePath = filePath
} else {
decoder := bytes.NewReader(configDescriptor.Data)
digester := digest.Canonical.Digester()
counter := ioutils.NewWriteCounter(digester.Hash())
if _, err := io.Copy(counter, decoder); err != nil {
return "", fmt.Errorf("digesting inlined artifact config data: %w", err)
}
configDescriptor.Size = counter.Count
configDescriptor.Digest = digester.Digest()
}
} else {
configDescriptor.Data = nil
configDescriptor.Digest = digest.Canonical.FromString("")
}
// Construct the manifest.
artifactManifest := v1.Manifest{
Versioned: imgspec.Versioned{
SchemaVersion: 2,
},
MediaType: v1.MediaTypeImageManifest,
ArtifactType: artifactType,
Config: *configDescriptor,
Layers: layers,
Subject: subject,
}
// Add in annotations, more or less exactly as specified.
if options.Annotations != nil {
artifactManifest.Annotations = maps.Clone(options.Annotations)
}
// Encode and save the data we care about.
artifactManifestBytes, err := json.Marshal(artifactManifest)
if err != nil {
return "", fmt.Errorf("marshalling the artifact manifest: %w", err)
}
artifactManifestDigest, err := manifest.Digest(artifactManifestBytes)
if err != nil {
return "", fmt.Errorf("digesting the artifact manifest: %w", err)
}
l.artifacts.Manifests[artifactManifestDigest] = string(artifactManifestBytes)
l.artifacts.Layers[artifactManifestDigest] = nil
if configFilePath != "" {
l.artifacts.Configs[artifactManifestDigest] = artifactManifest.Config.Digest
l.artifacts.Detached[artifactManifest.Config.Digest] = configFilePath
l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], configFilePath)
}
if len(artifactManifest.Config.Data) != 0 {
l.artifacts.Configs[artifactManifestDigest] = artifactManifest.Config.Digest
l.artifacts.Blobs[artifactManifest.Config.Digest] = slices.Clone(artifactManifest.Config.Data)
}
for filePath, fileDigest := range fileDigests {
l.artifacts.Layers[artifactManifestDigest] = append(l.artifacts.Layers[artifactManifestDigest], fileDigest)
l.artifacts.Detached[fileDigest] = filePath
l.artifacts.Files[artifactManifestDigest] = append(l.artifacts.Files[artifactManifestDigest], filePath)
}
// Add this artifact manifest to the image index.
if err := l.AddInstance(artifactManifestDigest, int64(len(artifactManifestBytes)), artifactManifest.MediaType, options.Platform.OS, options.Platform.Architecture, options.Platform.OSVersion, options.Platform.OSFeatures, options.Platform.Variant, nil, nil); err != nil {
return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err)
}
// Set the artifact type in the image index entry if we have one, since AddInstance() didn't do that for us.
if artifactManifest.ArtifactType != "" {
if err := l.List.SetArtifactType(&artifactManifestDigest, artifactManifest.ArtifactType); err != nil {
return "", fmt.Errorf("adding artifact manifest for %q to image index: %w", files, err)
}
}
return artifactManifestDigest, nil
}
// Remove filters out any instances in the list which match the specified digest.
func (l *list) Remove(instanceDigest digest.Digest) error {
err := l.List.Remove(instanceDigest)

View File

@ -6,10 +6,12 @@ import (
"fmt"
"os"
"github.com/containers/common/internal"
"github.com/containers/image/v5/manifest"
digest "github.com/opencontainers/go-digest"
imgspec "github.com/opencontainers/image-spec/specs-go"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"golang.org/x/exp/slices"
)
// List is a generic interface for manipulating a manifest list or an image
@ -36,8 +38,10 @@ type List interface {
OSFeatures(instanceDigest digest.Digest) ([]string, error)
SetMediaType(instanceDigest digest.Digest, mediaType string) error
MediaType(instanceDigest digest.Digest) (string, error)
SetArtifactType(instanceDigest digest.Digest, artifactType string) error
ArtifactType(instanceDigest digest.Digest) (string, error)
SetArtifactType(instanceDigest *digest.Digest, artifactType string) error
ArtifactType(instanceDigest *digest.Digest) (string, error)
SetSubject(subject *v1.Descriptor) error
Subject() (*v1.Descriptor, error)
Serialize(mimeType string) ([]byte, error)
Instances() []digest.Digest
OCIv1() *v1.Index
@ -469,22 +473,50 @@ func (l *list) MediaType(instanceDigest digest.Digest) (string, error) {
}
// SetArtifactType sets the ArtifactType field in the instance with the specified digest.
func (l *list) SetArtifactType(instanceDigest digest.Digest, artifactType string) error {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return err
func (l *list) SetArtifactType(instanceDigest *digest.Digest, artifactType string) error {
artifactTypePtr := &l.oci.ArtifactType
if instanceDigest != nil {
oci, err := l.findOCIv1(*instanceDigest)
if err != nil {
return err
}
artifactTypePtr = &oci.ArtifactType
}
oci.ArtifactType = artifactType
*artifactTypePtr = artifactType
return nil
}
// ArtifactType retrieves the ArtifactType field in the instance with the specified digest.
func (l *list) ArtifactType(instanceDigest digest.Digest) (string, error) {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return "", err
func (l *list) ArtifactType(instanceDigest *digest.Digest) (string, error) {
artifactTypePtr := &l.oci.ArtifactType
if instanceDigest != nil {
oci, err := l.findOCIv1(*instanceDigest)
if err != nil {
return "", err
}
artifactTypePtr = &oci.ArtifactType
}
return oci.ArtifactType, nil
return *artifactTypePtr, nil
}
// SetSubject sets the image index's subject.
// The field is specific to the OCI image index format, and is not present in Docker manifest lists.
func (l *list) SetSubject(subject *v1.Descriptor) error {
if subject != nil {
subject = internal.DeepCopyDescriptor(subject)
}
l.oci.Subject = subject
return nil
}
// Subject retrieves the subject which might have been set on the image index.
// The field is specific to the OCI image index format, and is not present in Docker manifest lists.
func (l *list) Subject() (*v1.Descriptor, error) {
s := l.oci.Subject
if s != nil {
s = internal.DeepCopyDescriptor(s)
}
return s, nil
}
// FromBlob builds a list from an encoded manifest list or image index.
@ -530,11 +562,19 @@ func FromBlob(manifestBytes []byte) (List, error) {
if platform == nil {
platform = &v1.Platform{}
}
if m.Platform != nil && m.Platform.OSFeatures != nil {
platform.OSFeatures = slices.Clone(m.Platform.OSFeatures)
}
var urls []string
if m.URLs != nil {
urls = slices.Clone(m.URLs)
}
list.docker.Manifests = append(list.docker.Manifests, manifest.Schema2ManifestDescriptor{
Schema2Descriptor: manifest.Schema2Descriptor{
MediaType: m.MediaType,
Size: m.Size,
Digest: m.Digest,
URLs: urls,
},
Platform: manifest.Schema2PlatformSpec{
Architecture: platform.Architecture,
@ -557,6 +597,9 @@ func (l *list) preferOCI() bool {
if l.oci.Subject != nil {
return true
}
if len(l.oci.Annotations) > 0 {
return true
}
for _, m := range l.oci.Manifests {
if m.ArtifactType != "" {
return true

View File

@ -0,0 +1,65 @@
package strongunits
// supported units
// B represents bytes
type B uint64
// KiB represents KiB
type KiB uint64
// MiB represents MiB
type MiB uint64
// GiB represents GiB
type GiB uint64
const (
// kibToB is the math convert from bytes to KiB
kibToB = 1 << 10
// mibToB is the math to convert from bytes to MiB
mibToB = 1 << 20
// gibToB s the math to convert from bytes to GiB
gibToB = 1 << 30
)
// StorageUnits is an interface for converting disk/memory storage
// units amongst each other.
type StorageUnits interface {
ToBytes() B
}
// ToBytes is a pass-through function for bytes
func (b B) ToBytes() B {
return b
}
// ToBytes converts KiB to bytes
func (k KiB) ToBytes() B {
return B(k * kibToB)
}
// ToBytes converts MiB to bytes
func (m MiB) ToBytes() B {
return B(m * mibToB)
}
// ToBytes converts GiB to bytes
func (g GiB) ToBytes() B {
return B(g * gibToB)
}
// ToKiB converts any StorageUnit type to KiB
func ToKiB(b StorageUnits) KiB {
return KiB(b.ToBytes() >> 10)
}
// ToMib converts any StorageUnit type to MiB
func ToMib(b StorageUnits) MiB {
return MiB(b.ToBytes() >> 20)
}
// ToGiB converts any StorageUnit type to GiB
func ToGiB(b StorageUnits) GiB {
return GiB(b.ToBytes() >> 30)
}

View File

@ -6,8 +6,8 @@ import (
"strconv"
"strings"
"github.com/containers/common/pkg/strongunits"
"github.com/containers/libhvee/pkg/wmiext"
"github.com/containers/podman/v4/pkg/strongunits"
)
// ResizeDisk takes a diskPath and strongly typed new size and uses powershell

6
vendor/modules.txt vendored
View File

@ -168,8 +168,9 @@ github.com/containers/buildah/pkg/sshagent
github.com/containers/buildah/pkg/util
github.com/containers/buildah/pkg/volumes
github.com/containers/buildah/util
# github.com/containers/common v0.57.1-0.20240205132223-de5cb00e891c
# github.com/containers/common v0.57.1-0.20240206153655-323e410f34bf
## explicit; go 1.20
github.com/containers/common/internal
github.com/containers/common/internal/attributedstring
github.com/containers/common/libimage
github.com/containers/common/libimage/define
@ -222,6 +223,7 @@ github.com/containers/common/pkg/secrets/shelldriver
github.com/containers/common/pkg/servicereaper
github.com/containers/common/pkg/signal
github.com/containers/common/pkg/ssh
github.com/containers/common/pkg/strongunits
github.com/containers/common/pkg/subscriptions
github.com/containers/common/pkg/supplemented
github.com/containers/common/pkg/sysinfo
@ -309,7 +311,7 @@ github.com/containers/image/v5/transports
github.com/containers/image/v5/transports/alltransports
github.com/containers/image/v5/types
github.com/containers/image/v5/version
# github.com/containers/libhvee v0.6.0
# github.com/containers/libhvee v0.6.1-0.20240205152934-3a16bce3e4be
## explicit; go 1.18
github.com/containers/libhvee/pkg/hypervctl
github.com/containers/libhvee/pkg/kvp/ginsu