//go:build !remote package libimage import ( "context" "errors" "fmt" "io" "net" "os" "strings" "time" "github.com/containers/common/libimage/platform" "github.com/containers/common/pkg/config" "github.com/containers/common/pkg/retry" "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/pkg/compression" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/signature/signer" storageTransport "github.com/containers/image/v5/storage" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" encconfig "github.com/containers/ocicrypt/config" "github.com/containers/storage" "github.com/sirupsen/logrus" ) const ( defaultMaxRetries = 3 defaultRetryDelay = time.Second ) // CopyOptions allow for customizing image-copy operations. type CopyOptions struct { // If set, will be used for copying the image. Fields below may // override certain settings. SystemContext *types.SystemContext // Allows for customizing the source reference lookup. This can be // used to use custom blob caches. SourceLookupReferenceFunc LookupReferenceFunc // Allows for customizing the destination reference lookup. This can // be used to use custom blob caches. DestinationLookupReferenceFunc LookupReferenceFunc // CompressionFormat is the format to use for the compression of the blobs CompressionFormat *compression.Algorithm // CompressionLevel specifies what compression level is used CompressionLevel *int // ForceCompressionFormat ensures that the compression algorithm set in // CompressionFormat is used exclusively, and blobs of other compression // algorithms are not reused. ForceCompressionFormat bool // containers-auth.json(5) file to use when authenticating against // container registries. AuthFilePath string // Custom path to a blob-info cache. BlobInfoCacheDirPath string // Path to the certificates directory. CertDirPath string // Force layer compression when copying to a `dir` transport destination. DirForceCompress bool // ImageListSelection is one of CopySystemImage, CopyAllImages, or // CopySpecificImages, to control whether, when the source reference is a list, // copy.Image() copies only an image which matches the current runtime // environment, or all images which match the supplied reference, or only // specific images from the source reference. ImageListSelection copy.ImageListSelection // Allow contacting registries over HTTP, or HTTPS with failed TLS // verification. Note that this does not affect other TLS connections. InsecureSkipTLSVerify types.OptionalBool // Maximum number of retries with exponential backoff when facing // transient network errors. A reasonable default is used if not set. // Default 3. MaxRetries *uint // RetryDelay used for the exponential back off of MaxRetries. // Default 1 time.Second. RetryDelay *time.Duration // ManifestMIMEType is the desired media type the image will be // converted to if needed. Note that it must contain the exact MIME // types. Short forms (e.g., oci, v2s2) used by some tools are not // supported. ManifestMIMEType string // Accept uncompressed layers when copying OCI images. OciAcceptUncompressedLayers bool // If OciEncryptConfig is non-nil, it indicates that an image should be // encrypted. The encryption options is derived from the construction // of EncryptConfig object. Note: During initial encryption process of // a layer, the resultant digest is not known during creation, so // newDigestingReader has to be set with validateDigest = false OciEncryptConfig *encconfig.EncryptConfig // OciEncryptLayers represents the list of layers to encrypt. If nil, // don't encrypt any layers. If non-nil and len==0, denotes encrypt // all layers. integers in the slice represent 0-indexed layer // indices, with support for negative indexing. i.e. 0 is the first // layer, -1 is the last (top-most) layer. OciEncryptLayers *[]int // OciDecryptConfig contains the config that can be used to decrypt an // image if it is encrypted if non-nil. If nil, it does not attempt to // decrypt an image. OciDecryptConfig *encconfig.DecryptConfig // Reported to when ProgressInterval has arrived for a single // artifact+offset. Progress chan types.ProgressProperties // If set, allow using the storage transport even if it's disabled by // the specified SignaturePolicyPath. PolicyAllowStorage bool // SignaturePolicyPath to overwrite the default one. SignaturePolicyPath string // If non-empty, asks for signatures to be added during the copy // using the provided signers. Signers []*signer.Signer // If non-empty, asks for a signature to be added during the copy, and // specifies a key ID. SignBy string // If non-empty, passphrase to use when signing with the key ID from SignBy. SignPassphrase string // If non-empty, asks for a signature to be added during the copy, using // a sigstore private key file at the provided path. SignBySigstorePrivateKeyFile string // Passphrase to use when signing with SignBySigstorePrivateKeyFile. SignSigstorePrivateKeyPassphrase []byte // Remove any pre-existing signatures. SignBy will still add a new // signature. RemoveSignatures bool // Writer is used to display copy information including progress bars. Writer io.Writer // ----- platform ----------------------------------------------------- // Architecture to use for choosing images. Architecture string // OS to use for choosing images. OS string // Variant to use when choosing images. Variant string // ----- credentials -------------------------------------------------- // Username to use when authenticating at a container registry. Username string // Password to use when authenticating at a container registry. Password string // Credentials is an alternative way to specify credentials in format // "username[:password]". Cannot be used in combination with // Username/Password. Credentials string // IdentityToken is used to authenticate the user and get // an access token for the registry. IdentityToken string `json:"identitytoken,omitempty"` // ----- internal ----------------------------------------------------- // Additional tags when creating or copying a docker-archive. dockerArchiveAdditionalTags []reference.NamedTagged // If set it points to a NOTIFY_SOCKET the copier will use to extend // the systemd timeout while copying. extendTimeoutSocket string } // Copier is a helper to conveniently copy images. type Copier struct { extendTimeoutSocket string imageCopyOptions copy.Options retryOptions retry.Options systemContext *types.SystemContext policyContext *signature.PolicyContext sourceLookup LookupReferenceFunc destinationLookup LookupReferenceFunc } // newCopier creates a Copier based on a runtime's system context. // Note that fields in options *may* overwrite the counterparts of // the specified system context. Please make sure to call `(*Copier).Close()`. func (r *Runtime) newCopier(options *CopyOptions) (*Copier, error) { return NewCopier(options, r.SystemContext()) } // storageAllowedPolicyScopes overrides the policy for local storage // to ensure that we can read images from it. var storageAllowedPolicyScopes = signature.PolicyTransportScopes{ "": []signature.PolicyRequirement{ signature.NewPRInsecureAcceptAnything(), }, } // getDockerAuthConfig extracts a docker auth config. Returns nil if // no credentials are set. func getDockerAuthConfig(name, passwd, creds, idToken string) (*types.DockerAuthConfig, error) { numCredsSources := 0 if name != "" { numCredsSources++ } if creds != "" { name, passwd, _ = strings.Cut(creds, ":") numCredsSources++ } if idToken != "" { numCredsSources++ } authConf := &types.DockerAuthConfig{ Username: name, Password: passwd, IdentityToken: idToken, } switch numCredsSources { case 0: // Return nil if there is no credential source. return nil, nil case 1: return authConf, nil default: // Cannot use the multiple credential sources. return nil, errors.New("cannot use the multiple credential sources") } } // NewCopier creates a Copier based on a provided system context. // Note that fields in options *may* overwrite the counterparts of // the specified system context. Please make sure to call `(*Copier).Close()`. func NewCopier(options *CopyOptions, sc *types.SystemContext) (*Copier, error) { c := Copier{extendTimeoutSocket: options.extendTimeoutSocket} sysContextCopy := *sc c.systemContext = &sysContextCopy if options.SourceLookupReferenceFunc != nil { c.sourceLookup = options.SourceLookupReferenceFunc } if options.DestinationLookupReferenceFunc != nil { c.destinationLookup = options.DestinationLookupReferenceFunc } if options.InsecureSkipTLSVerify != types.OptionalBoolUndefined { c.systemContext.DockerInsecureSkipTLSVerify = options.InsecureSkipTLSVerify c.systemContext.OCIInsecureSkipTLSVerify = options.InsecureSkipTLSVerify == types.OptionalBoolTrue c.systemContext.DockerDaemonInsecureSkipTLSVerify = options.InsecureSkipTLSVerify == types.OptionalBoolTrue } c.systemContext.DirForceCompress = c.systemContext.DirForceCompress || options.DirForceCompress if options.AuthFilePath != "" { c.systemContext.AuthFilePath = options.AuthFilePath } c.systemContext.DockerArchiveAdditionalTags = options.dockerArchiveAdditionalTags c.systemContext.OSChoice, c.systemContext.ArchitectureChoice, c.systemContext.VariantChoice = platform.Normalize(options.OS, options.Architecture, options.Variant) if options.SignaturePolicyPath != "" { c.systemContext.SignaturePolicyPath = options.SignaturePolicyPath } dockerAuthConfig, err := getDockerAuthConfig(options.Username, options.Password, options.Credentials, options.IdentityToken) if err != nil { return nil, err } if dockerAuthConfig != nil { c.systemContext.DockerAuthConfig = dockerAuthConfig } if options.BlobInfoCacheDirPath != "" { c.systemContext.BlobInfoCacheDir = options.BlobInfoCacheDirPath } if options.CertDirPath != "" { c.systemContext.DockerCertPath = options.CertDirPath } if options.CompressionFormat != nil { c.systemContext.CompressionFormat = options.CompressionFormat } if options.CompressionLevel != nil { c.systemContext.CompressionLevel = options.CompressionLevel } // NOTE: for the sake of consistency it's called Oci* in the CopyOptions. c.systemContext.OCIAcceptUncompressedLayers = options.OciAcceptUncompressedLayers policy, err := signature.DefaultPolicy(c.systemContext) if err != nil { return nil, err } // Buildah compatibility: even if the policy denies _all_ transports, // Buildah still wants the storage to be accessible. if options.PolicyAllowStorage { policy.Transports[storageTransport.Transport.Name()] = storageAllowedPolicyScopes } policyContext, err := signature.NewPolicyContext(policy) if err != nil { return nil, err } c.policyContext = policyContext c.retryOptions.MaxRetry = defaultMaxRetries if options.MaxRetries != nil { c.retryOptions.MaxRetry = int(*options.MaxRetries) } c.retryOptions.Delay = defaultRetryDelay if options.RetryDelay != nil { c.retryOptions.Delay = *options.RetryDelay } c.imageCopyOptions.Progress = options.Progress if c.imageCopyOptions.Progress != nil { c.imageCopyOptions.ProgressInterval = time.Second } c.imageCopyOptions.ImageListSelection = options.ImageListSelection c.imageCopyOptions.ForceCompressionFormat = options.ForceCompressionFormat c.imageCopyOptions.ForceManifestMIMEType = options.ManifestMIMEType c.imageCopyOptions.SourceCtx = c.systemContext c.imageCopyOptions.DestinationCtx = c.systemContext c.imageCopyOptions.OciEncryptConfig = options.OciEncryptConfig c.imageCopyOptions.OciEncryptLayers = options.OciEncryptLayers c.imageCopyOptions.OciDecryptConfig = options.OciDecryptConfig c.imageCopyOptions.RemoveSignatures = options.RemoveSignatures c.imageCopyOptions.Signers = options.Signers c.imageCopyOptions.SignBy = options.SignBy c.imageCopyOptions.SignPassphrase = options.SignPassphrase c.imageCopyOptions.SignBySigstorePrivateKeyFile = options.SignBySigstorePrivateKeyFile c.imageCopyOptions.SignSigstorePrivateKeyPassphrase = options.SignSigstorePrivateKeyPassphrase c.imageCopyOptions.ReportWriter = options.Writer defaultContainerConfig, err := config.Default() if err != nil { logrus.Warnf("Failed to get container config for copy options: %v", err) } else { c.imageCopyOptions.MaxParallelDownloads = defaultContainerConfig.Engine.ImageParallelCopies } return &c, nil } // Close open resources. func (c *Copier) Close() error { return c.policyContext.Destroy() } // Copy the source to the destination. Returns the bytes of the copied // manifest which may be used for digest computation. func (c *Copier) Copy(ctx context.Context, source, destination types.ImageReference) ([]byte, error) { return c.copyInternal(ctx, source, destination, nil) } // Copy the source to the destination. Returns the bytes of the copied // manifest which may be used for digest computation. func (c *Copier) copyInternal(ctx context.Context, source, destination types.ImageReference, reportResolvedReference *types.ImageReference) ([]byte, error) { logrus.Debugf("Copying source image %s to destination image %s", source.StringWithinTransport(), destination.StringWithinTransport()) // Avoid running out of time when running inside a systemd unit by // regularly increasing the timeout. if c.extendTimeoutSocket != "" { socketAddr := &net.UnixAddr{ Name: c.extendTimeoutSocket, Net: "unixgram", } conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr) if err != nil { return nil, err } defer conn.Close() numExtensions := 10 extension := 30 * time.Second timerFrequency := 25 * time.Second // Fire the timer at a higher frequency to avoid a race timer := time.NewTicker(timerFrequency) socketCtx, cancel := context.WithCancel(ctx) defer cancel() defer timer.Stop() if c.imageCopyOptions.ReportWriter != nil { fmt.Fprintf(c.imageCopyOptions.ReportWriter, "Pulling image %s inside systemd: setting pull timeout to %s\n", source.StringWithinTransport(), time.Duration(numExtensions)*extension, ) } // From `man systemd.service(5)`: // // "If a service of Type=notify/Type=notify-reload sends "EXTEND_TIMEOUT_USEC=...", this may cause // the start time to be extended beyond TimeoutStartSec=. The first receipt of this message must // occur before TimeoutStartSec= is exceeded, and once the start time has extended beyond // TimeoutStartSec=, the service manager will allow the service to continue to start, provided the // service repeats "EXTEND_TIMEOUT_USEC=..." within the interval specified until the service startup // status is finished by "READY=1"." extendValue := []byte(fmt.Sprintf("EXTEND_TIMEOUT_USEC=%d", extension.Microseconds())) extendTimeout := func() { if _, err := conn.Write(extendValue); err != nil { logrus.Errorf("Increasing EXTEND_TIMEOUT_USEC failed: %v", err) } numExtensions-- } extendTimeout() go func() { for { select { case <-socketCtx.Done(): return case <-timer.C: if numExtensions == 0 { return } extendTimeout() } } }() } var err error if c.sourceLookup != nil { source, err = c.sourceLookup(source) if err != nil { return nil, err } } if c.destinationLookup != nil { destination, err = c.destinationLookup(destination) if err != nil { return nil, err } } // Buildah compat: used when running in OpenShift. sourceInsecure, err := checkRegistrySourcesAllows(source) if err != nil { return nil, err } destinationInsecure, err := checkRegistrySourcesAllows(destination) if err != nil { return nil, err } // Sanity checks for Buildah. if sourceInsecure != nil && *sourceInsecure { if c.systemContext.DockerInsecureSkipTLSVerify == types.OptionalBoolFalse { return nil, errors.New("can't require tls verification on an insecured registry") } } if destinationInsecure != nil && *destinationInsecure { if c.systemContext.DockerInsecureSkipTLSVerify == types.OptionalBoolFalse { return nil, errors.New("can't require tls verification on an insecured registry") } } var returnManifest []byte f := func() error { opts := c.imageCopyOptions // This is already set when `newCopier` was called but there is an option // to override it by callers if needed. if reportResolvedReference != nil { opts.ReportResolvedReference = reportResolvedReference } if sourceInsecure != nil { value := types.NewOptionalBool(*sourceInsecure) opts.SourceCtx.DockerInsecureSkipTLSVerify = value } if destinationInsecure != nil { value := types.NewOptionalBool(*destinationInsecure) opts.DestinationCtx.DockerInsecureSkipTLSVerify = value } copiedManifest, err := copy.Image(ctx, c.policyContext, destination, source, &opts) if err == nil { returnManifest = copiedManifest } return err } return returnManifest, retry.IfNecessary(ctx, f, &c.retryOptions) } func (c *Copier) copyToStorage(ctx context.Context, source, destination types.ImageReference) (*storage.Image, error) { var resolvedReference types.ImageReference _, err := c.copyInternal(ctx, source, destination, &resolvedReference) if err != nil { return nil, fmt.Errorf("internal error: unable to copy from source %s: %w", transports.ImageName(source), err) } if resolvedReference == nil { return nil, fmt.Errorf("internal error: After attempting to copy %s, resolvedReference is nil", source) } _, image, err := storageTransport.ResolveReference(resolvedReference) if err != nil { return nil, fmt.Errorf("resolving an already-resolved reference %q to the pulled image: %w", transports.ImageName(resolvedReference), err) } return image, nil } // checkRegistrySourcesAllows checks the $BUILD_REGISTRY_SOURCES environment // variable, if it's set. The contents are expected to be a JSON-encoded // github.com/openshift/api/config/v1.Image, set by an OpenShift build // controller that arranged for us to be run in a container. // // If set, the insecure return value indicates whether the registry is set to // be insecure. // // NOTE: this functionality is required by Buildah for OpenShift. func checkRegistrySourcesAllows(dest types.ImageReference) (insecure *bool, err error) { registrySources, ok := os.LookupEnv("BUILD_REGISTRY_SOURCES") if !ok || registrySources == "" { return nil, nil } logrus.Debugf("BUILD_REGISTRY_SOURCES set %q", registrySources) dref := dest.DockerReference() if dref == nil || reference.Domain(dref) == "" { return nil, nil } // Use local struct instead of github.com/openshift/api/config/v1 RegistrySources var sources struct { InsecureRegistries []string `json:"insecureRegistries,omitempty"` BlockedRegistries []string `json:"blockedRegistries,omitempty"` AllowedRegistries []string `json:"allowedRegistries,omitempty"` } if err := json.Unmarshal([]byte(registrySources), &sources); err != nil { return nil, fmt.Errorf("parsing $BUILD_REGISTRY_SOURCES (%q) as JSON: %w", registrySources, err) } blocked := false if len(sources.BlockedRegistries) > 0 { for _, blockedDomain := range sources.BlockedRegistries { if blockedDomain == reference.Domain(dref) { blocked = true } } } if blocked { return nil, fmt.Errorf("registry %q denied by policy: it is in the blocked registries list (%s)", reference.Domain(dref), registrySources) } allowed := true if len(sources.AllowedRegistries) > 0 { allowed = false for _, allowedDomain := range sources.AllowedRegistries { if allowedDomain == reference.Domain(dref) { allowed = true } } } if !allowed { return nil, fmt.Errorf("registry %q denied by policy: not in allowed registries list (%s)", reference.Domain(dref), registrySources) } for _, insecureDomain := range sources.InsecureRegistries { if insecureDomain == reference.Domain(dref) { insecure := true return &insecure, nil } } return nil, nil }