Merge pull request #5449 from baude/manifests

apiv2 addition of manifests
This commit is contained in:
OpenShift Merge Robot
2020-03-17 16:00:05 +01:00
committed by GitHub
17 changed files with 2073 additions and 1 deletions

15
vendor/github.com/containers/buildah/manifests/copy.go generated vendored Normal file
View File

@ -0,0 +1,15 @@
package manifests
import (
"github.com/containers/image/v5/signature"
)
var (
// storageAllowedPolicyScopes overrides the policy for local storage
// to ensure that we can read images from it.
storageAllowedPolicyScopes = signature.PolicyTransportScopes{
"": []signature.PolicyRequirement{
signature.NewPRInsecureAcceptAnything(),
},
}
)

View File

@ -0,0 +1,397 @@
package manifests
import (
"context"
"encoding/json"
stderrors "errors"
"io"
"github.com/containers/buildah/pkg/manifests"
"github.com/containers/buildah/pkg/supplemented"
cp "github.com/containers/image/v5/copy"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/signature"
is "github.com/containers/image/v5/storage"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
digest "github.com/opencontainers/go-digest"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const instancesData = "instances.json"
// ErrListImageUnknown is returned when we attempt to create an image reference
// for a List that has not yet been saved to an image.
var ErrListImageUnknown = stderrors.New("unable to determine which image holds the manifest list")
type list struct {
manifests.List
instances map[digest.Digest]string
}
// List is a manifest list or image index, either created using Create(), or
// loaded from local storage using LoadFromImage().
type List interface {
manifests.List
SaveToImage(store storage.Store, imageID string, names []string, mimeType string) (string, error)
Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error)
Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error)
Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error)
}
// PushOptions includes various settings which are needed for pushing the
// manifest list and its instances.
type PushOptions struct {
Store storage.Store
SystemContext *types.SystemContext // github.com/containers/image/types.SystemContext
ImageListSelection cp.ImageListSelection // set to either CopySystemImage, CopyAllImages, or CopySpecificImages
Instances []digest.Digest // instances to copy if ImageListSelection == CopySpecificImages
ReportWriter io.Writer // will be used to log the writing of the list and any blobs
SignBy string // fingerprint of GPG key to use to sign images
RemoveSignatures bool // true to discard signatures in images
ManifestType string // the format to use when saving the list - possible options are oci, v2s1, and v2s2
}
// Create creates a new list containing information about the specified image,
// computing its manifest's digest, and retrieving OS and architecture
// information from its configuration blob. Returns the new list, and the
// instanceDigest for the initial image.
func Create() List {
return &list{
List: manifests.Create(),
instances: make(map[digest.Digest]string),
}
}
// LoadFromImage reads the manifest list or image index, and additional
// information about where the various instances that it contains live, from an
// image record with the specified ID in local storage.
func LoadFromImage(store storage.Store, image string) (string, List, error) {
img, err := store.Image(image)
if err != nil {
return "", nil, errors.Wrapf(err, "error locating image %q for loading manifest list", image)
}
manifestBytes, err := store.ImageBigData(img.ID, storage.ImageDigestManifestBigDataNamePrefix)
if err != nil {
return "", nil, errors.Wrapf(err, "error locating image %q for loading manifest list", image)
}
manifestList, err := manifests.FromBlob(manifestBytes)
if err != nil {
return "", nil, err
}
list := &list{
List: manifestList,
instances: make(map[digest.Digest]string),
}
instancesBytes, err := store.ImageBigData(img.ID, instancesData)
if err != nil {
return "", nil, errors.Wrapf(err, "error locating image %q for loading instance list", image)
}
if err := json.Unmarshal(instancesBytes, &list.instances); err != nil {
return "", nil, errors.Wrapf(err, "error decoding instance list for image %q", image)
}
list.instances[""] = img.ID
return img.ID, list, err
}
// SaveToImage saves the manifest list or image index as the manifest of an
// Image record with the specified names in local storage, generating a random
// image ID if none is specified. It also stores information about where the
// images whose manifests are included in the list can be found.
func (l *list) SaveToImage(store storage.Store, imageID string, names []string, mimeType string) (string, error) {
manifestBytes, err := l.List.Serialize(mimeType)
if err != nil {
return "", err
}
instancesBytes, err := json.Marshal(&l.instances)
if err != nil {
return "", err
}
img, err := store.CreateImage(imageID, names, "", "", &storage.ImageOptions{})
if err == nil || errors.Cause(err) == storage.ErrDuplicateID {
created := (err == nil)
if created {
imageID = img.ID
l.instances[""] = img.ID
}
err := store.SetImageBigData(imageID, storage.ImageDigestManifestBigDataNamePrefix, manifestBytes, manifest.Digest)
if err != nil {
if created {
if _, err2 := store.DeleteImage(img.ID, true); err2 != nil {
logrus.Errorf("error deleting image %q after failing to save manifest for it", img.ID)
}
}
return "", errors.Wrapf(err, "error saving manifest list to image %q", imageID)
}
err = store.SetImageBigData(imageID, instancesData, instancesBytes, nil)
if err != nil {
if created {
if _, err2 := store.DeleteImage(img.ID, true); err2 != nil {
logrus.Errorf("error deleting image %q after failing to save instance locations for it", img.ID)
}
}
return "", errors.Wrapf(err, "error saving instance list to image %q", imageID)
}
return imageID, nil
}
return "", errors.Wrapf(err, "error creating image to hold manifest list")
}
// Reference returns an image reference for the composite image being built
// in the list, or an error if the list has never been saved to a local image.
func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) {
if l.instances[""] == "" {
return nil, errors.Wrap(ErrListImageUnknown, "error building reference to list")
}
s, err := is.Transport.ParseStoreReference(store, l.instances[""])
if err != nil {
return nil, errors.Wrapf(err, "error creating ImageReference from image %q", l.instances[""])
}
references := make([]types.ImageReference, 0, len(l.instances))
whichInstances := make([]digest.Digest, 0, len(l.instances))
switch multiple {
case cp.CopyAllImages, cp.CopySystemImage:
for instance := range l.instances {
if instance != "" {
whichInstances = append(whichInstances, instance)
}
}
case cp.CopySpecificImages:
for instance := range l.instances {
for _, allowed := range instances {
if instance == allowed {
whichInstances = append(whichInstances, instance)
}
}
}
}
for _, instance := range whichInstances {
imageName := l.instances[instance]
ref, err := alltransports.ParseImageName(imageName)
if err != nil {
return nil, errors.Wrapf(err, "error creating ImageReference from image %q", imageName)
}
references = append(references, ref)
}
return supplemented.Reference(s, references, multiple, instances), nil
}
// Push saves the manifest list and whichever blobs are needed to a destination location.
func (l *list) Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error) {
// Load the system signing policy.
pushPolicy, err := signature.DefaultPolicy(options.SystemContext)
if err != nil {
return nil, "", errors.Wrapf(err, "error obtaining default signature policy")
}
// Override the settings for local storage to make sure that we can always read the source "image".
pushPolicy.Transports[is.Transport.Name()] = storageAllowedPolicyScopes
policyContext, err := signature.NewPolicyContext(pushPolicy)
if err != nil {
return nil, "", errors.Wrapf(err, "error creating new signature policy context")
}
defer func() {
if err2 := policyContext.Destroy(); err2 != nil {
logrus.Errorf("error destroying signature policy context: %v", err2)
}
}()
// If we were given a media type that corresponds to a multiple-images
// type, reset it to a valid corresponding single-image type, since we
// already expect the image library to infer the list type from the
// image type that we're telling it to force.
singleImageManifestType := options.ManifestType
switch singleImageManifestType {
case v1.MediaTypeImageIndex:
singleImageManifestType = v1.MediaTypeImageManifest
case manifest.DockerV2ListMediaType:
singleImageManifestType = manifest.DockerV2Schema2MediaType
}
// Build a source reference for our list and grab bag full of blobs.
src, err := l.Reference(options.Store, options.ImageListSelection, options.Instances)
if err != nil {
return nil, "", err
}
copyOptions := &cp.Options{
ImageListSelection: options.ImageListSelection,
Instances: options.Instances,
SourceCtx: options.SystemContext,
DestinationCtx: options.SystemContext,
ReportWriter: options.ReportWriter,
RemoveSignatures: options.RemoveSignatures,
SignBy: options.SignBy,
ForceManifestMIMEType: singleImageManifestType,
}
// Copy whatever we were asked to copy.
manifestBytes, err := cp.Image(ctx, policyContext, dest, src, copyOptions)
if err != nil {
return nil, "", err
}
manifestDigest, err := manifest.Digest(manifestBytes)
if err != nil {
return nil, "", err
}
return nil, manifestDigest, nil
}
// Add adds information about the specified image to the list, computing the
// image's manifest's digest, retrieving OS and architecture information from
// the image's configuration, and recording the image's reference so that it
// can be found at push-time. Returns the instanceDigest for the image. If
// the reference points to an image list, either all instances are added (if
// "all" is true), or the instance which matches "sys" (if "all" is false) will
// be added.
func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error) {
src, err := ref.NewImageSource(ctx, sys)
if err != nil {
return "", errors.Wrapf(err, "error setting up to read manifest and configuration from %q", transports.ImageName(ref))
}
defer src.Close()
type instanceInfo struct {
instanceDigest *digest.Digest
OS, Architecture, OSVersion, Variant string
Features, OSFeatures, Annotations []string
Size int64
}
var instanceInfos []instanceInfo
var manifestDigest digest.Digest
primaryManifestBytes, primaryManifestType, err := src.GetManifest(ctx, nil)
if err != nil {
return "", errors.Wrapf(err, "error reading manifest from %q", transports.ImageName(ref))
}
if manifest.MIMETypeIsMultiImage(primaryManifestType) {
lists, err := manifests.FromBlob(primaryManifestBytes)
if err != nil {
return "", errors.Wrapf(err, "error parsing manifest list in %q", transports.ImageName(ref))
}
if all {
for i, instance := range lists.OCIv1().Manifests {
platform := instance.Platform
if platform == nil {
platform = &v1.Platform{}
}
instanceDigest := instance.Digest
instanceInfo := instanceInfo{
instanceDigest: &instanceDigest,
OS: platform.OS,
Architecture: platform.Architecture,
OSVersion: platform.OSVersion,
Variant: platform.Variant,
Features: append([]string{}, lists.Docker().Manifests[i].Platform.Features...),
OSFeatures: append([]string{}, platform.OSFeatures...),
Size: instance.Size,
}
instanceInfos = append(instanceInfos, instanceInfo)
}
} else {
list, err := manifest.ListFromBlob(primaryManifestBytes, primaryManifestType)
if err != nil {
return "", errors.Wrapf(err, "error parsing manifest list in %q", transports.ImageName(ref))
}
instanceDigest, err := list.ChooseInstance(sys)
if err != nil {
return "", errors.Wrapf(err, "error selecting image from manifest list in %q", transports.ImageName(ref))
}
added := false
for i, instance := range lists.OCIv1().Manifests {
if instance.Digest != instanceDigest {
continue
}
platform := instance.Platform
if platform == nil {
platform = &v1.Platform{}
}
instanceInfo := instanceInfo{
instanceDigest: &instanceDigest,
OS: platform.OS,
Architecture: platform.Architecture,
OSVersion: platform.OSVersion,
Variant: platform.Variant,
Features: append([]string{}, lists.Docker().Manifests[i].Platform.Features...),
OSFeatures: append([]string{}, platform.OSFeatures...),
Size: instance.Size,
}
instanceInfos = append(instanceInfos, instanceInfo)
added = true
}
if !added {
instanceInfo := instanceInfo{
instanceDigest: &instanceDigest,
}
instanceInfos = append(instanceInfos, instanceInfo)
}
}
} else {
instanceInfo := instanceInfo{
instanceDigest: nil,
}
instanceInfos = append(instanceInfos, instanceInfo)
}
for _, instanceInfo := range instanceInfos {
if instanceInfo.OS == "" || instanceInfo.Architecture == "" {
img, err := image.FromUnparsedImage(ctx, sys, image.UnparsedInstance(src, instanceInfo.instanceDigest))
if err != nil {
return "", errors.Wrapf(err, "error reading configuration blob from %q", transports.ImageName(ref))
}
config, err := img.OCIConfig(ctx)
if err != nil {
return "", errors.Wrapf(err, "error reading info about config blob from %q", transports.ImageName(ref))
}
if instanceInfo.OS == "" {
instanceInfo.OS = config.OS
}
if instanceInfo.Architecture == "" {
instanceInfo.Architecture = config.Architecture
}
}
manifestBytes, manifestType, err := src.GetManifest(ctx, instanceInfo.instanceDigest)
if err != nil {
return "", errors.Wrapf(err, "error reading manifest from %q, instance %q", transports.ImageName(ref), instanceInfo.instanceDigest)
}
if instanceInfo.instanceDigest == nil {
manifestDigest, err = manifest.Digest(manifestBytes)
if err != nil {
return "", errors.Wrapf(err, "error computing digest of manifest from %q", transports.ImageName(ref))
}
instanceInfo.instanceDigest = &manifestDigest
instanceInfo.Size = int64(len(manifestBytes))
} else {
if manifestDigest == "" {
manifestDigest = *instanceInfo.instanceDigest
}
}
err = l.List.AddInstance(*instanceInfo.instanceDigest, instanceInfo.Size, manifestType, instanceInfo.OS, instanceInfo.Architecture, instanceInfo.OSVersion, instanceInfo.OSFeatures, instanceInfo.Variant, instanceInfo.Features, instanceInfo.Annotations)
if err != nil {
return "", errors.Wrapf(err, "error adding instance with digest %q", *instanceInfo.instanceDigest)
}
if _, ok := l.instances[*instanceInfo.instanceDigest]; !ok {
l.instances[*instanceInfo.instanceDigest] = transports.ImageName(ref)
}
}
return manifestDigest, nil
}
// Remove filters out any instances in the list which match the specified digest.
func (l *list) Remove(instanceDigest digest.Digest) error {
err := l.List.Remove(instanceDigest)
if err == nil {
if _, needToDelete := l.instances[instanceDigest]; needToDelete {
delete(l.instances, instanceDigest)
}
}
return err
}

View File

@ -0,0 +1,16 @@
package manifests
import (
"errors"
)
var (
// ErrDigestNotFound is returned when we look for an image instance
// with a particular digest in a list or index, and fail to find it.
ErrDigestNotFound = errors.New("no image instance matching the specified digest was found in the list or index")
// ErrManifestTypeNotSupported is returned when we attempt to parse a
// manifest with a known MIME type as a list or index, or when we attempt
// to serialize a list or index to a manifest with a MIME type that we
// don't know how to encode.
ErrManifestTypeNotSupported = errors.New("manifest type not supported")
)

View File

@ -0,0 +1,493 @@
package manifests
import (
"encoding/json"
"os"
"github.com/containers/image/v5/manifest"
digest "github.com/opencontainers/go-digest"
imgspec "github.com/opencontainers/image-spec/specs-go"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
// List is a generic interface for manipulating a manifest list or an image
// index.
type List interface {
AddInstance(manifestDigest digest.Digest, manifestSize int64, manifestType, os, architecture, osVersion string, osFeatures []string, variant string, features []string, annotations []string) error
Remove(instanceDigest digest.Digest) error
SetURLs(instanceDigest digest.Digest, urls []string) error
URLs(instanceDigest digest.Digest) ([]string, error)
SetAnnotations(instanceDigest *digest.Digest, annotations map[string]string) error
Annotations(instanceDigest *digest.Digest) (map[string]string, error)
SetOS(instanceDigest digest.Digest, os string) error
OS(instanceDigest digest.Digest) (string, error)
SetArchitecture(instanceDigest digest.Digest, arch string) error
Architecture(instanceDigest digest.Digest) (string, error)
SetOSVersion(instanceDigest digest.Digest, osVersion string) error
OSVersion(instanceDigest digest.Digest) (string, error)
SetVariant(instanceDigest digest.Digest, variant string) error
Variant(instanceDigest digest.Digest) (string, error)
SetFeatures(instanceDigest digest.Digest, features []string) error
Features(instanceDigest digest.Digest) ([]string, error)
SetOSFeatures(instanceDigest digest.Digest, osFeatures []string) error
OSFeatures(instanceDigest digest.Digest) ([]string, error)
Serialize(mimeType string) ([]byte, error)
Instances() []digest.Digest
OCIv1() *v1.Index
Docker() *manifest.Schema2List
findDocker(instanceDigest digest.Digest) (*manifest.Schema2ManifestDescriptor, error)
findOCIv1(instanceDigest digest.Digest) (*v1.Descriptor, error)
}
type list struct {
docker manifest.Schema2List
oci v1.Index
}
// OCIv1 returns the list as a Docker schema 2 list. The returned structure should NOT be modified.
func (l *list) Docker() *manifest.Schema2List {
return &l.docker
}
// OCIv1 returns the list as an OCI image index. The returned structure should NOT be modified.
func (l *list) OCIv1() *v1.Index {
return &l.oci
}
// Create creates a new list.
func Create() List {
return &list{
docker: manifest.Schema2List{
SchemaVersion: 2,
MediaType: manifest.DockerV2ListMediaType,
},
oci: v1.Index{
Versioned: imgspec.Versioned{SchemaVersion: 2},
},
}
}
// AddInstance adds an entry for the specified manifest digest, with assorted
// additional information specified in parameters, to the list or index.
func (l *list) AddInstance(manifestDigest digest.Digest, manifestSize int64, manifestType, osName, architecture, osVersion string, osFeatures []string, variant string, features []string, annotations []string) error {
if err := l.Remove(manifestDigest); err != nil && !os.IsNotExist(errors.Cause(err)) {
return err
}
schema2platform := manifest.Schema2PlatformSpec{
Architecture: architecture,
OS: osName,
OSVersion: osVersion,
OSFeatures: osFeatures,
Variant: variant,
Features: features,
}
l.docker.Manifests = append(l.docker.Manifests, manifest.Schema2ManifestDescriptor{
Schema2Descriptor: manifest.Schema2Descriptor{
MediaType: manifestType,
Size: manifestSize,
Digest: manifestDigest,
},
Platform: schema2platform,
})
ociv1platform := v1.Platform{
Architecture: architecture,
OS: osName,
OSVersion: osVersion,
OSFeatures: osFeatures,
Variant: variant,
}
l.oci.Manifests = append(l.oci.Manifests, v1.Descriptor{
MediaType: manifestType,
Size: manifestSize,
Digest: manifestDigest,
Platform: &ociv1platform,
})
return nil
}
// Remove filters out any instances in the list which match the specified digest.
func (l *list) Remove(instanceDigest digest.Digest) error {
err := errors.Wrapf(os.ErrNotExist, "no instance matching digest %q found in manifest list", instanceDigest)
newDockerManifests := make([]manifest.Schema2ManifestDescriptor, 0, len(l.docker.Manifests))
for i := range l.docker.Manifests {
if l.docker.Manifests[i].Digest != instanceDigest {
newDockerManifests = append(newDockerManifests, l.docker.Manifests[i])
} else {
err = nil
}
}
l.docker.Manifests = newDockerManifests
newOCIv1Manifests := make([]v1.Descriptor, 0, len(l.oci.Manifests))
for i := range l.oci.Manifests {
if l.oci.Manifests[i].Digest != instanceDigest {
newOCIv1Manifests = append(newOCIv1Manifests, l.oci.Manifests[i])
} else {
err = nil
}
}
l.oci.Manifests = newOCIv1Manifests
return err
}
func (l *list) findDocker(instanceDigest digest.Digest) (*manifest.Schema2ManifestDescriptor, error) {
for i := range l.docker.Manifests {
if l.docker.Manifests[i].Digest == instanceDigest {
return &l.docker.Manifests[i], nil
}
}
return nil, errors.Wrapf(ErrDigestNotFound, "no Docker manifest matching digest %q was found in list", instanceDigest.String())
}
func (l *list) findOCIv1(instanceDigest digest.Digest) (*v1.Descriptor, error) {
for i := range l.oci.Manifests {
if l.oci.Manifests[i].Digest == instanceDigest {
return &l.oci.Manifests[i], nil
}
}
return nil, errors.Wrapf(ErrDigestNotFound, "no OCI manifest matching digest %q was found in list", instanceDigest.String())
}
// SetURLs sets the URLs where the manifest might also be found.
func (l *list) SetURLs(instanceDigest digest.Digest, urls []string) error {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return err
}
docker, err := l.findDocker(instanceDigest)
if err != nil {
return err
}
oci.URLs = append([]string{}, urls...)
docker.URLs = append([]string{}, urls...)
return nil
}
// URLs retrieves the locations from which this object might possibly be downloaded.
func (l *list) URLs(instanceDigest digest.Digest) ([]string, error) {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return nil, err
}
return append([]string{}, oci.URLs...), nil
}
// SetAnnotations sets annotations on the image index, or on a specific manifest.
// The field is specific to the OCI image index format, and is not present in Docker manifest lists.
func (l *list) SetAnnotations(instanceDigest *digest.Digest, annotations map[string]string) error {
a := &l.oci.Annotations
if instanceDigest != nil {
oci, err := l.findOCIv1(*instanceDigest)
if err != nil {
return err
}
a = &oci.Annotations
}
(*a) = make(map[string]string)
for k, v := range annotations {
(*a)[k] = v
}
return nil
}
// Annotations retrieves the annotations which have been set on the image index, or on one instance.
// The field is specific to the OCI image index format, and is not present in Docker manifest lists.
func (l *list) Annotations(instanceDigest *digest.Digest) (map[string]string, error) {
a := l.oci.Annotations
if instanceDigest != nil {
oci, err := l.findOCIv1(*instanceDigest)
if err != nil {
return nil, err
}
a = oci.Annotations
}
annotations := make(map[string]string)
for k, v := range a {
annotations[k] = v
}
return annotations, nil
}
// SetOS sets the OS field in the platform information associated with the instance with the specified digest.
func (l *list) SetOS(instanceDigest digest.Digest, os string) error {
docker, err := l.findDocker(instanceDigest)
if err != nil {
return err
}
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return err
}
docker.Platform.OS = os
oci.Platform.OS = os
return nil
}
// OS retrieves the OS field in the platform information associated with the instance with the specified digest.
func (l *list) OS(instanceDigest digest.Digest) (string, error) {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return "", err
}
return oci.Platform.OS, nil
}
// SetArchitecture sets the Architecture field in the platform information associated with the instance with the specified digest.
func (l *list) SetArchitecture(instanceDigest digest.Digest, arch string) error {
docker, err := l.findDocker(instanceDigest)
if err != nil {
return err
}
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return err
}
docker.Platform.Architecture = arch
oci.Platform.Architecture = arch
return nil
}
// Architecture retrieves the Architecture field in the platform information associated with the instance with the specified digest.
func (l *list) Architecture(instanceDigest digest.Digest) (string, error) {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return "", err
}
return oci.Platform.Architecture, nil
}
// SetOSVersion sets the OSVersion field in the platform information associated with the instance with the specified digest.
func (l *list) SetOSVersion(instanceDigest digest.Digest, osVersion string) error {
docker, err := l.findDocker(instanceDigest)
if err != nil {
return err
}
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return err
}
docker.Platform.OSVersion = osVersion
oci.Platform.OSVersion = osVersion
return nil
}
// OSVersion retrieves the OSVersion field in the platform information associated with the instance with the specified digest.
func (l *list) OSVersion(instanceDigest digest.Digest) (string, error) {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return "", err
}
return oci.Platform.OSVersion, nil
}
// SetVariant sets the Variant field in the platform information associated with the instance with the specified digest.
func (l *list) SetVariant(instanceDigest digest.Digest, variant string) error {
docker, err := l.findDocker(instanceDigest)
if err != nil {
return err
}
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return err
}
docker.Platform.Variant = variant
oci.Platform.Variant = variant
return nil
}
// Variant retrieves the Variant field in the platform information associated with the instance with the specified digest.
func (l *list) Variant(instanceDigest digest.Digest) (string, error) {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return "", err
}
return oci.Platform.Variant, nil
}
// SetFeatures sets the features list in the platform information associated with the instance with the specified digest.
// The field is specific to the Docker manifest list format, and is not present in OCI's image indexes.
func (l *list) SetFeatures(instanceDigest digest.Digest, features []string) error {
docker, err := l.findDocker(instanceDigest)
if err != nil {
return err
}
docker.Platform.Features = append([]string{}, features...)
// no OCI equivalent
return nil
}
// Features retrieves the features list from the platform information associated with the instance with the specified digest.
// The field is specific to the Docker manifest list format, and is not present in OCI's image indexes.
func (l *list) Features(instanceDigest digest.Digest) ([]string, error) {
docker, err := l.findDocker(instanceDigest)
if err != nil {
return nil, err
}
return append([]string{}, docker.Platform.Features...), nil
}
// SetOSFeatures sets the OS features list in the platform information associated with the instance with the specified digest.
func (l *list) SetOSFeatures(instanceDigest digest.Digest, osFeatures []string) error {
docker, err := l.findDocker(instanceDigest)
if err != nil {
return err
}
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return err
}
docker.Platform.OSFeatures = append([]string{}, osFeatures...)
oci.Platform.OSFeatures = append([]string{}, osFeatures...)
return nil
}
// OSFeatures retrieves the OS features list from the platform information associated with the instance with the specified digest.
func (l *list) OSFeatures(instanceDigest digest.Digest) ([]string, error) {
oci, err := l.findOCIv1(instanceDigest)
if err != nil {
return nil, err
}
return append([]string{}, oci.Platform.OSFeatures...), nil
}
// FromBlob builds a list from an encoded manifest list or image index.
func FromBlob(manifestBytes []byte) (List, error) {
manifestType := manifest.GuessMIMEType(manifestBytes)
list := &list{
docker: manifest.Schema2List{
SchemaVersion: 2,
MediaType: manifest.DockerV2ListMediaType,
},
oci: v1.Index{
Versioned: imgspec.Versioned{SchemaVersion: 2},
},
}
switch manifestType {
default:
return nil, errors.Wrapf(ErrManifestTypeNotSupported, "unable to load manifest list: unsupported format %q", manifestType)
case manifest.DockerV2ListMediaType:
if err := json.Unmarshal(manifestBytes, &list.docker); err != nil {
return nil, errors.Wrapf(err, "unable to parse Docker manifest list from image")
}
for _, m := range list.docker.Manifests {
list.oci.Manifests = append(list.oci.Manifests, v1.Descriptor{
MediaType: m.Schema2Descriptor.MediaType,
Size: m.Schema2Descriptor.Size,
Digest: m.Schema2Descriptor.Digest,
Platform: &v1.Platform{
Architecture: m.Platform.Architecture,
OS: m.Platform.OS,
OSVersion: m.Platform.OSVersion,
OSFeatures: m.Platform.OSFeatures,
Variant: m.Platform.Variant,
},
})
}
case v1.MediaTypeImageIndex:
if err := json.Unmarshal(manifestBytes, &list.oci); err != nil {
return nil, errors.Wrapf(err, "unable to parse OCIv1 manifest list")
}
for _, m := range list.oci.Manifests {
platform := m.Platform
if platform == nil {
platform = &v1.Platform{}
}
list.docker.Manifests = append(list.docker.Manifests, manifest.Schema2ManifestDescriptor{
Schema2Descriptor: manifest.Schema2Descriptor{
MediaType: m.MediaType,
Size: m.Size,
Digest: m.Digest,
},
Platform: manifest.Schema2PlatformSpec{
Architecture: platform.Architecture,
OS: platform.OS,
OSVersion: platform.OSVersion,
OSFeatures: platform.OSFeatures,
Variant: platform.Variant,
},
})
}
}
return list, nil
}
func (l *list) preferOCI() bool {
// If we have any data that's only in the OCI format, use that.
for _, m := range l.oci.Manifests {
if len(m.URLs) > 0 {
return true
}
if len(m.Annotations) > 0 {
return true
}
}
// If we have any data that's only in the Docker format, use that.
for _, m := range l.docker.Manifests {
if len(m.Platform.Features) > 0 {
return false
}
}
// If we have no manifests, remember that the Docker format is
// explicitly typed, so use that. Otherwise, default to using the OCI
// format.
return len(l.docker.Manifests) != 0
}
// Serialize encodes the list using the specified format, or by selecting one
// which it thinks is appropriate.
func (l *list) Serialize(mimeType string) ([]byte, error) {
var manifestBytes []byte
switch mimeType {
case "":
if l.preferOCI() {
manifest, err := json.Marshal(&l.oci)
if err != nil {
return nil, errors.Wrapf(err, "error marshalling OCI image index")
}
manifestBytes = manifest
} else {
manifest, err := json.Marshal(&l.docker)
if err != nil {
return nil, errors.Wrapf(err, "error marshalling Docker manifest list")
}
manifestBytes = manifest
}
case v1.MediaTypeImageIndex:
manifest, err := json.Marshal(&l.oci)
if err != nil {
return nil, errors.Wrapf(err, "error marshalling OCI image index")
}
manifestBytes = manifest
case manifest.DockerV2ListMediaType:
manifest, err := json.Marshal(&l.docker)
if err != nil {
return nil, errors.Wrapf(err, "error marshalling Docker manifest list")
}
manifestBytes = manifest
default:
return nil, errors.Wrapf(ErrManifestTypeNotSupported, "serializing list to type %q not implemented", mimeType)
}
return manifestBytes, nil
}
// Instances returns the list of image instances mentioned in this list.
func (l *list) Instances() []digest.Digest {
instances := make([]digest.Digest, 0, len(l.oci.Manifests))
for _, instance := range l.oci.Manifests {
instances = append(instances, instance.Digest)
}
return instances
}

View File

@ -0,0 +1,17 @@
package supplemented
import (
"errors"
"github.com/containers/buildah/pkg/manifests"
)
var (
// ErrDigestNotFound is returned when we look for an image instance
// with a particular digest in a list or index, and fail to find it.
ErrDigestNotFound = manifests.ErrDigestNotFound
// ErrBlobNotFound is returned when try to figure out which supplemental
// image we should ask for a blob with the specified characteristics,
// based on the information in each of the supplemental images' manifests.
ErrBlobNotFound = errors.New("location of blob could not be determined")
)

View File

@ -0,0 +1,393 @@
package supplemented
import (
"container/list"
"context"
"io"
cp "github.com/containers/image/v5/copy"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
multierror "github.com/hashicorp/go-multierror"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// supplementedImageReference groups multiple references together.
type supplementedImageReference struct {
types.ImageReference
references []types.ImageReference
multiple cp.ImageListSelection
instances []digest.Digest
}
// supplementedImageSource represents an image, plus all of the blobs of other images.
type supplementedImageSource struct {
types.ImageSource
reference types.ImageReference
manifest []byte // The manifest list or image index.
manifestType string // The MIME type of the manifest list or image index.
sourceDefaultInstances map[types.ImageSource]digest.Digest // The default manifest instances of open ImageSource objects.
sourceInstancesByInstance map[digest.Digest]types.ImageSource // A map from manifest instance digests to open ImageSource objects.
instancesByBlobDigest map[digest.Digest]digest.Digest // A map from blob digests to manifest instance digests.
}
// Reference groups one reference and some number of additional references
// together as a group. The first reference's default instance will be treated
// as the default instance of the resulting reference, with the other
// references' instances made available as instances for their respective
// digests.
func Reference(ref types.ImageReference, supplemental []types.ImageReference, multiple cp.ImageListSelection, instances []digest.Digest) types.ImageReference {
if len(instances) > 0 {
i := make([]digest.Digest, len(instances))
copy(i, instances)
instances = i
}
return &supplementedImageReference{
ImageReference: ref,
references: append([]types.ImageReference{}, supplemental...),
multiple: multiple,
instances: instances,
}
}
// NewImage returns a new higher-level view of the image.
func (s *supplementedImageReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) {
src, err := s.NewImageSource(ctx, sys)
if err != nil {
return nil, errors.Wrapf(err, "error building a new Image using an ImageSource")
}
return image.FromSource(ctx, sys, src)
}
// NewImageSource opens the referenced images, scans their manifests for
// instances, and builds mappings from each blob mentioned in them to their
// instances.
func (s *supplementedImageReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (iss types.ImageSource, err error) {
sources := make(map[digest.Digest]types.ImageSource)
defaultInstances := make(map[types.ImageSource]digest.Digest)
instances := make(map[digest.Digest]digest.Digest)
var sis *supplementedImageSource
// Open the default instance for reading.
top, err := s.ImageReference.NewImageSource(ctx, sys)
if err != nil {
return nil, errors.Wrapf(err, "error opening %q as image source", transports.ImageName(s.ImageReference))
}
defer func() {
if err != nil {
if iss != nil {
// The composite source has been created. Use its Close method.
if err2 := iss.Close(); err2 != nil {
logrus.Errorf("error opening image: %v", err2)
}
} else if top != nil {
// The composite source has not been created, but the top was already opened. Close it.
if err2 := top.Close(); err2 != nil {
logrus.Errorf("error opening image: %v", err2)
}
}
}
}()
var addSingle, addMulti func(manifestBytes []byte, manifestType string, src types.ImageSource) error
type manifestToRead struct {
src types.ImageSource
instance *digest.Digest
}
manifestsToRead := list.New()
addSingle = func(manifestBytes []byte, manifestType string, src types.ImageSource) error {
// Mark this instance as being associated with this ImageSource.
manifestDigest, err := manifest.Digest(manifestBytes)
if err != nil {
return errors.Wrapf(err, "error computing digest over manifest %q", string(manifestBytes))
}
sources[manifestDigest] = src
// Parse the manifest as a single image.
man, err := manifest.FromBlob(manifestBytes, manifestType)
if err != nil {
return errors.Wrapf(err, "error parsing manifest %q", string(manifestBytes))
}
// Log the config blob's digest and the blobs of its layers as associated with this manifest.
config := man.ConfigInfo()
if config.Digest != "" {
instances[config.Digest] = manifestDigest
logrus.Debugf("blob %q belongs to %q", config.Digest, manifestDigest)
}
layers := man.LayerInfos()
for _, layer := range layers {
instances[layer.Digest] = manifestDigest
logrus.Debugf("layer %q belongs to %q", layer.Digest, manifestDigest)
}
return nil
}
addMulti = func(manifestBytes []byte, manifestType string, src types.ImageSource) error {
// Mark this instance as being associated with this ImageSource.
manifestDigest, err := manifest.Digest(manifestBytes)
if err != nil {
return errors.Wrapf(err, "error computing manifest digest")
}
sources[manifestDigest] = src
// Parse the manifest as a list of images.
list, err := manifest.ListFromBlob(manifestBytes, manifestType)
if err != nil {
return errors.Wrapf(err, "error parsing manifest blob %q as a %q", string(manifestBytes), manifestType)
}
// Figure out which of its instances we want to look at.
var chaseInstances []digest.Digest
switch s.multiple {
case cp.CopySystemImage:
instance, err := list.ChooseInstance(sys)
if err != nil {
return errors.Wrapf(err, "error selecting appropriate instance from list")
}
chaseInstances = []digest.Digest{instance}
case cp.CopySpecificImages:
chaseInstances = s.instances
case cp.CopyAllImages:
chaseInstances = list.Instances()
}
// Queue these manifest instances for reading from this
// ImageSource later, if we don't stumble across them somewhere
// else first.
for _, instanceIterator := range chaseInstances {
instance := instanceIterator
next := &manifestToRead{
src: src,
instance: &instance,
}
if src == top {
// Prefer any other source.
manifestsToRead.PushBack(next)
} else {
// Prefer this source over the first ("main") one.
manifestsToRead.PushFront(next)
}
}
return nil
}
visitedReferences := make(map[types.ImageReference]struct{})
for i, ref := range append([]types.ImageReference{s.ImageReference}, s.references...) {
if _, visited := visitedReferences[ref]; visited {
continue
}
visitedReferences[ref] = struct{}{}
// Open this image for reading.
var src types.ImageSource
if ref == s.ImageReference {
src = top
} else {
src, err = ref.NewImageSource(ctx, sys)
if err != nil {
return nil, errors.Wrapf(err, "error opening %q as image source", transports.ImageName(ref))
}
}
// Read the default manifest for the image.
manifestBytes, manifestType, err := src.GetManifest(ctx, nil)
if err != nil {
return nil, errors.Wrapf(err, "error reading default manifest from image %q", transports.ImageName(ref))
}
// If this is the first image, mark it as our starting point.
if i == 0 {
sources[""] = src
sis = &supplementedImageSource{
ImageSource: top,
reference: s,
manifest: manifestBytes,
manifestType: manifestType,
sourceDefaultInstances: defaultInstances,
sourceInstancesByInstance: sources,
instancesByBlobDigest: instances,
}
iss = sis
}
// Record the digest of the ImageSource's default instance's manifest.
manifestDigest, err := manifest.Digest(manifestBytes)
if err != nil {
return nil, errors.Wrapf(err, "error computing digest of manifest from image %q", transports.ImageName(ref))
}
sis.sourceDefaultInstances[src] = manifestDigest
// If the ImageSource's default manifest is a list, parse each of its instances.
if manifest.MIMETypeIsMultiImage(manifestType) {
if err = addMulti(manifestBytes, manifestType, src); err != nil {
return nil, errors.Wrapf(err, "error adding multi-image %q", transports.ImageName(ref))
}
} else {
if err = addSingle(manifestBytes, manifestType, src); err != nil {
return nil, errors.Wrapf(err, "error adding single image %q", transports.ImageName(ref))
}
}
}
// Parse the rest of the instances.
for manifestsToRead.Front() != nil {
front := manifestsToRead.Front()
value := front.Value
manifestToRead, ok := value.(*manifestToRead)
if !ok {
panic("bug: wrong type looking for *manifestToRead in list?")
}
manifestsToRead.Remove(front)
// If we already read this manifest, no need to read it again.
if _, alreadyRead := sources[*manifestToRead.instance]; alreadyRead {
continue
}
// Read the instance's manifest.
manifestBytes, manifestType, err := manifestToRead.src.GetManifest(ctx, manifestToRead.instance)
if err != nil {
// if errors.Cause(err) == storage.ErrImageUnknown || os.IsNotExist(errors.Cause(err)) {
// Trust that we either don't need it, or that it's in another reference.
// continue
// }
return nil, errors.Wrapf(err, "error reading manifest for instance %q", manifestToRead.instance)
}
if manifest.MIMETypeIsMultiImage(manifestType) {
// Add the list's contents.
if err = addMulti(manifestBytes, manifestType, manifestToRead.src); err != nil {
return nil, errors.Wrapf(err, "error adding single image instance %q", manifestToRead.instance)
}
} else {
// Add the single image's contents.
if err = addSingle(manifestBytes, manifestType, manifestToRead.src); err != nil {
return nil, errors.Wrapf(err, "error adding single image instance %q", manifestToRead.instance)
}
}
}
return iss, nil
}
func (s *supplementedImageReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error {
return errors.Errorf("deletion of images not implemented")
}
func (s *supplementedImageSource) Close() error {
var returnErr *multierror.Error
closed := make(map[types.ImageSource]struct{})
for _, sourceInstance := range s.sourceInstancesByInstance {
if _, closed := closed[sourceInstance]; closed {
continue
}
if err := sourceInstance.Close(); err != nil {
returnErr = multierror.Append(returnErr, err)
}
closed[sourceInstance] = struct{}{}
}
return returnErr.ErrorOrNil()
}
func (s *supplementedImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
requestInstanceDigest := instanceDigest
if instanceDigest == nil {
return s.manifest, s.manifestType, nil
}
if sourceInstance, ok := s.sourceInstancesByInstance[*instanceDigest]; ok {
if *instanceDigest == s.sourceDefaultInstances[sourceInstance] {
requestInstanceDigest = nil
}
return sourceInstance.GetManifest(ctx, requestInstanceDigest)
}
return nil, "", errors.Wrapf(ErrDigestNotFound, "error getting manifest for digest %q", *instanceDigest)
}
func (s *supplementedImageSource) GetBlob(ctx context.Context, blob types.BlobInfo, bic types.BlobInfoCache) (io.ReadCloser, int64, error) {
sourceInstance, ok := s.instancesByBlobDigest[blob.Digest]
if !ok {
return nil, -1, errors.Wrapf(ErrBlobNotFound, "error blob %q in known instances", blob.Digest)
}
src, ok := s.sourceInstancesByInstance[sourceInstance]
if !ok {
return nil, -1, errors.Wrapf(ErrDigestNotFound, "error getting image source for instance %q", sourceInstance)
}
return src.GetBlob(ctx, blob, bic)
}
func (s *supplementedImageSource) HasThreadSafeGetBlob() bool {
checked := make(map[types.ImageSource]struct{})
for _, sourceInstance := range s.sourceInstancesByInstance {
if _, checked := checked[sourceInstance]; checked {
continue
}
if !sourceInstance.HasThreadSafeGetBlob() {
return false
}
checked[sourceInstance] = struct{}{}
}
return true
}
func (s *supplementedImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) {
var src types.ImageSource
requestInstanceDigest := instanceDigest
if instanceDigest == nil {
if sourceInstance, ok := s.sourceInstancesByInstance[""]; ok {
src = sourceInstance
}
} else {
if sourceInstance, ok := s.sourceInstancesByInstance[*instanceDigest]; ok {
src = sourceInstance
}
if *instanceDigest == s.sourceDefaultInstances[src] {
requestInstanceDigest = nil
}
}
if src != nil {
return src.GetSignatures(ctx, requestInstanceDigest)
}
return nil, errors.Wrapf(ErrDigestNotFound, "error finding instance for instance digest %q to read signatures", *instanceDigest)
}
func (s *supplementedImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) {
var src types.ImageSource
requestInstanceDigest := instanceDigest
if instanceDigest == nil {
if sourceInstance, ok := s.sourceInstancesByInstance[""]; ok {
src = sourceInstance
}
} else {
if sourceInstance, ok := s.sourceInstancesByInstance[*instanceDigest]; ok {
src = sourceInstance
}
if *instanceDigest == s.sourceDefaultInstances[src] {
requestInstanceDigest = nil
}
}
if src != nil {
blobInfos, err := src.LayerInfosForCopy(ctx, requestInstanceDigest)
if err != nil {
return nil, errors.Wrapf(err, "error reading layer infos for copy from instance %q", instanceDigest)
}
var manifestDigest digest.Digest
if instanceDigest != nil {
manifestDigest = *instanceDigest
}
for _, blobInfo := range blobInfos {
s.instancesByBlobDigest[blobInfo.Digest] = manifestDigest
}
return blobInfos, nil
}
return nil, errors.Wrapf(ErrDigestNotFound, "error finding instance for instance digest %q to copy layers", *instanceDigest)
}