enabled hyperv image downloads

now that we have public hypverv fcos artifacts, we can download them
instead of requiring a special build.

Signed-off-by: Brent Baude <bbaude@redhat.com>

[NO NEW TESTS NEEDED]
This commit is contained in:
Brent Baude
2023-08-07 13:46:04 -05:00
parent 3e8f19be9d
commit 9e14e3ebe5
21 changed files with 745 additions and 301 deletions

View File

@ -49,6 +49,10 @@ type CopyOptions struct {
CompressionFormat *compression.Algorithm
// CompressionLevel specifies what compression level is used
CompressionLevel *int
// ForceCompressionFormat ensures that the compression algorithm set in
// CompressionFormat is used exclusively, and blobs of other compression
// algorithms are not reused.
ForceCompressionFormat bool
// containers-auth.json(5) file to use when authenticating against
// container registries.
@ -294,6 +298,7 @@ func (r *Runtime) newCopier(options *CopyOptions) (*copier, error) {
c.imageCopyOptions.ProgressInterval = time.Second
}
c.imageCopyOptions.ForceCompressionFormat = options.ForceCompressionFormat
c.imageCopyOptions.ForceManifestMIMEType = options.ManifestMIMEType
c.imageCopyOptions.SourceCtx = c.systemContext
c.imageCopyOptions.DestinationCtx = c.systemContext

View File

@ -461,6 +461,7 @@ func (m *ManifestList) Push(ctx context.Context, destination string, options *Ma
SignSigstorePrivateKeyPassphrase: options.SignSigstorePrivateKeyPassphrase,
RemoveSignatures: options.RemoveSignatures,
ManifestType: options.ManifestMIMEType,
ForceCompressionFormat: options.ForceCompressionFormat,
}
_, d, err := m.list.Push(ctx, dest, pushOptions)

View File

@ -72,6 +72,7 @@ type PushOptions struct {
ManifestType string // the format to use when saving the list - possible options are oci, v2s1, and v2s2
SourceFilter LookupReferenceFunc // filter the list source
AddCompression []string // add existing instances with requested compression algorithms to manifest list
ForceCompressionFormat bool // force push with requested compression ignoring the blobs which can be reused.
}
// Create creates a new list containing information about the specified image,
@ -259,6 +260,7 @@ func (l *list) Push(ctx context.Context, dest types.ImageReference, options Push
SignSigstorePrivateKeyPassphrase: options.SignSigstorePrivateKeyPassphrase,
ForceManifestMIMEType: singleImageManifestType,
EnsureCompressionVariantsExist: compressionVariants,
ForceCompressionFormat: options.ForceCompressionFormat,
}
// Copy whatever we were asked to copy.

View File

@ -5,12 +5,14 @@ import "os"
// getDefaultImage returns the default machine image stream
// On Windows this refers to the Fedora major release number
func getDefaultMachineImage() string {
return "35"
return "testing"
}
// getDefaultMachineUser returns the user to use for rootless podman
// This is only for the hyperv and qemu implementations. WSL's user
// will be hardcoded in podman to "user"
func getDefaultMachineUser() string {
return "user"
return "core"
}
// isCgroup2UnifiedMode returns whether we are running in cgroup2 mode.

View File

@ -133,6 +133,10 @@ type Options struct {
// Invalid when copying a non-multi-architecture image. That will probably
// change in the future.
EnsureCompressionVariantsExist []OptionCompressionVariant
// ForceCompressionFormat ensures that the compression algorithm set in
// DestinationCtx.CompressionFormat is used exclusively, and blobs of other
// compression algorithms are not reused.
ForceCompressionFormat bool
}
// OptionCompressionVariant allows to supply information about
@ -163,6 +167,14 @@ type copier struct {
signersToClose []*signer.Signer // Signers that should be closed when this copier is destroyed.
}
// Internal function to validate `requireCompressionFormatMatch` for copySingleImageOptions
func shouldRequireCompressionFormatMatch(options *Options) (bool, error) {
if options.ForceCompressionFormat && (options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil) {
return false, fmt.Errorf("cannot use ForceCompressionFormat with undefined default compression format")
}
return options.ForceCompressionFormat, nil
}
// Image copies image from srcRef to destRef, using policyContext to validate
// source image admissibility. It returns the manifest which was written to
// the new copy of the image.
@ -269,8 +281,12 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
if len(options.EnsureCompressionVariantsExist) > 0 {
return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
}
requireCompressionFormatMatch, err := shouldRequireCompressionFormatMatch(options)
if err != nil {
return nil, err
}
// The simple case: just copy a single image.
single, err := c.copySingleImage(ctx, c.unparsedToplevel, nil, copySingleImageOptions{requireCompressionFormatMatch: false})
single, err := c.copySingleImage(ctx, c.unparsedToplevel, nil, copySingleImageOptions{requireCompressionFormatMatch: requireCompressionFormatMatch})
if err != nil {
return nil, err
}
@ -279,6 +295,10 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
if len(options.EnsureCompressionVariantsExist) > 0 {
return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
}
requireCompressionFormatMatch, err := shouldRequireCompressionFormatMatch(options)
if err != nil {
return nil, err
}
// This is a manifest list, and we weren't asked to copy multiple images. Choose a single image that
// matches the current system to copy, and copy it.
mfest, manifestType, err := c.unparsedToplevel.Manifest(ctx)
@ -295,7 +315,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
}
logrus.Debugf("Source is a manifest list; copying (only) instance %s for current system", instanceDigest)
unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest)
single, err := c.copySingleImage(ctx, unparsedInstance, nil, copySingleImageOptions{requireCompressionFormatMatch: false})
single, err := c.copySingleImage(ctx, unparsedInstance, nil, copySingleImageOptions{requireCompressionFormatMatch: requireCompressionFormatMatch})
if err != nil {
return nil, fmt.Errorf("copying system image from manifest list: %w", err)
}

View File

@ -32,6 +32,10 @@ type instanceCopy struct {
op instanceCopyKind
sourceDigest digest.Digest
// Fields which can be used by callers when operation
// is `instanceCopyCopy`
copyForceCompressionFormat bool
// Fields which can be used by callers when operation
// is `instanceCopyClone`
cloneCompressionVariant OptionCompressionVariant
@ -122,9 +126,14 @@ func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest.
if err != nil {
return res, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err)
}
forceCompressionFormat, err := shouldRequireCompressionFormatMatch(options)
if err != nil {
return nil, err
}
res = append(res, instanceCopy{
op: instanceCopyCopy,
sourceDigest: instanceDigest,
op: instanceCopyCopy,
sourceDigest: instanceDigest,
copyForceCompressionFormat: forceCompressionFormat,
})
platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform)
compressionList := compressionsByPlatform[platform]
@ -230,7 +239,7 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte,
logrus.Debugf("Copying instance %s (%d/%d)", instance.sourceDigest, i+1, len(instanceCopyList))
c.Printf("Copying image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList))
unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceCopyList[i].sourceDigest)
updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{requireCompressionFormatMatch: false})
updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{requireCompressionFormatMatch: instance.copyForceCompressionFormat})
if err != nil {
return nil, fmt.Errorf("copying image %d/%d from manifest list: %w", i+1, len(instanceCopyList), err)
}

View File

@ -170,8 +170,19 @@ func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit) error {
index.Manifests = append(index.Manifests, addedEntries...)
}
if len(addedEntries) != 0 || updatedAnnotations {
slices.SortStableFunc(index.Manifests, func(a, b imgspecv1.Descriptor) bool {
return !instanceIsZstd(a) && instanceIsZstd(b)
slices.SortStableFunc(index.Manifests, func(a, b imgspecv1.Descriptor) int {
// FIXME? With Go 1.21 and cmp.Compare available, turn instanceIsZstd into an integer score that can be compared, and generalizes
// into more algorithms?
aZstd := instanceIsZstd(a)
bZstd := instanceIsZstd(b)
switch {
case aZstd == bZstd:
return 0
case !aZstd: // Implies bZstd
return -1
default: // aZstd && !bZstd
return 1
}
})
}
return nil

View File

@ -82,6 +82,7 @@ func (css *candidateSortState) Swap(i, j int) {
func destructivelyPrioritizeReplacementCandidatesWithMax(cs []CandidateWithTime, primaryDigest, uncompressedDigest digest.Digest, maxCandidates int) []blobinfocache.BICReplacementCandidate2 {
// We don't need to use sort.Stable() because nanosecond timestamps are (presumably?) unique, so no two elements should
// compare equal.
// FIXME: Use slices.SortFunc after we update to Go 1.20 (Go 1.21?) and Time.Compare and cmp.Compare are available.
sort.Sort(&candidateSortState{
cs: cs,
primaryDigest: primaryDigest,

View File

@ -57,7 +57,7 @@ type storageImageDestination struct {
imageRef storageReference
directory string // Temporary directory where we store blobs until Commit() time
nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs
nextTempFileID atomic.Int32 // A counter that we use for computing filenames to assign to blobs
manifest []byte // Manifest contents, temporary
manifestDigest digest.Digest // Valid if len(manifest) != 0
signatures []byte // Signature contents, temporary
@ -154,7 +154,7 @@ func (s *storageImageDestination) Close() error {
}
func (s *storageImageDestination) computeNextBlobCacheFile() string {
return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1)))
return filepath.Join(s.directory, fmt.Sprintf("%d", s.nextTempFileID.Add(1)))
}
// PutBlobWithOptions writes contents of stream and returns data representing the result.
@ -763,7 +763,7 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
if len(layerBlobs) > 0 { // Can happen when using caches
prev := s.indexToStorageID[len(layerBlobs)-1]
if prev == nil {
return fmt.Errorf("Internal error: StorageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1)
return fmt.Errorf("Internal error: storageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1)
}
lastLayer = *prev
}
@ -775,6 +775,78 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
logrus.Debugf("setting image creation date to %s", inspect.Created)
options.CreationDate = *inspect.Created
}
// Set up to save the non-layer blobs as data items. Since we only share layers, they should all be in files, so
// we just need to screen out the ones that are actually layers to get the list of non-layers.
dataBlobs := set.New[digest.Digest]()
for blob := range s.filenames {
dataBlobs.Add(blob)
}
for _, layerBlob := range layerBlobs {
dataBlobs.Delete(layerBlob.Digest)
}
for _, blob := range dataBlobs.Values() {
v, err := os.ReadFile(s.filenames[blob])
if err != nil {
return fmt.Errorf("copying non-layer blob %q to image: %w", blob, err)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: blob.String(),
Data: v,
Digest: digest.Canonical.FromBytes(v),
})
}
// Set up to save the unparsedToplevel's manifest if it differs from
// the per-platform one, which is saved below.
if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) {
manifestDigest, err := manifest.Digest(toplevelManifest)
if err != nil {
return fmt.Errorf("digesting top-level manifest: %w", err)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: manifestBigDataKey(manifestDigest),
Data: toplevelManifest,
Digest: manifestDigest,
})
}
// Set up to save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store.
// Record the manifest twice: using a digest-specific key to allow references to that specific digest instance,
// and using storage.ImageDigestBigDataKey for future users that dont specify any digest and for compatibility with older readers.
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: manifestBigDataKey(s.manifestDigest),
Data: s.manifest,
Digest: s.manifestDigest,
})
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: storage.ImageDigestBigDataKey,
Data: s.manifest,
Digest: s.manifestDigest,
})
// Set up to save the signatures, if we have any.
if len(s.signatures) > 0 {
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: "signatures",
Data: s.signatures,
Digest: digest.Canonical.FromBytes(s.signatures),
})
}
for instanceDigest, signatures := range s.signatureses {
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: signatureBigDataKey(instanceDigest),
Data: signatures,
Digest: digest.Canonical.FromBytes(signatures),
})
}
// Set up to save our metadata.
metadata, err := json.Marshal(s)
if err != nil {
return fmt.Errorf("encoding metadata for image: %w", err)
}
if len(metadata) != 0 {
options.Metadata = string(metadata)
}
// Create the image record, pointing to the most-recently added layer.
intendedID := s.imageRef.id
if intendedID == "" {
@ -797,8 +869,26 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
}
logrus.Debugf("reusing image ID %q", img.ID)
oldNames = append(oldNames, img.Names...)
// set the data items and metadata on the already-present image
// FIXME: this _replaces_ any "signatures" blobs and their
// sizes (tracked in the metadata) which might have already
// been present with new values, when ideally we'd find a way
// to merge them since they all apply to the same image
for _, data := range options.BigData {
if err := s.imageRef.transport.store.SetImageBigData(img.ID, data.Key, data.Data, manifest.Digest); err != nil {
logrus.Debugf("error saving big data %q for image %q: %v", data.Key, img.ID, err)
return fmt.Errorf("saving big data %q for image %q: %w", data.Key, img.ID, err)
}
}
if options.Metadata != "" {
if err := s.imageRef.transport.store.SetMetadata(img.ID, options.Metadata); err != nil {
logrus.Debugf("error saving metadata for image %q: %v", img.ID, err)
return fmt.Errorf("saving metadata for image %q: %w", img.ID, err)
}
logrus.Debugf("saved image metadata %q", options.Metadata)
}
} else {
logrus.Debugf("created new image ID %q", img.ID)
logrus.Debugf("created new image ID %q with metadata %q", img.ID, options.Metadata)
}
// Clean up the unfinished image on any error.
@ -813,78 +903,7 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
}
}()
// Add the non-layer blobs as data items. Since we only share layers, they should all be in files, so
// we just need to screen out the ones that are actually layers to get the list of non-layers.
dataBlobs := set.New[digest.Digest]()
for blob := range s.filenames {
dataBlobs.Add(blob)
}
for _, layerBlob := range layerBlobs {
dataBlobs.Delete(layerBlob.Digest)
}
for _, blob := range dataBlobs.Values() {
v, err := os.ReadFile(s.filenames[blob])
if err != nil {
return fmt.Errorf("copying non-layer blob %q to image: %w", blob, err)
}
if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v, manifest.Digest); err != nil {
logrus.Debugf("error saving big data %q for image %q: %v", blob.String(), img.ID, err)
return fmt.Errorf("saving big data %q for image %q: %w", blob.String(), img.ID, err)
}
}
// Save the unparsedToplevel's manifest if it differs from the per-platform one, which is saved below.
if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) {
manifestDigest, err := manifest.Digest(toplevelManifest)
if err != nil {
return fmt.Errorf("digesting top-level manifest: %w", err)
}
key := manifestBigDataKey(manifestDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, toplevelManifest, manifest.Digest); err != nil {
logrus.Debugf("error saving top-level manifest for image %q: %v", img.ID, err)
return fmt.Errorf("saving top-level manifest for image %q: %w", img.ID, err)
}
}
// Save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store.
// Record the manifest twice: using a digest-specific key to allow references to that specific digest instance,
// and using storage.ImageDigestBigDataKey for future users that dont specify any digest and for compatibility with older readers.
key := manifestBigDataKey(s.manifestDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil {
logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
return fmt.Errorf("saving manifest for image %q: %w", img.ID, err)
}
key = storage.ImageDigestBigDataKey
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil {
logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
return fmt.Errorf("saving manifest for image %q: %w", img.ID, err)
}
// Save the signatures, if we have any.
if len(s.signatures) > 0 {
if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures, manifest.Digest); err != nil {
logrus.Debugf("error saving signatures for image %q: %v", img.ID, err)
return fmt.Errorf("saving signatures for image %q: %w", img.ID, err)
}
}
for instanceDigest, signatures := range s.signatureses {
key := signatureBigDataKey(instanceDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, signatures, manifest.Digest); err != nil {
logrus.Debugf("error saving signatures for image %q: %v", img.ID, err)
return fmt.Errorf("saving signatures for image %q: %w", img.ID, err)
}
}
// Save our metadata.
metadata, err := json.Marshal(s)
if err != nil {
logrus.Debugf("error encoding metadata for image %q: %v", img.ID, err)
return fmt.Errorf("encoding metadata for image %q: %w", img.ID, err)
}
if len(metadata) != 0 {
if err = s.imageRef.transport.store.SetMetadata(img.ID, string(metadata)); err != nil {
logrus.Debugf("error saving metadata for image %q: %v", img.ID, err)
return fmt.Errorf("saving metadata for image %q: %w", img.ID, err)
}
logrus.Debugf("saved image metadata %q", string(metadata))
}
// Adds the reference's name on the image. We don't need to worry about avoiding duplicate
// Add the reference's name on the image. We don't need to worry about avoiding duplicate
// values because AddNames() will deduplicate the list that we pass to it.
if name := s.imageRef.DockerReference(); name != nil {
if err := s.imageRef.transport.store.AddNames(img.ID, []string{name.String()}); err != nil {
@ -921,10 +940,7 @@ func (s *storageImageDestination) PutSignaturesWithFormat(ctx context.Context, s
return err
}
sizes = append(sizes, len(sig))
newblob := make([]byte, len(sigblob)+len(sig))
copy(newblob, sigblob)
copy(newblob[len(sigblob):], sig)
sigblob = newblob
sigblob = append(sigblob, sig...)
}
if instanceDigest == nil {
s.signatures = sigblob