Merge pull request #19596 from baude/allowdownload

enabled hyperv image downloads

vendor/github.com/containers/common/libimage/copier.go

@@ -49,6 +49,10 @@ type CopyOptions struct {
 	CompressionFormat *compression.Algorithm
 	// CompressionLevel specifies what compression level is used
 	CompressionLevel *int
+	// ForceCompressionFormat ensures that the compression algorithm set in
+	// CompressionFormat is used exclusively, and blobs of other compression
+	// algorithms are not reused.
+	ForceCompressionFormat bool
 
 	// containers-auth.json(5) file to use when authenticating against
 	// container registries.
@@ -294,6 +298,7 @@ func (r *Runtime) newCopier(options *CopyOptions) (*copier, error) {
 		c.imageCopyOptions.ProgressInterval = time.Second
 	}
 
+	c.imageCopyOptions.ForceCompressionFormat = options.ForceCompressionFormat
 	c.imageCopyOptions.ForceManifestMIMEType = options.ManifestMIMEType
 	c.imageCopyOptions.SourceCtx = c.systemContext
 	c.imageCopyOptions.DestinationCtx = c.systemContext

vendor/github.com/containers/common/libimage/manifest_list.go

@@ -461,6 +461,7 @@ func (m *ManifestList) Push(ctx context.Context, destination string, options *Ma
 		SignSigstorePrivateKeyPassphrase: options.SignSigstorePrivateKeyPassphrase,
 		RemoveSignatures:                 options.RemoveSignatures,
 		ManifestType:                     options.ManifestMIMEType,
+		ForceCompressionFormat:           options.ForceCompressionFormat,
 	}
 
 	_, d, err := m.list.Push(ctx, dest, pushOptions)

vendor/github.com/containers/common/libimage/manifests/manifests.go

@@ -72,6 +72,7 @@ type PushOptions struct {
 	ManifestType                     string                // the format to use when saving the list - possible options are oci, v2s1, and v2s2
 	SourceFilter                     LookupReferenceFunc   // filter the list source
 	AddCompression                   []string              // add existing instances with requested compression algorithms to manifest list
+	ForceCompressionFormat           bool                  // force push with requested compression ignoring the blobs which can be reused.
 }
 
 // Create creates a new list containing information about the specified image,
@@ -259,6 +260,7 @@ func (l *list) Push(ctx context.Context, dest types.ImageReference, options Push
 		SignSigstorePrivateKeyPassphrase: options.SignSigstorePrivateKeyPassphrase,
 		ForceManifestMIMEType:            singleImageManifestType,
 		EnsureCompressionVariantsExist:   compressionVariants,
+		ForceCompressionFormat:           options.ForceCompressionFormat,
 	}
 
 	// Copy whatever we were asked to copy.

vendor/github.com/containers/common/pkg/config/default_windows.go

@@ -5,12 +5,14 @@ import "os"
 // getDefaultImage returns the default machine image stream
 // On Windows this refers to the Fedora major release number
 func getDefaultMachineImage() string {
-	return "35"
+	return "testing"
 }
 
 // getDefaultMachineUser returns the user to use for rootless podman
+// This is only for the hyperv and qemu implementations.  WSL's user
+// will be hardcoded in podman to "user"
 func getDefaultMachineUser() string {
-	return "user"
+	return "core"
 }
 
 // isCgroup2UnifiedMode returns whether we are running in cgroup2 mode.

vendor/github.com/containers/image/v5/copy/copy.go

@@ -133,6 +133,10 @@ type Options struct {
 	// Invalid when copying a non-multi-architecture image. That will probably
 	// change in the future.
 	EnsureCompressionVariantsExist []OptionCompressionVariant
+	// ForceCompressionFormat ensures that the compression algorithm set in
+	// DestinationCtx.CompressionFormat is used exclusively, and blobs of other
+	// compression algorithms are not reused.
+	ForceCompressionFormat bool
 }
 
 // OptionCompressionVariant allows to supply information about
@@ -163,6 +167,14 @@ type copier struct {
 	signersToClose                []*signer.Signer    // Signers that should be closed when this copier is destroyed.
 }
 
+// Internal function to validate `requireCompressionFormatMatch` for copySingleImageOptions
+func shouldRequireCompressionFormatMatch(options *Options) (bool, error) {
+	if options.ForceCompressionFormat && (options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil) {
+		return false, fmt.Errorf("cannot use ForceCompressionFormat with undefined default compression format")
+	}
+	return options.ForceCompressionFormat, nil
+}
+
 // Image copies image from srcRef to destRef, using policyContext to validate
 // source image admissibility.  It returns the manifest which was written to
 // the new copy of the image.
@@ -269,8 +281,12 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
 		if len(options.EnsureCompressionVariantsExist) > 0 {
 			return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
 		}
+		requireCompressionFormatMatch, err := shouldRequireCompressionFormatMatch(options)
+		if err != nil {
+			return nil, err
+		}
 		// The simple case: just copy a single image.
-		single, err := c.copySingleImage(ctx, c.unparsedToplevel, nil, copySingleImageOptions{requireCompressionFormatMatch: false})
+		single, err := c.copySingleImage(ctx, c.unparsedToplevel, nil, copySingleImageOptions{requireCompressionFormatMatch: requireCompressionFormatMatch})
 		if err != nil {
 			return nil, err
 		}
@@ -279,6 +295,10 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
 		if len(options.EnsureCompressionVariantsExist) > 0 {
 			return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image")
 		}
+		requireCompressionFormatMatch, err := shouldRequireCompressionFormatMatch(options)
+		if err != nil {
+			return nil, err
+		}
 		// This is a manifest list, and we weren't asked to copy multiple images.  Choose a single image that
 		// matches the current system to copy, and copy it.
 		mfest, manifestType, err := c.unparsedToplevel.Manifest(ctx)
@@ -295,7 +315,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
 		}
 		logrus.Debugf("Source is a manifest list; copying (only) instance %s for current system", instanceDigest)
 		unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest)
-		single, err := c.copySingleImage(ctx, unparsedInstance, nil, copySingleImageOptions{requireCompressionFormatMatch: false})
+		single, err := c.copySingleImage(ctx, unparsedInstance, nil, copySingleImageOptions{requireCompressionFormatMatch: requireCompressionFormatMatch})
 		if err != nil {
 			return nil, fmt.Errorf("copying system image from manifest list: %w", err)
 		}

vendor/github.com/containers/image/v5/copy/multiple.go

@@ -32,6 +32,10 @@ type instanceCopy struct {
 	op           instanceCopyKind
 	sourceDigest digest.Digest
 
+	// Fields which can be used by callers when operation
+	// is `instanceCopyCopy`
+	copyForceCompressionFormat bool
+
 	// Fields which can be used by callers when operation
 	// is `instanceCopyClone`
 	cloneCompressionVariant OptionCompressionVariant
@@ -122,9 +126,14 @@ func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest.
 		if err != nil {
 			return res, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err)
 		}
+		forceCompressionFormat, err := shouldRequireCompressionFormatMatch(options)
+		if err != nil {
+			return nil, err
+		}
 		res = append(res, instanceCopy{
-			op:           instanceCopyCopy,
-			sourceDigest: instanceDigest,
+			op:                         instanceCopyCopy,
+			sourceDigest:               instanceDigest,
+			copyForceCompressionFormat: forceCompressionFormat,
 		})
 		platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform)
 		compressionList := compressionsByPlatform[platform]
@@ -230,7 +239,7 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte,
 			logrus.Debugf("Copying instance %s (%d/%d)", instance.sourceDigest, i+1, len(instanceCopyList))
 			c.Printf("Copying image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList))
 			unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceCopyList[i].sourceDigest)
-			updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{requireCompressionFormatMatch: false})
+			updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{requireCompressionFormatMatch: instance.copyForceCompressionFormat})
 			if err != nil {
 				return nil, fmt.Errorf("copying image %d/%d from manifest list: %w", i+1, len(instanceCopyList), err)
 			}

vendor/github.com/containers/image/v5/internal/manifest/oci_index.go

@@ -170,8 +170,19 @@ func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit) error {
 		index.Manifests = append(index.Manifests, addedEntries...)
 	}
 	if len(addedEntries) != 0 || updatedAnnotations {
-		slices.SortStableFunc(index.Manifests, func(a, b imgspecv1.Descriptor) bool {
-			return !instanceIsZstd(a) && instanceIsZstd(b)
+		slices.SortStableFunc(index.Manifests, func(a, b imgspecv1.Descriptor) int {
+			// FIXME? With Go 1.21 and cmp.Compare available, turn instanceIsZstd into an integer score that can be compared, and generalizes
+			// into more algorithms?
+			aZstd := instanceIsZstd(a)
+			bZstd := instanceIsZstd(b)
+			switch {
+			case aZstd == bZstd:
+				return 0
+			case !aZstd: // Implies bZstd
+				return -1
+			default: // aZstd && !bZstd
+				return 1
+			}
 		})
 	}
 	return nil

vendor/github.com/containers/image/v5/pkg/blobinfocache/internal/prioritize/prioritize.go

@@ -82,6 +82,7 @@ func (css *candidateSortState) Swap(i, j int) {
 func destructivelyPrioritizeReplacementCandidatesWithMax(cs []CandidateWithTime, primaryDigest, uncompressedDigest digest.Digest, maxCandidates int) []blobinfocache.BICReplacementCandidate2 {
 	// We don't need to use sort.Stable() because nanosecond timestamps are (presumably?) unique, so no two elements should
 	// compare equal.
+	// FIXME: Use slices.SortFunc after we update to Go 1.20 (Go 1.21?) and Time.Compare and cmp.Compare are available.
 	sort.Sort(&candidateSortState{
 		cs:                 cs,
 		primaryDigest:      primaryDigest,

vendor/github.com/containers/image/v5/storage/storage_dest.go

@@ -57,7 +57,7 @@ type storageImageDestination struct {
 
 	imageRef        storageReference
 	directory       string                   // Temporary directory where we store blobs until Commit() time
-	nextTempFileID  int32                    // A counter that we use for computing filenames to assign to blobs
+	nextTempFileID  atomic.Int32             // A counter that we use for computing filenames to assign to blobs
 	manifest        []byte                   // Manifest contents, temporary
 	manifestDigest  digest.Digest            // Valid if len(manifest) != 0
 	signatures      []byte                   // Signature contents, temporary
@@ -154,7 +154,7 @@ func (s *storageImageDestination) Close() error {
 }
 
 func (s *storageImageDestination) computeNextBlobCacheFile() string {
-	return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1)))
+	return filepath.Join(s.directory, fmt.Sprintf("%d", s.nextTempFileID.Add(1)))
 }
 
 // PutBlobWithOptions writes contents of stream and returns data representing the result.
@@ -763,7 +763,7 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
 	if len(layerBlobs) > 0 { // Can happen when using caches
 		prev := s.indexToStorageID[len(layerBlobs)-1]
 		if prev == nil {
-			return fmt.Errorf("Internal error: StorageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1)
+			return fmt.Errorf("Internal error: storageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1)
 		}
 		lastLayer = *prev
 	}
@@ -775,6 +775,78 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
 		logrus.Debugf("setting image creation date to %s", inspect.Created)
 		options.CreationDate = *inspect.Created
 	}
+
+	// Set up to save the non-layer blobs as data items.  Since we only share layers, they should all be in files, so
+	// we just need to screen out the ones that are actually layers to get the list of non-layers.
+	dataBlobs := set.New[digest.Digest]()
+	for blob := range s.filenames {
+		dataBlobs.Add(blob)
+	}
+	for _, layerBlob := range layerBlobs {
+		dataBlobs.Delete(layerBlob.Digest)
+	}
+	for _, blob := range dataBlobs.Values() {
+		v, err := os.ReadFile(s.filenames[blob])
+		if err != nil {
+			return fmt.Errorf("copying non-layer blob %q to image: %w", blob, err)
+		}
+		options.BigData = append(options.BigData, storage.ImageBigDataOption{
+			Key:    blob.String(),
+			Data:   v,
+			Digest: digest.Canonical.FromBytes(v),
+		})
+	}
+	// Set up to save the unparsedToplevel's manifest if it differs from
+	// the per-platform one, which is saved below.
+	if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) {
+		manifestDigest, err := manifest.Digest(toplevelManifest)
+		if err != nil {
+			return fmt.Errorf("digesting top-level manifest: %w", err)
+		}
+		options.BigData = append(options.BigData, storage.ImageBigDataOption{
+			Key:    manifestBigDataKey(manifestDigest),
+			Data:   toplevelManifest,
+			Digest: manifestDigest,
+		})
+	}
+	// Set up to save the image's manifest.  Allow looking it up by digest by using the key convention defined by the Store.
+	// Record the manifest twice: using a digest-specific key to allow references to that specific digest instance,
+	// and using storage.ImageDigestBigDataKey for future users that don’t specify any digest and for compatibility with older readers.
+	options.BigData = append(options.BigData, storage.ImageBigDataOption{
+		Key:    manifestBigDataKey(s.manifestDigest),
+		Data:   s.manifest,
+		Digest: s.manifestDigest,
+	})
+	options.BigData = append(options.BigData, storage.ImageBigDataOption{
+		Key:    storage.ImageDigestBigDataKey,
+		Data:   s.manifest,
+		Digest: s.manifestDigest,
+	})
+	// Set up to save the signatures, if we have any.
+	if len(s.signatures) > 0 {
+		options.BigData = append(options.BigData, storage.ImageBigDataOption{
+			Key:    "signatures",
+			Data:   s.signatures,
+			Digest: digest.Canonical.FromBytes(s.signatures),
+		})
+	}
+	for instanceDigest, signatures := range s.signatureses {
+		options.BigData = append(options.BigData, storage.ImageBigDataOption{
+			Key:    signatureBigDataKey(instanceDigest),
+			Data:   signatures,
+			Digest: digest.Canonical.FromBytes(signatures),
+		})
+	}
+
+	// Set up to save our metadata.
+	metadata, err := json.Marshal(s)
+	if err != nil {
+		return fmt.Errorf("encoding metadata for image: %w", err)
+	}
+	if len(metadata) != 0 {
+		options.Metadata = string(metadata)
+	}
+
 	// Create the image record, pointing to the most-recently added layer.
 	intendedID := s.imageRef.id
 	if intendedID == "" {
@@ -797,8 +869,26 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
 		}
 		logrus.Debugf("reusing image ID %q", img.ID)
 		oldNames = append(oldNames, img.Names...)
+		// set the data items and metadata on the already-present image
+		// FIXME: this _replaces_ any "signatures" blobs and their
+		// sizes (tracked in the metadata) which might have already
+		// been present with new values, when ideally we'd find a way
+		// to merge them since they all apply to the same image
+		for _, data := range options.BigData {
+			if err := s.imageRef.transport.store.SetImageBigData(img.ID, data.Key, data.Data, manifest.Digest); err != nil {
+				logrus.Debugf("error saving big data %q for image %q: %v", data.Key, img.ID, err)
+				return fmt.Errorf("saving big data %q for image %q: %w", data.Key, img.ID, err)
+			}
+		}
+		if options.Metadata != "" {
+			if err := s.imageRef.transport.store.SetMetadata(img.ID, options.Metadata); err != nil {
+				logrus.Debugf("error saving metadata for image %q: %v", img.ID, err)
+				return fmt.Errorf("saving metadata for image %q: %w", img.ID, err)
+			}
+			logrus.Debugf("saved image metadata %q", options.Metadata)
+		}
 	} else {
-		logrus.Debugf("created new image ID %q", img.ID)
+		logrus.Debugf("created new image ID %q with metadata %q", img.ID, options.Metadata)
 	}
 
 	// Clean up the unfinished image on any error.
@@ -813,78 +903,7 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
 		}
 	}()
 
-	// Add the non-layer blobs as data items.  Since we only share layers, they should all be in files, so
-	// we just need to screen out the ones that are actually layers to get the list of non-layers.
-	dataBlobs := set.New[digest.Digest]()
-	for blob := range s.filenames {
-		dataBlobs.Add(blob)
-	}
-	for _, layerBlob := range layerBlobs {
-		dataBlobs.Delete(layerBlob.Digest)
-	}
-	for _, blob := range dataBlobs.Values() {
-		v, err := os.ReadFile(s.filenames[blob])
-		if err != nil {
-			return fmt.Errorf("copying non-layer blob %q to image: %w", blob, err)
-		}
-		if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v, manifest.Digest); err != nil {
-			logrus.Debugf("error saving big data %q for image %q: %v", blob.String(), img.ID, err)
-			return fmt.Errorf("saving big data %q for image %q: %w", blob.String(), img.ID, err)
-		}
-	}
-	// Save the unparsedToplevel's manifest if it differs from the per-platform one, which is saved below.
-	if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) {
-		manifestDigest, err := manifest.Digest(toplevelManifest)
-		if err != nil {
-			return fmt.Errorf("digesting top-level manifest: %w", err)
-		}
-		key := manifestBigDataKey(manifestDigest)
-		if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, toplevelManifest, manifest.Digest); err != nil {
-			logrus.Debugf("error saving top-level manifest for image %q: %v", img.ID, err)
-			return fmt.Errorf("saving top-level manifest for image %q: %w", img.ID, err)
-		}
-	}
-	// Save the image's manifest.  Allow looking it up by digest by using the key convention defined by the Store.
-	// Record the manifest twice: using a digest-specific key to allow references to that specific digest instance,
-	// and using storage.ImageDigestBigDataKey for future users that don’t specify any digest and for compatibility with older readers.
-	key := manifestBigDataKey(s.manifestDigest)
-	if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil {
-		logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
-		return fmt.Errorf("saving manifest for image %q: %w", img.ID, err)
-	}
-	key = storage.ImageDigestBigDataKey
-	if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil {
-		logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
-		return fmt.Errorf("saving manifest for image %q: %w", img.ID, err)
-	}
-	// Save the signatures, if we have any.
-	if len(s.signatures) > 0 {
-		if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures, manifest.Digest); err != nil {
-			logrus.Debugf("error saving signatures for image %q: %v", img.ID, err)
-			return fmt.Errorf("saving signatures for image %q: %w", img.ID, err)
-		}
-	}
-	for instanceDigest, signatures := range s.signatureses {
-		key := signatureBigDataKey(instanceDigest)
-		if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, signatures, manifest.Digest); err != nil {
-			logrus.Debugf("error saving signatures for image %q: %v", img.ID, err)
-			return fmt.Errorf("saving signatures for image %q: %w", img.ID, err)
-		}
-	}
-	// Save our metadata.
-	metadata, err := json.Marshal(s)
-	if err != nil {
-		logrus.Debugf("error encoding metadata for image %q: %v", img.ID, err)
-		return fmt.Errorf("encoding metadata for image %q: %w", img.ID, err)
-	}
-	if len(metadata) != 0 {
-		if err = s.imageRef.transport.store.SetMetadata(img.ID, string(metadata)); err != nil {
-			logrus.Debugf("error saving metadata for image %q: %v", img.ID, err)
-			return fmt.Errorf("saving metadata for image %q: %w", img.ID, err)
-		}
-		logrus.Debugf("saved image metadata %q", string(metadata))
-	}
-	// Adds the reference's name on the image.  We don't need to worry about avoiding duplicate
+	// Add the reference's name on the image.  We don't need to worry about avoiding duplicate
 	// values because AddNames() will deduplicate the list that we pass to it.
 	if name := s.imageRef.DockerReference(); name != nil {
 		if err := s.imageRef.transport.store.AddNames(img.ID, []string{name.String()}); err != nil {
@@ -921,10 +940,7 @@ func (s *storageImageDestination) PutSignaturesWithFormat(ctx context.Context, s
 			return err
 		}
 		sizes = append(sizes, len(sig))
-		newblob := make([]byte, len(sigblob)+len(sig))
-		copy(newblob, sigblob)
-		copy(newblob[len(sigblob):], sig)
-		sigblob = newblob
+		sigblob = append(sigblob, sig...)
 	}
 	if instanceDigest == nil {
 		s.signatures = sigblob