podman image tree: restore previous behavior

The initial version of libimage changed the order of layers which has
now been restored to remain backwards compatible.

Further changes:

 * Fix a bug in the journald logging which requires to strip trailing
   new lines from the message.  The system tests did not pass due to
   empty new lines.  Triggered by changing the default logger to
   journald in containers/common.

 * Fix another bug in the journald logging which embedded the container
   ID inside the message rather than the specifid field.  That surfaced
   in a preceeding whitespace of each log line which broke the system
   tests.

 * Alter the system tests to make sure that the k8s-file and the
   journald logging drivers are executed.

 * A number of e2e tests have been changed to force the k8s-file driver
   to make them pass when running inside a root container.

 * Increase the timeout in a kill test which seems to take longer now.
   Reasons are unknown.  Tests passed earlier and no signal-related
   changes happend.  It may be CI VM flake since some system tests but
   other flaked.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>
This commit is contained in:
Valentin Rothberg
2021-05-05 17:08:17 +02:00
parent 59dd357509
commit d32863bbb4
32 changed files with 278 additions and 123 deletions

View File

@ -1,14 +1,18 @@
package libimage
import "time"
import (
"time"
// EventType indicates the type of an event. Currrently, there is only one
"github.com/sirupsen/logrus"
)
// EventType indicates the type of an event. Currently, there is only one
// supported type for container image but we may add more (e.g., for manifest
// lists) in the future.
type EventType int
const (
// EventTypeUnknow is an unitialized EventType.
// EventTypeUnknown is an uninitialized EventType.
EventTypeUnknown EventType = iota
// EventTypeImagePull represents an image pull.
EventTypeImagePull
@ -26,7 +30,7 @@ const (
EventTypeImageUntag
// EventTypeImageMount represents an image being mounted.
EventTypeImageMount
// EventTypeImageUnmounted represents an image being unmounted.
// EventTypeImageUnmount represents an image being unmounted.
EventTypeImageUnmount
)
@ -41,3 +45,18 @@ type Event struct {
// Type of the event.
Type EventType
}
// writeEvent writes the specified event to the Runtime's event channel. The
// event is discarded if no event channel has been registered (yet).
func (r *Runtime) writeEvent(event *Event) {
select {
case r.eventChannel <- event:
// Done
case <-time.After(2 * time.Second):
// The Runtime's event channel has a buffer of size 100 which
// should be enough even under high load. However, we
// shouldn't block too long in case the buffer runs full (could
// be an honest user error or bug).
logrus.Warnf("Discarding libimage event which was not read within 2 seconds: %v", event)
}
}

View File

@ -277,6 +277,10 @@ func (i *Image) remove(ctx context.Context, rmMap map[string]*RemoveImageReport,
return errors.Errorf("cannot remove read-only image %q", i.ID())
}
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: referencedBy, Time: time.Now(), Type: EventTypeImageRemove})
}
// Check if already visisted this image.
report, exists := rmMap[i.ID()]
if exists {
@ -423,6 +427,9 @@ func (i *Image) Tag(name string) error {
}
logrus.Debugf("Tagging image %s with %q", i.ID(), ref.String())
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: name, Time: time.Now(), Type: EventTypeImageTag})
}
newNames := append(i.Names(), ref.String())
if err := i.runtime.store.SetNames(i.ID(), newNames); err != nil {
@ -454,6 +461,9 @@ func (i *Image) Untag(name string) error {
name = ref.String()
logrus.Debugf("Untagging %q from image %s", ref.String(), i.ID())
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: name, Time: time.Now(), Type: EventTypeImageUntag})
}
removedName := false
newNames := []string{}
@ -593,6 +603,10 @@ func (i *Image) RepoDigests() ([]string, error) {
// are directly passed down to the containers storage. Returns the fully
// evaluated path to the mount point.
func (i *Image) Mount(ctx context.Context, mountOptions []string, mountLabel string) (string, error) {
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: "", Time: time.Now(), Type: EventTypeImageMount})
}
mountPoint, err := i.runtime.store.MountImage(i.ID(), mountOptions, mountLabel)
if err != nil {
return "", err
@ -634,6 +648,9 @@ func (i *Image) Mountpoint() (string, error) {
// Unmount the image. Use force to ignore the reference counter and forcefully
// unmount.
func (i *Image) Unmount(force bool) error {
if i.runtime.eventChannel != nil {
i.runtime.writeEvent(&Event{ID: i.ID(), Name: "", Time: time.Now(), Type: EventTypeImageUnmount})
}
logrus.Debugf("Unmounted image %s", i.ID())
_, err := i.runtime.store.UnmountImage(i.ID(), force)
return err

View File

@ -35,36 +35,45 @@ func (i *Image) Tree(traverseChildren bool) (string, error) {
fmt.Fprintf(sb, "No Image Layers")
}
tree := gotree.New(sb.String())
layerTree, err := i.runtime.layerTree()
if err != nil {
return "", err
}
imageNode := layerTree.node(i.TopLayer())
// Traverse the entire tree down to all children.
if traverseChildren {
tree := gotree.New(sb.String())
if err := imageTreeTraverseChildren(imageNode, tree); err != nil {
return "", err
}
} else {
// Walk all layers of the image and assemlbe their data.
for parentNode := imageNode; parentNode != nil; parentNode = parentNode.parent {
if parentNode.layer == nil {
break // we're done
}
var tags string
repoTags, err := parentNode.repoTags()
if err != nil {
return "", err
}
if len(repoTags) > 0 {
tags = fmt.Sprintf(" Top Layer of: %s", repoTags)
}
tree.Add(fmt.Sprintf("ID: %s Size: %7v%s", parentNode.layer.ID[:12], units.HumanSizeWithPrecision(float64(parentNode.layer.UncompressedSize), 4), tags))
return tree.Print(), nil
}
// Walk all layers of the image and assemlbe their data. Note that the
// tree is constructed in reverse order to remain backwards compatible
// with Podman.
contents := []string{}
for parentNode := imageNode; parentNode != nil; parentNode = parentNode.parent {
if parentNode.layer == nil {
break // we're done
}
var tags string
repoTags, err := parentNode.repoTags()
if err != nil {
return "", err
}
if len(repoTags) > 0 {
tags = fmt.Sprintf(" Top Layer of: %s", repoTags)
}
content := fmt.Sprintf("ID: %s Size: %7v%s", parentNode.layer.ID[:12], units.HumanSizeWithPrecision(float64(parentNode.layer.UncompressedSize), 4), tags)
contents = append(contents, content)
}
contents = append(contents, sb.String())
tree := gotree.New(contents[len(contents)-1])
for i := len(contents) - 2; i >= 0; i-- {
tree.Add(contents[i])
}
return tree.Print(), nil
@ -80,14 +89,22 @@ func imageTreeTraverseChildren(node *layerNode, parent gotree.Tree) error {
tags = fmt.Sprintf(" Top Layer of: %s", repoTags)
}
newNode := parent.Add(fmt.Sprintf("ID: %s Size: %7v%s", node.layer.ID[:12], units.HumanSizeWithPrecision(float64(node.layer.UncompressedSize), 4), tags))
content := fmt.Sprintf("ID: %s Size: %7v%s", node.layer.ID[:12], units.HumanSizeWithPrecision(float64(node.layer.UncompressedSize), 4), tags)
if len(node.children) <= 1 {
newNode = parent
var newTree gotree.Tree
if node.parent == nil || len(node.parent.children) <= 1 {
// No parent or no siblings, so we can go linear.
parent.Add(content)
newTree = parent
} else {
// Each siblings gets a new tree, so we can branch.
newTree = gotree.New(content)
parent.AddTree(newTree)
}
for i := range node.children {
child := node.children[i]
if err := imageTreeTraverseChildren(child, newNode); err != nil {
if err := imageTreeTraverseChildren(child, newTree); err != nil {
return err
}
}

View File

@ -59,7 +59,7 @@ func (l *layerNode) repoTags() ([]string, error) {
return nil, err
}
for _, tag := range repoTags {
if _, visted := visitedTags[tag]; visted {
if _, visited := visitedTags[tag]; visited {
continue
}
visitedTags[tag] = true

View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"os"
"time"
dirTransport "github.com/containers/image/v5/directory"
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
@ -23,6 +24,10 @@ type LoadOptions struct {
func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) ([]string, error) {
logrus.Debugf("Loading image from %q", path)
if r.eventChannel != nil {
r.writeEvent(&Event{ID: "", Name: path, Time: time.Now(), Type: EventTypeImageLoad})
}
var (
loadedImages []string
loadError error

View File

@ -3,6 +3,7 @@ package libimage
import (
"context"
"fmt"
"time"
"github.com/containers/common/libimage/manifests"
imageCopy "github.com/containers/image/v5/copy"
@ -364,6 +365,10 @@ func (m *ManifestList) Push(ctx context.Context, destination string, options *Ma
}
}
if m.image.runtime.eventChannel != nil {
m.image.runtime.writeEvent(&Event{ID: m.ID(), Name: destination, Time: time.Now(), Type: EventTypeImagePush})
}
// NOTE: we're using the logic in copier to create a proper
// types.SystemContext. This prevents us from having an error prone
// code duplicate here.

View File

@ -5,10 +5,10 @@ import (
"fmt"
"io"
"strings"
"time"
"github.com/containers/common/pkg/config"
dirTransport "github.com/containers/image/v5/directory"
dockerTransport "github.com/containers/image/v5/docker"
registryTransport "github.com/containers/image/v5/docker"
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
"github.com/containers/image/v5/docker/reference"
ociArchiveTransport "github.com/containers/image/v5/oci/archive"
@ -42,7 +42,7 @@ type PullOptions struct {
// policies (e.g., buildah-bud versus podman-build). Making the pull-policy
// choice explicit is an attempt to prevent silent regressions.
//
// The errror is storage.ErrImageUnknown iff the pull policy is set to "never"
// The error is storage.ErrImageUnknown iff the pull policy is set to "never"
// and no local image has been found. This allows for an easier integration
// into some users of this package (e.g., Buildah).
func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullPolicy, options *PullOptions) ([]*Image, error) {
@ -56,7 +56,7 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
if err != nil {
// If the image clearly refers to a local one, we can look it up directly.
// In fact, we need to since they are not parseable.
if strings.HasPrefix(name, "sha256:") || (len(name) == 64 && !strings.Contains(name, "/.:@")) {
if strings.HasPrefix(name, "sha256:") || (len(name) == 64 && !strings.ContainsAny(name, "/.:@")) {
if pullPolicy == config.PullPolicyAlways {
return nil, errors.Errorf("pull policy is always but image has been referred to by ID (%s)", name)
}
@ -76,10 +76,14 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
ref = dockerRef
}
if options.AllTags && ref.Transport().Name() != dockerTransport.Transport.Name() {
if options.AllTags && ref.Transport().Name() != registryTransport.Transport.Name() {
return nil, errors.Errorf("pulling all tags is not supported for %s transport", ref.Transport().Name())
}
if r.eventChannel != nil {
r.writeEvent(&Event{ID: "", Name: name, Time: time.Now(), Type: EventTypeImagePull})
}
var (
pulledImages []string
pullError error
@ -88,29 +92,17 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP
// Dispatch the copy operation.
switch ref.Transport().Name() {
// DOCKER/REGISTRY
case dockerTransport.Transport.Name():
// DOCKER REGISTRY
case registryTransport.Transport.Name():
pulledImages, pullError = r.copyFromRegistry(ctx, ref, strings.TrimPrefix(name, "docker://"), pullPolicy, options)
// DOCKER ARCHIVE
case dockerArchiveTransport.Transport.Name():
pulledImages, pullError = r.copyFromDockerArchive(ctx, ref, &options.CopyOptions)
// OCI
case ociTransport.Transport.Name():
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
// OCI ARCHIVE
case ociArchiveTransport.Transport.Name():
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
// DIR
case dirTransport.Transport.Name():
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
// UNSUPPORTED
// ALL OTHER TRANSPORTS
default:
return nil, errors.Errorf("unsupported transport %q for pulling", ref.Transport().Name())
pulledImages, pullError = r.copyFromDefault(ctx, ref, &options.CopyOptions)
}
if pullError != nil {
@ -162,7 +154,12 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference,
imageName = "sha256:" + storageName[1:]
} else {
storageName = manifest.Annotations["org.opencontainers.image.ref.name"]
imageName = storageName
named, err := NormalizeName(storageName)
if err != nil {
return nil, err
}
imageName = named.String()
storageName = imageName
}
default:
@ -275,7 +272,7 @@ func (r *Runtime) copyFromRegistry(ctx context.Context, ref types.ImageReference
}
named := reference.TrimNamed(ref.DockerReference())
tags, err := dockerTransport.GetRepositoryTags(ctx, &r.systemContext, ref)
tags, err := registryTransport.GetRepositoryTags(ctx, &r.systemContext, ref)
if err != nil {
return nil, err
}
@ -399,7 +396,7 @@ func (r *Runtime) copySingleImageFromRegistry(ctx context.Context, imageName str
for _, candidate := range resolved.PullCandidates {
candidateString := candidate.Value.String()
logrus.Debugf("Attempting to pull candidate %s for %s", candidateString, imageName)
srcRef, err := dockerTransport.NewReference(candidate.Value)
srcRef, err := registryTransport.NewReference(candidate.Value)
if err != nil {
return nil, err
}

View File

@ -2,6 +2,7 @@ package libimage
import (
"context"
"time"
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
"github.com/containers/image/v5/docker/reference"
@ -61,8 +62,12 @@ func (r *Runtime) Push(ctx context.Context, source, destination string, options
destRef = dockerRef
}
if r.eventChannel != nil {
r.writeEvent(&Event{ID: image.ID(), Name: destination, Time: time.Now(), Type: EventTypeImagePush})
}
// Buildah compat: Make sure to tag the destination image if it's a
// Docker archive. This way, we preseve the image name.
// Docker archive. This way, we preserve the image name.
if destRef.Transport().Name() == dockerArchiveTransport.Transport.Name() {
if named, err := reference.ParseNamed(resolvedSource); err == nil {
tagged, isTagged := named.(reference.NamedTagged)

View File

@ -20,6 +20,9 @@ import (
// RuntimeOptions allow for creating a customized Runtime.
type RuntimeOptions struct {
// The base system context of the runtime which will be used throughout
// the entire lifespan of the Runtime. Certain options in some
// functions may override specific fields.
SystemContext *types.SystemContext
}
@ -41,6 +44,8 @@ func setRegistriesConfPath(systemContext *types.SystemContext) {
// Runtime is responsible for image management and storing them in a containers
// storage.
type Runtime struct {
// Use to send events out to users.
eventChannel chan *Event
// Underlying storage store.
store storage.Store
// Global system context. No pointer to simplify copying and modifying
@ -55,6 +60,18 @@ func (r *Runtime) systemContextCopy() *types.SystemContext {
return &sys
}
// EventChannel creates a buffered channel for events that the Runtime will use
// to write events to. Callers are expected to read from the channel in a
// timely manner.
// Can be called once for a given Runtime.
func (r *Runtime) EventChannel() chan *Event {
if r.eventChannel != nil {
return r.eventChannel
}
r.eventChannel = make(chan *Event, 100)
return r.eventChannel
}
// RuntimeFromStore returns a Runtime for the specified store.
func RuntimeFromStore(store storage.Store, options *RuntimeOptions) (*Runtime, error) {
if options == nil {
@ -99,6 +116,9 @@ func RuntimeFromStoreOptions(runtimeOptions *RuntimeOptions, storeOptions *stora
// is considered to be an error condition.
func (r *Runtime) Shutdown(force bool) error {
_, err := r.store.Shutdown(force)
if r.eventChannel != nil {
close(r.eventChannel)
}
return err
}

View File

@ -3,6 +3,7 @@ package libimage
import (
"context"
"strings"
"time"
dirTransport "github.com/containers/image/v5/directory"
dockerArchiveTransport "github.com/containers/image/v5/docker/archive"
@ -46,7 +47,7 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
// All formats support saving 1.
default:
if format != "docker-archive" {
return errors.Errorf("unspported format %q for saving multiple images (only docker-archive)", format)
return errors.Errorf("unsupported format %q for saving multiple images (only docker-archive)", format)
}
if len(options.AdditionalTags) > 0 {
return errors.Errorf("cannot save multiple images with multiple tags")
@ -62,7 +63,7 @@ func (r *Runtime) Save(ctx context.Context, names []string, format, path string,
return r.saveDockerArchive(ctx, names, path, options)
}
return errors.Errorf("unspported format %q for saving images", format)
return errors.Errorf("unsupported format %q for saving images", format)
}
@ -74,6 +75,10 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
return err
}
if r.eventChannel != nil {
r.writeEvent(&Event{ID: image.ID(), Name: path, Time: time.Now(), Type: EventTypeImageSave})
}
// Unless the image was referenced by ID, use the resolved name as a
// tag.
var tag string
@ -101,7 +106,7 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string
options.ManifestMIMEType = manifest.DockerV2Schema2MediaType
default:
return errors.Errorf("unspported format %q for saving images", format)
return errors.Errorf("unsupported format %q for saving images", format)
}
if err != nil {
@ -160,6 +165,9 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st
}
}
localImages[image.ID()] = local
if r.eventChannel != nil {
r.writeEvent(&Event{ID: image.ID(), Name: path, Time: time.Now(), Type: EventTypeImageSave})
}
}
writer, err := dockerArchiveTransport.NewWriter(r.systemContextCopy(), path)

View File

@ -7,7 +7,7 @@ import (
"strings"
"sync"
dockerTransport "github.com/containers/image/v5/docker"
registryTransport "github.com/containers/image/v5/docker"
"github.com/containers/image/v5/pkg/sysregistriesv2"
"github.com/containers/image/v5/transports/alltransports"
"github.com/containers/image/v5/types"
@ -193,7 +193,7 @@ func (r *Runtime) searchImageInRegistry(ctx context.Context, term, registry stri
return results, nil
}
results, err := dockerTransport.SearchRegistry(ctx, sys, registry, term, limit)
results, err := registryTransport.SearchRegistry(ctx, sys, registry, term, limit)
if err != nil {
return []SearchResult{}, err
}
@ -255,7 +255,7 @@ func (r *Runtime) searchImageInRegistry(ctx context.Context, term, registry stri
func searchRepositoryTags(ctx context.Context, sys *types.SystemContext, registry, term string, options *SearchOptions) ([]SearchResult, error) {
dockerPrefix := "docker://"
imageRef, err := alltransports.ParseImageName(fmt.Sprintf("%s/%s", registry, term))
if err == nil && imageRef.Transport().Name() != dockerTransport.Transport.Name() {
if err == nil && imageRef.Transport().Name() != registryTransport.Transport.Name() {
return nil, errors.Errorf("reference %q must be a docker reference", term)
} else if err != nil {
imageRef, err = alltransports.ParseImageName(fmt.Sprintf("%s%s", dockerPrefix, fmt.Sprintf("%s/%s", registry, term)))
@ -263,7 +263,7 @@ func searchRepositoryTags(ctx context.Context, sys *types.SystemContext, registr
return nil, errors.Errorf("reference %q must be a docker reference", term)
}
}
tags, err := dockerTransport.GetRepositoryTags(ctx, sys, imageRef)
tags, err := registryTransport.GetRepositoryTags(ctx, sys, imageRef)
if err != nil {
return nil, errors.Errorf("error getting repository tags: %v", err)
}
@ -288,18 +288,18 @@ func searchRepositoryTags(ctx context.Context, sys *types.SystemContext, registr
return paramsArr, nil
}
func (f *SearchFilter) matchesStarFilter(result dockerTransport.SearchResult) bool {
func (f *SearchFilter) matchesStarFilter(result registryTransport.SearchResult) bool {
return result.StarCount >= f.Stars
}
func (f *SearchFilter) matchesAutomatedFilter(result dockerTransport.SearchResult) bool {
func (f *SearchFilter) matchesAutomatedFilter(result registryTransport.SearchResult) bool {
if f.IsAutomated != types.OptionalBoolUndefined {
return result.IsAutomated == (f.IsAutomated == types.OptionalBoolTrue)
}
return true
}
func (f *SearchFilter) matchesOfficialFilter(result dockerTransport.SearchResult) bool {
func (f *SearchFilter) matchesOfficialFilter(result registryTransport.SearchResult) bool {
if f.IsOfficial != types.OptionalBoolUndefined {
return result.IsOfficial == (f.IsOfficial == types.OptionalBoolTrue)
}