From 4929e37507b530cb1d0e92cfcd252b0abefb4f2f Mon Sep 17 00:00:00 2001 From: baude Date: Mon, 19 Feb 2018 08:40:43 -0600 Subject: [PATCH] Performance enhancement for podman images Previous code was using slow routines to collect some of the information needed to output images. Specifically size was being calculated instead of using the cached, already known size already available. Also, straight- lined several of the code paths. Overall assessment is that these improvements cut the time for images in half. Signed-off-by: baude Closes: #365 Approved by: mheon --- cmd/podman/images.go | 335 +++++++++++++++++----------------------- libpod/runtime_img.go | 178 +++++++++++++++++++++ pkg/inspect/inspect.go | 15 ++ test/e2e/images_test.go | 57 +++++++ 4 files changed, 391 insertions(+), 194 deletions(-) diff --git a/cmd/podman/images.go b/cmd/podman/images.go index 3b41204f18..2dcd743cf7 100644 --- a/cmd/podman/images.go +++ b/cmd/podman/images.go @@ -1,18 +1,15 @@ package main import ( - "fmt" "reflect" "strings" "time" - "github.com/containers/storage" "github.com/docker/go-units" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/projectatomic/libpod/cmd/podman/formats" "github.com/projectatomic/libpod/libpod" - "github.com/projectatomic/libpod/libpod/common" "github.com/projectatomic/libpod/pkg/inspect" "github.com/urfave/cli" ) @@ -31,15 +28,16 @@ type imagesJSONParams struct { Name []string `json:"names"` Digest digest.Digest `json:"digest"` Created time.Time `json:"created"` - Size int64 `json:"size"` + Size *uint64 `json:"size"` } type imagesOptions struct { - quiet bool - noHeading bool - noTrunc bool - digests bool - format string + quiet bool + noHeading bool + noTrunc bool + digests bool + format string + outputformat string } var ( @@ -64,7 +62,7 @@ var ( Name: "format", Usage: "Change the output format to JSON or a Go template", }, - cli.StringFlag{ + cli.StringSliceFlag{ Name: "filter, f", Usage: "filter output based on conditions provided (default [])", }, @@ -92,38 +90,32 @@ func imagesCmd(c *cli.Context) error { return errors.Wrapf(err, "Could not get runtime") } defer runtime.Shutdown(false) + var filterFuncs []libpod.ImageResultFilter + var imageInput string + if len(c.Args()) == 1 { + imageInput = c.Args().Get(0) + } - format := genImagesFormat(c.String("format"), c.Bool("quiet"), c.Bool("noheading"), c.Bool("digests")) + if len(c.Args()) > 1 { + return errors.New("'podman images' requires at most 1 argument") + } + + if len(c.StringSlice("filter")) > 0 || len(strings.TrimSpace(imageInput)) != 0 { + filterFuncs, err = CreateFilterFuncs(runtime, c, imageInput) + if err != nil { + return err + } + } opts := imagesOptions{ quiet: c.Bool("quiet"), noHeading: c.Bool("noheading"), noTrunc: c.Bool("no-trunc"), digests: c.Bool("digests"), - format: format, + format: c.String("format"), } - var imageInput string - if len(c.Args()) == 1 { - imageInput = c.Args().Get(0) - } - if len(c.Args()) > 1 { - return errors.New("'podman images' requires at most 1 argument") - } - - params, err := runtime.ParseImageFilter(imageInput, c.String("filter")) - if err != nil { - return errors.Wrapf(err, "error parsing filter") - } - - // generate the different filters - labelFilter := generateImagesFilter(params, "label") - beforeImageFilter := generateImagesFilter(params, "before-image") - sinceImageFilter := generateImagesFilter(params, "since-image") - danglingFilter := generateImagesFilter(params, "dangling") - referenceFilter := generateImagesFilter(params, "reference") - imageInputFilter := generateImagesFilter(params, "image-input") - + opts.outputformat = opts.setOutputFormat() /* podman does not implement --all for images @@ -131,29 +123,36 @@ func imagesCmd(c *cli.Context) error { children to the image once built. until buildah supports caching builds, it will not generate these intermediate images. */ - - images, err := runtime.GetImages(params, labelFilter, beforeImageFilter, sinceImageFilter, danglingFilter, referenceFilter, imageInputFilter) + images, err := runtime.GetImageResults() if err != nil { - return errors.Wrapf(err, "could not get list of images matching filter") + return errors.Wrapf(err, "unable to get images") } - return generateImagesOutput(runtime, images, opts) + var filteredImages []inspect.ImageResult + // filter the images + if len(c.StringSlice("filter")) > 0 || len(strings.TrimSpace(imageInput)) != 0 { + filteredImages = libpod.FilterImages(images, filterFuncs) + } else { + filteredImages = images + } + + return generateImagesOutput(runtime, filteredImages, opts) } -func genImagesFormat(format string, quiet, noHeading, digests bool) string { - if format != "" { +func (i imagesOptions) setOutputFormat() string { + if i.format != "" { // "\t" from the command line is not being recognized as a tab // replacing the string "\t" to a tab character if the user passes in "\t" - return strings.Replace(format, `\t`, "\t", -1) + return strings.Replace(i.format, `\t`, "\t", -1) } - if quiet { + if i.quiet { return formats.IDString } - format = "table {{.Repository}}\t{{.Tag}}\t" - if noHeading { + format := "table {{.Repository}}\t{{.Tag}}\t" + if i.noHeading { format = "{{.Repository}}\t{{.Tag}}\t" } - if digests { + if i.digests { format += "{{.Digest}}\t" } format += "{{.ID}}\t{{.Created}}\t{{.Size}}\t" @@ -174,8 +173,65 @@ func imagesToGeneric(templParams []imagesTemplateParams, JSONParams []imagesJSON return } -// generate the header based on the template provided -func (i *imagesTemplateParams) headerMap() map[string]string { +// getImagesTemplateOutput returns the images information to be printed in human readable format +func getImagesTemplateOutput(runtime *libpod.Runtime, images []inspect.ImageResult, opts imagesOptions) (imagesOutput []imagesTemplateParams) { + for _, img := range images { + createdTime := img.Created + + imageID := "sha256:" + img.ID + if !opts.noTrunc { + imageID = shortID(img.ID) + } + params := imagesTemplateParams{ + Repository: img.Repository, + Tag: img.Tag, + ID: imageID, + Digest: img.Digest, + Created: units.HumanDuration(time.Since((createdTime))) + " ago", + Size: units.HumanSizeWithPrecision(float64(*img.Size), 3), + } + imagesOutput = append(imagesOutput, params) + } + return +} + +// getImagesJSONOutput returns the images information in its raw form +func getImagesJSONOutput(runtime *libpod.Runtime, images []inspect.ImageResult) (imagesOutput []imagesJSONParams) { + for _, img := range images { + params := imagesJSONParams{ + ID: img.ID, + Name: img.RepoTags, + Digest: img.Digest, + Created: img.Created, + Size: img.Size, + } + imagesOutput = append(imagesOutput, params) + } + return +} + +// generateImagesOutput generates the images based on the format provided + +func generateImagesOutput(runtime *libpod.Runtime, images []inspect.ImageResult, opts imagesOptions) error { + if len(images) == 0 { + return nil + } + var out formats.Writer + + switch opts.format { + case formats.JSONString: + imagesOutput := getImagesJSONOutput(runtime, images) + out = formats.JSONStructArray{Output: imagesToGeneric([]imagesTemplateParams{}, imagesOutput)} + default: + imagesOutput := getImagesTemplateOutput(runtime, images, opts) + out = formats.StdoutTemplateArray{Output: imagesToGeneric(imagesOutput, []imagesJSONParams{}), Template: opts.outputformat, Fields: imagesOutput[0].HeaderMap()} + } + return formats.Writer(out).Out() +} + +// HeaderMap produces a generic map of "headers" based on a line +// of output +func (i *imagesTemplateParams) HeaderMap() map[string]string { v := reflect.Indirect(reflect.ValueOf(i)) values := make(map[string]string) @@ -190,157 +246,48 @@ func (i *imagesTemplateParams) headerMap() map[string]string { return values } -// getImagesTemplateOutput returns the images information to be printed in human readable format -func getImagesTemplateOutput(runtime *libpod.Runtime, images []*storage.Image, opts imagesOptions) (imagesOutput []imagesTemplateParams) { - var ( - lastID string - ) - for _, img := range images { - if opts.quiet && lastID == img.ID { - continue // quiet should not show the same ID multiple times - } - createdTime := img.Created +// CreateFilterFuncs returns an array of filter functions based on the user inputs +// and is later used to filter images for output +func CreateFilterFuncs(r *libpod.Runtime, c *cli.Context, userInput string) ([]libpod.ImageResultFilter, error) { + var filterFuncs []libpod.ImageResultFilter + for _, filter := range c.StringSlice("filter") { + splitFilter := strings.Split(filter, "=") + switch splitFilter[0] { + case "before": + before := r.NewImage(splitFilter[1]) + _, beforeID, _ := before.GetLocalImageName() - imageID := "sha256:" + img.ID - if !opts.noTrunc { - imageID = shortID(img.ID) - } - - repository := "" - tag := "" - if len(img.Names) > 0 { - arr := strings.Split(img.Names[0], ":") - repository = arr[0] - if len(arr) == 2 { - tag = arr[1] + if before.LocalName == "" { + return nil, errors.Errorf("unable to find image % in local stores", splitFilter[1]) } - } + img, err := r.GetImage(beforeID) + if err != nil { + return nil, err + } + filterFuncs = append(filterFuncs, libpod.ImageCreatedBefore(img.Created)) + case "after": + after := r.NewImage(splitFilter[1]) + _, afterID, _ := after.GetLocalImageName() - imgData, _ := runtime.GetImageInspectInfo(*img) - if imgData != nil { - createdTime = *imgData.Created + if after.LocalName == "" { + return nil, errors.Errorf("unable to find image % in local stores", splitFilter[1]) + } + img, err := r.GetImage(afterID) + if err != nil { + return nil, err + } + filterFuncs = append(filterFuncs, libpod.ImageCreatedAfter(img.Created)) + case "dangling": + filterFuncs = append(filterFuncs, libpod.ImageDangling()) + case "label": + labelFilter := strings.Join(splitFilter[1:], "=") + filterFuncs = append(filterFuncs, libpod.ImageLabel(labelFilter)) + default: + return nil, errors.Errorf("invalid filter %s ", splitFilter[0]) } - - params := imagesTemplateParams{ - Repository: repository, - Tag: tag, - ID: imageID, - Digest: imgData.Digest, - Created: units.HumanDuration(time.Since((createdTime))) + " ago", - Size: units.HumanSizeWithPrecision(float64(imgData.Size), 3), - } - imagesOutput = append(imagesOutput, params) - } - return -} - -// getImagesJSONOutput returns the images information in its raw form -func getImagesJSONOutput(runtime *libpod.Runtime, images []*storage.Image) (imagesOutput []imagesJSONParams) { - for _, img := range images { - createdTime := img.Created - - imgData, _ := runtime.GetImageInspectInfo(*img) - if imgData != nil { - createdTime = *imgData.Created - } - - params := imagesJSONParams{ - ID: img.ID, - Name: img.Names, - Digest: imgData.Digest, - Created: createdTime, - Size: imgData.Size, - } - imagesOutput = append(imagesOutput, params) - } - return -} - -// generateImagesOutput generates the images based on the format provided -func generateImagesOutput(runtime *libpod.Runtime, images []*storage.Image, opts imagesOptions) error { - if len(images) == 0 { - return nil - } - - var out formats.Writer - - switch opts.format { - case formats.JSONString: - imagesOutput := getImagesJSONOutput(runtime, images) - out = formats.JSONStructArray{Output: imagesToGeneric([]imagesTemplateParams{}, imagesOutput)} - default: - imagesOutput := getImagesTemplateOutput(runtime, images, opts) - out = formats.StdoutTemplateArray{Output: imagesToGeneric(imagesOutput, []imagesJSONParams{}), Template: opts.format, Fields: imagesOutput[0].headerMap()} - - } - - return formats.Writer(out).Out() -} - -// generateImagesFilter returns an ImageFilter based on filterType -// to add more filters, define a new case and write what the ImageFilter function should do -func generateImagesFilter(params *libpod.ImageFilterParams, filterType string) libpod.ImageFilter { - switch filterType { - case "label": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.Label == "" { - return true - } - - pair := strings.SplitN(params.Label, "=", 2) - if val, ok := info.Labels[pair[0]]; ok { - if len(pair) == 2 && val == pair[1] { - return true - } - if len(pair) == 1 { - return true - } - } - return false - } - case "before-image": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.BeforeImage.IsZero() { - return true - } - return info.Created.Before(params.BeforeImage) - } - case "since-image": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.SinceImage.IsZero() { - return true - } - return info.Created.After(params.SinceImage) - } - case "dangling": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.Dangling == "" { - return true - } - if common.IsFalse(params.Dangling) && params.ImageName != "" { - return true - } - if common.IsTrue(params.Dangling) && params.ImageName == "" { - return true - } - return false - } - case "reference": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.ReferencePattern == "" { - return true - } - return libpod.MatchesReference(params.ImageName, params.ReferencePattern) - } - case "image-input": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.ImageInput == "" { - return true - } - return libpod.MatchesReference(params.ImageName, params.ImageInput) - } - default: - fmt.Println("invalid filter type", filterType) - return nil } + if len(strings.TrimSpace(userInput)) != 0 { + filterFuncs = append(filterFuncs, libpod.OutputImageFilter(userInput)) + } + return filterFuncs, nil } diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go index 76687351d1..cc62754947 100644 --- a/libpod/runtime_img.go +++ b/libpod/runtime_img.go @@ -108,6 +108,9 @@ type imageDecomposeStruct struct { transport string } +// ImageResultFilter is a mock function for image filtering +type ImageResultFilter func(inspect.ImageResult) bool + func (k *Image) assembleFqName() string { return fmt.Sprintf("%s/%s:%s", k.Registry, k.ImageName, k.Tag) } @@ -1262,3 +1265,178 @@ func getPolicyContext(ctx *types.SystemContext) (*signature.PolicyContext, error } return policyContext, nil } + +// sizer knows its size. +type sizer interface { + Size() (int64, error) +} + +func imageSize(img types.ImageSource) *uint64 { + if s, ok := img.(sizer); ok { + if sum, err := s.Size(); err == nil { + usum := uint64(sum) + return &usum + } + } + return nil +} + +func reposToMap(repotags []string) map[string]string { + // map format is repo -> tag + repos := make(map[string]string) + for _, repo := range repotags { + var repository, tag string + if len(repo) > 0 { + li := strings.LastIndex(repo, ":") + repository = repo[0:li] + tag = repo[li+1:] + } + repos[repository] = tag + } + if len(repos) == 0 { + repos[""] = "", 2)) + Expect(session.LineInOuputStartsWith("docker.io/library/alpine")).To(BeTrue()) + Expect(session.LineInOuputStartsWith("docker.io/library/busybox")).To(BeTrue()) + }) + It("podman images in JSON format", func() { session := podmanTest.Podman([]string{"images", "--format=json"}) session.WaitWithDefaultTimeout() @@ -44,10 +53,58 @@ var _ = Describe("Podman images", func() { Expect(session.IsJSONOutputValid()).To(BeTrue()) }) + It("podman images in GO template format", func() { + session := podmanTest.Podman([]string{"images", "--format={{.ID}}"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + }) + It("podman images with short options", func() { session := podmanTest.Podman([]string{"images", "-qn"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(BeNumerically(">", 1)) }) + + It("podman images filter by image name", func() { + session := podmanTest.Podman([]string{"images", "-q", ALPINE}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(1)) + }) + + It("podman images filter before image", func() { + dockerfile := `FROM docker.io/library/alpine:latest +` + podmanTest.BuildImage(dockerfile, "foobar.com/before:latest") + result := podmanTest.Podman([]string{"images", "-q", "-f", "before=foobar.com/before:latest"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(len(result.OutputToStringArray())).To(Equal(2)) + }) + + It("podman images filter after image", func() { + rmi := podmanTest.Podman([]string{"rmi", "busybox"}) + rmi.WaitWithDefaultTimeout() + Expect(rmi.ExitCode()).To(Equal(0)) + + dockerfile := `FROM docker.io/library/alpine:latest +` + podmanTest.BuildImage(dockerfile, "foobar.com/before:latest") + result := podmanTest.Podman([]string{"images", "-q", "-f", "after=docker.io/library/alpine:latest"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(len(result.OutputToStringArray())).To(Equal(1)) + }) + + It("podman images filter dangling", func() { + dockerfile := `FROM docker.io/library/alpine:latest +` + podmanTest.BuildImage(dockerfile, "foobar.com/before:latest") + podmanTest.BuildImage(dockerfile, "foobar.com/before:latest") + result := podmanTest.Podman([]string{"images", "-q", "-f", "dangling=true"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(len(result.OutputToStringArray())).To(Equal(1)) + }) })