benchmarks: add more image benchmarks

Add more benchmarks for the most common and performance-critical image
commands.  Benchmarks for `podman build` should go into a separate
section.

Signed-off-by: Valentin Rothberg <vrothberg@redhat.com>

Makefile

@@ -134,7 +134,7 @@ ifeq ($(GOBIN),)
 GOBIN := $(FIRST_GOPATH)/bin
 endif
 
-export PATH := $(PATH):$(GOBIN)
+export PATH := $(PATH):$(GOBIN):$(CURDIR)/hack
 
 GOMD2MAN ?= $(shell command -v go-md2man || echo '$(GOBIN)/go-md2man')
 

hack/podman-registry-go/registry.go

@@ -9,10 +9,10 @@ import (
 )
 
 const (
-	imageKey = "PODMAN_REGISTRY_IMAGE"
+	ImageKey = "PODMAN_REGISTRY_IMAGE"
-	userKey  = "PODMAN_REGISTRY_USER"
+	UserKey  = "PODMAN_REGISTRY_USER"
-	passKey  = "PODMAN_REGISTRY_PASS"
+	PassKey  = "PODMAN_REGISTRY_PASS"
-	portKey  = "PODMAN_REGISTRY_PORT"
+	PortKey  = "PODMAN_REGISTRY_PORT"
 )
 
 var binary = "podman-registry"
@@ -52,13 +52,13 @@ func Start() (*Registry, error) {
 		key := spl[0]
 		val := strings.TrimSuffix(strings.TrimPrefix(spl[1], "\""), "\"")
 		switch key {
-		case imageKey:
+		case ImageKey:
 			registry.Image = val
-		case userKey:
+		case UserKey:
 			registry.User = val
-		case passKey:
+		case PassKey:
 			registry.Password = val
-		case portKey:
+		case PortKey:
 			registry.Port = val
 		default:
 			logrus.Errorf("Unexpected podman-registry output: %q", s)
@@ -67,16 +67,16 @@ func Start() (*Registry, error) {
 
 	// Extra sanity check.
 	if registry.Image == "" {
-		return nil, errors.Errorf("unexpected output %q: %q missing", out, imageKey)
+		return nil, errors.Errorf("unexpected output %q: %q missing", out, ImageKey)
 	}
 	if registry.User == "" {
-		return nil, errors.Errorf("unexpected output %q: %q missing", out, userKey)
+		return nil, errors.Errorf("unexpected output %q: %q missing", out, UserKey)
 	}
 	if registry.Password == "" {
-		return nil, errors.Errorf("unexpected output %q: %q missing", out, passKey)
+		return nil, errors.Errorf("unexpected output %q: %q missing", out, PassKey)
 	}
 	if registry.Port == "" {
-		return nil, errors.Errorf("unexpected output %q: %q missing", out, portKey)
+		return nil, errors.Errorf("unexpected output %q: %q missing", out, PortKey)
 	}
 
 	registry.running = true

test/e2e/benchmarks_test.go

@@ -11,10 +11,12 @@ import (
 	"strconv"
 	"strings"
 
+	podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go"
 	. "github.com/containers/podman/v4/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -30,21 +32,23 @@ type benchmark struct {
 	name string
 	// The function to execute.
 	main func()
-	// Function is run before `main`.
+	// Allows for extending a benchmark.
-	init func()
+	options newBenchmarkOptions
 }
 
 // Allows for customizing the benchnmark in an easy to extend way.
 type newBenchmarkOptions struct {
 	// Sets the benchmark's init function.
 	init func()
+	// Run a local registry for this benchmark.
+	needsRegistry bool
 }
 
 // Queue a new benchmark.
 func newBenchmark(name string, main func(), options *newBenchmarkOptions) {
 	bm := benchmark{name: name, main: main}
 	if options != nil {
-		bm.init = options.init
+		bm.options = *options
 	}
 	allBenchmarks = append(allBenchmarks, bm)
 }
@@ -109,8 +113,23 @@ var _ = Describe("Podman Benchmark Suite", func() {
 		for i := range allBenchmarks {
 			setup()
 			bm := allBenchmarks[i]
-			if bm.init != nil {
+
-				bm.init()
+			// Start a local registry if requested.
+			var registry *podmanRegistry.Registry
+			if bm.options.needsRegistry {
+				reg, err := podmanRegistry.Start()
+				if err != nil {
+					logrus.Errorf("Error starting registry: %v", err)
+					os.Exit(1)
+				}
+				registry = reg
+				os.Setenv(podmanRegistry.UserKey, reg.User)
+				os.Setenv(podmanRegistry.PassKey, reg.Password)
+				os.Setenv(podmanRegistry.PortKey, reg.Port)
+			}
+
+			if bm.options.init != nil {
+				bm.options.init()
 			}
 
 			// Set the time dir only for the main() function.
@@ -120,6 +139,18 @@ var _ = Describe("Podman Benchmark Suite", func() {
 
 			mem := totalMemoryInKb()
 			b.RecordValueWithPrecision("[MEM] "+bm.name, float64(mem), "KB", 1)
+
+			// Stop the local registry.
+			if bm.options.needsRegistry {
+				os.Unsetenv(podmanRegistry.UserKey)
+				os.Unsetenv(podmanRegistry.PassKey)
+				os.Unsetenv(podmanRegistry.PortKey)
+				if err := registry.Stop(); err != nil {
+					logrus.Errorf("Error stopping registry: %v", err)
+					os.Exit(1)
+				}
+			}
+
 			cleanup()
 		}
 	}, numBenchmarkSamples)
@@ -142,6 +173,44 @@ var _ = Describe("Podman Benchmark Suite", func() {
 			Expect(session).Should(Exit(0))
 		}, nil)
 
+		newBenchmark("podman load [docker]", func() {
+			session := podmanTest.Podman([]string{"load", "-i", "./testdata/docker-two-images.tar.xz"})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+		}, nil)
+
+		newBenchmark("podman load [oci]", func() {
+			session := podmanTest.Podman([]string{"load", "-i", "./testdata/oci-registry-name.tar.gz"})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+		}, nil)
+
+		newBenchmark("podman save", func() {
+			session := podmanTest.Podman([]string{"save", ALPINE, "-o", path.Join(podmanTest.TempDir, "alpine.tar")})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+		}, nil)
+
+		newBenchmark("podman image inspect", func() {
+			session := podmanTest.Podman([]string{"inspect", ALPINE})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+		}, nil)
+
+		newBenchmark("podman login + logout", func() {
+			user := os.Getenv(podmanRegistry.UserKey)
+			pass := os.Getenv(podmanRegistry.PassKey)
+			port := os.Getenv(podmanRegistry.PortKey)
+
+			session := podmanTest.Podman([]string{"login", "-u", user, "-p", pass, "--tls-verify=false", "localhost:" + port})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+
+			session = podmanTest.Podman([]string{"logout", "localhost:" + port})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+		}, &newBenchmarkOptions{needsRegistry: true})
+
 		// --------------------------------------------------------------------------
 		// CONTAINER BENCHMARKS
 		// --------------------------------------------------------------------------