podman/test/e2e/benchmarks_test.go

//go:build benchmarks
// +build benchmarks

package integration

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"

	podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go"
	. "github.com/containers/podman/v4/test/utils"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
	"github.com/sirupsen/logrus"
)

var (
	// Number of times to execute each benchmark.
	numBenchmarkSamples = 3
	// All benchmarks are ququed here.
	allBenchmarks []benchmark
)

// An internal struct for queuing benchmarks.
type benchmark struct {
	// The name of the benchmark.
	name string
	// The function to execute.
	main func()
	// Allows for extending a benchmark.
	options newBenchmarkOptions
}

var benchmarkRegistry *podmanRegistry.Registry

// Allows for customizing the benchnmark in an easy to extend way.
type newBenchmarkOptions struct {
	// Sets the benchmark's init function.
	init func()
	// Run a local registry for this benchmark. Use `getPortUserPass()` in
	// the benchmark to get the port, user and password.
	needsRegistry bool
}

// Queue a new benchmark.
func newBenchmark(name string, main func(), options *newBenchmarkOptions) {
	bm := benchmark{name: name, main: main}
	if options != nil {
		bm.options = *options
	}
	allBenchmarks = append(allBenchmarks, bm)
}

// getPortUserPass returns the port, user and password of the currently running
// registry.
func getPortUserPass() (string, string, string) {
	if benchmarkRegistry == nil {
		return "", "", ""
	}
	return benchmarkRegistry.Port, benchmarkRegistry.User, benchmarkRegistry.Password
}

var _ = Describe("Podman Benchmark Suite", func() {
	var (
		timedir    string
		podmanTest *PodmanTestIntegration
	)

	setup := func() {
		tempdir, err := CreateTempDirInTempDir()
		if err != nil {
			os.Exit(1)
		}
		podmanTest = PodmanTestCreate(tempdir)
		podmanTest.Setup()
		timedir, err = CreateTempDirInTempDir()
		if err != nil {
			os.Exit(1)
		}
	}

	cleanup := func() {
		podmanTest.Cleanup()
		os.RemoveAll(timedir)

		// Stop the local registry.
		if benchmarkRegistry != nil {
			if err := benchmarkRegistry.Stop(); err != nil {
				logrus.Errorf("Error stopping registry: %v", err)
				os.Exit(1)
			}
			benchmarkRegistry = nil
		}
	}

	totalMemoryInKb := func() (total uint64) {
		files, err := ioutil.ReadDir(timedir)
		if err != nil {
			Fail(fmt.Sprintf("Error reading timing dir: %v", err))
		}

		for _, f := range files {
			if f.IsDir() {
				continue
			}
			raw, err := ioutil.ReadFile(path.Join(timedir, f.Name()))
			if err != nil {
				Fail(fmt.Sprintf("Error reading timing file: %v", err))
			}
			rawS := strings.TrimSuffix(string(raw), "\n")
			number, err := strconv.ParseUint(rawS, 10, 64)
			if err != nil {
				Fail(fmt.Sprintf("Error converting timing file to numeric value: %v", err))
			}
			total += number
		}

		return total
	}

	// Make sure to clean up after the benchmarks.
	AfterEach(func() {
		cleanup()
	})

	// All benchmarks are executed here to have *one* table listing all data.
	Measure("Podman Benchmark Suite", func(b Benchmarker) {

		registryOptions := &podmanRegistry.Options{
			Image: "docker-archive:" + imageTarPath(REGISTRY_IMAGE),
		}

		for i := range allBenchmarks {
			setup()
			bm := allBenchmarks[i]

			// Start a local registry if requested.
			if bm.options.needsRegistry {
				reg, err := podmanRegistry.StartWithOptions(registryOptions)
				if err != nil {
					logrus.Errorf("Error starting registry: %v", err)
					os.Exit(1)
				}
				benchmarkRegistry = reg
			}

			if bm.options.init != nil {
				bm.options.init()
			}

			// Set the time dir only for the main() function.
			os.Setenv(EnvTimeDir, timedir)
			b.Time("[CPU] "+bm.name, bm.main)
			os.Unsetenv(EnvTimeDir)

			mem := totalMemoryInKb()
			b.RecordValueWithPrecision("[MEM] "+bm.name, float64(mem), "KB", 1)

			cleanup()
		}
	}, numBenchmarkSamples)

	BeforeEach(func() {

		// --------------------------------------------------------------------------
		// IMAGE BENCHMARKS
		// --------------------------------------------------------------------------

		newBenchmark("podman images", func() {
			session := podmanTest.Podman([]string{"images"})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, nil)

		newBenchmark("podman push", func() {
			port, user, pass := getPortUserPass()
			session := podmanTest.Podman([]string{"push", "--tls-verify=false", "--creds", user + ":" + pass, UBI_INIT, "localhost:" + port + "/repo/image:tag"})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, &newBenchmarkOptions{needsRegistry: true})

		newBenchmark("podman pull", func() {
			port, user, pass := getPortUserPass()
			session := podmanTest.Podman([]string{"pull", "--tls-verify=false", "--creds", user + ":" + pass, "localhost:" + port + "/repo/image:tag"})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, &newBenchmarkOptions{
			needsRegistry: true,
			init: func() {
				port, user, pass := getPortUserPass()
				session := podmanTest.Podman([]string{"push", "--tls-verify=false", "--creds", user + ":" + pass, UBI_INIT, "localhost:" + port + "/repo/image:tag"})
				session.WaitWithDefaultTimeout()
				Expect(session).Should(Exit(0))
			},
		})

		newBenchmark("podman load [docker]", func() {
			session := podmanTest.Podman([]string{"load", "-i", "./testdata/docker-two-images.tar.xz"})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, nil)

		newBenchmark("podman load [oci]", func() {
			session := podmanTest.Podman([]string{"load", "-i", "./testdata/oci-registry-name.tar.gz"})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, nil)

		newBenchmark("podman save", func() {
			session := podmanTest.Podman([]string{"save", ALPINE, "-o", path.Join(podmanTest.TempDir, "alpine.tar")})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, nil)

		newBenchmark("podman image inspect", func() {
			session := podmanTest.Podman([]string{"inspect", ALPINE})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, nil)

		newBenchmark("podman login + logout", func() {
			port, user, pass := getPortUserPass()

			session := podmanTest.Podman([]string{"login", "-u", user, "-p", pass, "--tls-verify=false", "localhost:" + port})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))

			session = podmanTest.Podman([]string{"logout", "localhost:" + port})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, &newBenchmarkOptions{needsRegistry: true})

		// --------------------------------------------------------------------------
		// CONTAINER BENCHMARKS
		// --------------------------------------------------------------------------

		newBenchmark("podman create", func() {
			session := podmanTest.Podman([]string{"run", ALPINE, "true"})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, nil)

		newBenchmark("podman start", func() {
			session := podmanTest.Podman([]string{"start", "foo"})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, &newBenchmarkOptions{
			init: func() {
				session := podmanTest.Podman([]string{"create", "--name=foo", ALPINE, "true"})
				session.WaitWithDefaultTimeout()
				Expect(session).Should(Exit(0))
			},
		})

		newBenchmark("podman run", func() {
			session := podmanTest.Podman([]string{"run", ALPINE, "true"})
			session.WaitWithDefaultTimeout()
			Expect(session).Should(Exit(0))
		}, nil)
	})
})