Merge pull request #24216 from Honny1/dev/jrodak/healthcheck-log

[v5.2-rhel] Add --health-max-log-count, --health-max-log-size, --health-log-destination flags

cmd/podman/common/create.go

@@ -184,6 +184,30 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
 		)
 		_ = cmd.RegisterFlagCompletionFunc(healthIntervalFlagName, completion.AutocompleteNone)
 
+		healthLogDestinationFlagName := "health-log-destination"
+		createFlags.StringVar(
+			&cf.HealthLogDestination,
+			healthLogDestinationFlagName, define.DefaultHealthCheckLocalDestination,
+			"set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file)",
+		)
+		_ = cmd.RegisterFlagCompletionFunc(healthLogDestinationFlagName, completion.AutocompleteNone)
+
+		healthMaxLogCountFlagName := "health-max-log-count"
+		createFlags.UintVar(
+			&cf.HealthMaxLogCount,
+			healthMaxLogCountFlagName, define.DefaultHealthMaxLogCount,
+			"set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file)",
+		)
+		_ = cmd.RegisterFlagCompletionFunc(healthMaxLogCountFlagName, completion.AutocompleteNone)
+
+		healthMaxLogSizeFlagName := "health-max-log-size"
+		createFlags.UintVar(
+			&cf.HealthMaxLogSize,
+			healthMaxLogSizeFlagName, define.DefaultHealthMaxLogSize,
+			"set maximum length in characters of stored HealthCheck log. ('0' value means an infinite log length)",
+		)
+		_ = cmd.RegisterFlagCompletionFunc(healthMaxLogSizeFlagName, completion.AutocompleteNone)
+
 		healthRetriesFlagName := "health-retries"
 		createFlags.UintVar(
 			&cf.HealthRetries,

cmd/podman/common/create_opts.go

@@ -85,4 +85,7 @@ func DefineCreateDefaults(opts *entities.ContainerCreateOptions) {
 	opts.Ulimit = ulimits()
 	opts.SeccompPolicy = "default"
 	opts.Volume = volumes()
+	opts.HealthLogDestination = define.DefaultHealthCheckLocalDestination
+	opts.HealthMaxLogCount = define.DefaultHealthMaxLogCount
+	opts.HealthMaxLogSize = define.DefaultHealthMaxLogSize
 }

docs/source/markdown/options/health-log-destination.md

@@ -0,0 +1,11 @@
+####> This option file is used in:
+####>   podman create, run
+####> If file is edited, make sure the changes
+####> are applicable to all of those.
+#### **--health-log-destination**=*directory_path*
+
+Set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file) (Default: local)
+
+* `local`: (default) HealthCheck logs are stored in overlay containers. (For example: `$runroot/healthcheck.log`)
+* `directory`: creates a log file named `<container-ID>-healthcheck.log` with HealthCheck logs in the specified directory.
+* `events_logger`: The log will be written with logging mechanism set by events_logger. It also saves the log to a default directory, for performance on a system with a large number of logs.

docs/source/markdown/options/health-max-log-count.md

@@ -0,0 +1,7 @@
+####> This option file is used in:
+####>   podman create, run
+####> If file is edited, make sure the changes
+####> are applicable to all of those.
+#### **--health-max-log-count**=*number of stored logs*
+
+Set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file) (Default: 5 attempts)

docs/source/markdown/options/health-max-log-size.md

@@ -0,0 +1,7 @@
+####> This option file is used in:
+####>   podman create, run
+####> If file is edited, make sure the changes
+####> are applicable to all of those.
+#### **--health-max-log-size**=*size of stored logs*
+
+Set maximum length in characters of stored HealthCheck log. ("0" value means an infinite log length) (Default: 500 characters)

docs/source/markdown/podman-create.1.md.in

@@ -169,6 +169,12 @@ See [**Environment**](#environment) note below for precedence and examples.
 
 @@option health-interval
 
+@@option health-log-destination
+
+@@option health-max-log-count
+
+@@option health-max-log-size
+
 @@option health-on-failure
 
 @@option health-retries

docs/source/markdown/podman-run.1.md.in

@@ -203,6 +203,12 @@ See [**Environment**](#environment) note below for precedence and examples.
 
 @@option health-interval
 
+@@option health-log-destination
+
+@@option health-max-log-count
+
+@@option health-max-log-size
+
 @@option health-on-failure
 
 @@option health-retries

docs/source/markdown/podman-systemd.unit.5.md

@@ -267,6 +267,9 @@ Valid options for `[Container]` are listed below:
 | GroupAdd=keep-groups                 | --group-add=keep-groups                              |
 | HealthCmd=/usr/bin/command           | --health-cmd=/usr/bin/command                        |
 | HealthInterval=2m                    | --health-interval=2m                                 |
+| HealthLogDestination=/foo/log        | --health-log-destination=/foo/log                    |
+| HealthMaxLogCount=5                  | --health-max-log-count=5                             |
+| HealthMaxLogSize=500                 | --health-max-log-size=500                            |
 | HealthOnFailure=kill                 | --health-on-failure=kill                             |
 | HealthRetries=5                      | --health-retries=5                                   |
 | HealthStartPeriod=1m                 | --health-start-period=period=1m                      |
@@ -475,6 +478,28 @@ Equivalent to the Podman `--health-cmd` option.
 Set an interval for the healthchecks. An interval of disable results in no automatic timer setup.
 Equivalent to the Podman `--health-interval` option.
 
+### `HealthLogDestination=`
+
+Set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file)
+(Default: local)
+Equivalent to the Podman `--health-log-destination` option.
+
+* `local`: (default) HealthCheck logs are stored in overlay containers. (For example: `$runroot/healthcheck.log`)
+* `directory`: creates a log file named `<container-ID>-healthcheck.log` with HealthCheck logs in the specified directory.
+* `events_logger`: The log will be written with logging mechanism set by events_logger. It also saves the log to a default directory, for performance on a system with a large number of logs.
+
+### `HealthMaxLogCount=`
+
+Set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file)
+(Default: 5 attempts)
+Equivalent to the Podman `--Health-max-log-count` option.
+
+### `HealthMaxLogSize=`
+
+Set maximum length in characters of stored HealthCheck log. ("0" value means an infinite log length)
+(Default: 500 characters)
+Equivalent to the Podman `--Health-max-log-size` option.
+
 ### `HealthOnFailure=`
 
 Action to take once the container transitions to an unhealthy state.

libpod/container_config.go

@@ -413,6 +413,14 @@ type ContainerMiscConfig struct {
 	HealthCheckConfig *manifest.Schema2HealthConfig `json:"healthcheck"`
 	// HealthCheckOnFailureAction defines an action to take once the container turns unhealthy.
 	HealthCheckOnFailureAction define.HealthCheckOnFailureAction `json:"healthcheck_on_failure_action"`
+	// HealthLogDestination defines the destination where the log is stored
+	HealthLogDestination string `json:"healthLogDestination,omitempty"`
+	// HealthMaxLogCount is maximum number of attempts in the HealthCheck log file.
+	// ('0' value means an infinite number of attempts in the log file)
+	HealthMaxLogCount uint `json:"healthMaxLogCount,omitempty"`
+	// HealthMaxLogSize is the maximum length in characters of stored HealthCheck log
+	// ("0" value means an infinite log length)
+	HealthMaxLogSize uint `json:"healthMaxLogSize,omitempty"`
 	// StartupHealthCheckConfig is the configuration of the startup
 	// healthcheck for the container. This will run before the regular HC
 	// runs, and when it passes the regular HC will be activated.

libpod/container_inspect.go

@@ -195,7 +195,7 @@ func (c *Container) getContainerInspectData(size bool, driverData *define.Driver
 	// inspect status should be set to nil.
 	if c.config.HealthCheckConfig != nil && !(len(c.config.HealthCheckConfig.Test) == 1 && c.config.HealthCheckConfig.Test[0] == "NONE") {
 		// This container has a healthcheck defined in it; we need to add its state
-		healthCheckState, err := c.getHealthCheckLog()
+		healthCheckState, err := c.readHealthCheckLog()
 		if err != nil {
 			// An error here is not considered fatal; no health state will be displayed
 			logrus.Error(err)
@@ -417,6 +417,12 @@ func (c *Container) generateInspectContainerConfig(spec *spec.Spec) *define.Insp
 
 	ctrConfig.HealthcheckOnFailureAction = c.config.HealthCheckOnFailureAction.String()
 
+	ctrConfig.HealthLogDestination = c.config.HealthLogDestination
+
+	ctrConfig.HealthMaxLogCount = c.config.HealthMaxLogCount
+
+	ctrConfig.HealthMaxLogSize = c.config.HealthMaxLogSize
+
 	ctrConfig.CreateCommand = c.config.CreateCommand
 
 	ctrConfig.Timezone = c.config.Timezone

libpod/container_internal.go

@@ -1123,10 +1123,9 @@ func (c *Container) init(ctx context.Context, retainRetries bool) error {
 	// bugzilla.redhat.com/show_bug.cgi?id=2144754:
 	// In case of a restart, make sure to remove the healthcheck log to
 	// have a clean state.
-	if path := c.healthCheckLogPath(); path != "" {
-		if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
-			logrus.Error(err)
-		}
+	err = c.writeHealthCheckLog(define.HealthCheckResults{Status: define.HealthCheckReset})
+	if err != nil {
+		return err
 	}
 
 	if err := c.save(); err != nil {

libpod/define/container_inspect.go

@@ -61,6 +61,14 @@ type InspectContainerConfig struct {
 	Healthcheck *manifest.Schema2HealthConfig `json:"Healthcheck,omitempty"`
 	// HealthcheckOnFailureAction defines an action to take once the container turns unhealthy.
 	HealthcheckOnFailureAction string `json:"HealthcheckOnFailureAction,omitempty"`
+	// HealthLogDestination defines the destination where the log is stored
+	HealthLogDestination string `json:"HealthLogDestination,omitempty"`
+	// HealthMaxLogCount is maximum number of attempts in the HealthCheck log file.
+	// ('0' value means an infinite number of attempts in the log file)
+	HealthMaxLogCount uint `json:"HealthcheckMaxLogCount,omitempty"`
+	// HealthMaxLogSize is the maximum length in characters of stored HealthCheck log
+	// ("0" value means an infinite log length)
+	HealthMaxLogSize uint `json:"HealthcheckMaxLogSize,omitempty"`
 	// CreateCommand is the full command plus arguments of the process the
 	// container has been created with.
 	CreateCommand []string `json:"CreateCommand,omitempty"`

libpod/define/healthchecks.go

@@ -16,6 +16,8 @@ const (
 	// and the start-period (time allowed for the container to start and application
 	// to be running) expires.
 	HealthCheckStarting string = "starting"
+	// HealthCheckReset describes reset of HealthCheck logs
+	HealthCheckReset string = "reset"
 )
 
 // HealthCheckStatus represents the current state of a container
@@ -56,8 +58,16 @@ const (
 	DefaultHealthCheckStartPeriod = "0s"
 	// DefaultHealthCheckTimeout default value
 	DefaultHealthCheckTimeout = "30s"
+	// DefaultHealthMaxLogCount default value
+	DefaultHealthMaxLogCount uint = 5
+	// DefaultHealthMaxLogSize default value
+	DefaultHealthMaxLogSize uint = 500
+	// DefaultHealthCheckLocalDestination default value
+	DefaultHealthCheckLocalDestination string = "local"
 )
 
+const HealthCheckEventsLoggerDestination string = "events_logger"
+
 // HealthConfig.Test options
 const (
 	// HealthConfigTestNone disables healthcheck

libpod/events.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"sync"
 
+	"github.com/containers/podman/v5/libpod/define"
 	"github.com/containers/podman/v5/libpod/events"
 	"github.com/sirupsen/logrus"
 )
@@ -28,27 +29,37 @@ func (r *Runtime) newEventer() (events.Eventer, error) {
 
 // newContainerEvent creates a new event based on a container
 func (c *Container) newContainerEvent(status events.Status) {
-	if err := c.newContainerEventWithInspectData(status, "", false); err != nil {
+	if err := c.newContainerEventWithInspectData(status, define.HealthCheckResults{}, false); err != nil {
 		logrus.Errorf("Unable to write container event: %v", err)
 	}
 }
 
 // newContainerHealthCheckEvent creates a new healthcheck event with the given status
-func (c *Container) newContainerHealthCheckEvent(healthStatus string) {
-	if err := c.newContainerEventWithInspectData(events.HealthStatus, healthStatus, false); err != nil {
+func (c *Container) newContainerHealthCheckEvent(healthCheckResult define.HealthCheckResults) {
+	if err := c.newContainerEventWithInspectData(events.HealthStatus, healthCheckResult, false); err != nil {
 		logrus.Errorf("Unable to write container event: %v", err)
 	}
 }
 
 // newContainerEventWithInspectData creates a new event and sets the
 // ContainerInspectData field if inspectData is set.
-func (c *Container) newContainerEventWithInspectData(status events.Status, healthStatus string, inspectData bool) error {
+func (c *Container) newContainerEventWithInspectData(status events.Status, healthCheckResult define.HealthCheckResults, inspectData bool) error {
 	e := events.NewEvent(status)
 	e.ID = c.ID()
 	e.Name = c.Name()
 	e.Image = c.config.RootfsImageName
 	e.Type = events.Container
-	e.HealthStatus = healthStatus
+	e.HealthStatus = healthCheckResult.Status
+	if c.config.HealthLogDestination == define.HealthCheckEventsLoggerDestination {
+		if len(healthCheckResult.Log) > 0 {
+			logData, err := json.Marshal(healthCheckResult.Log[len(healthCheckResult.Log)-1])
+			if err != nil {
+				return fmt.Errorf("unable to marshall healthcheck log for writing: %w", err)
+			}
+			e.HealthLog = string(logData)
+		}
+	}
+	e.HealthFailingStreak = healthCheckResult.FailingStreak
 
 	e.Details = events.Details{
 		PodID:      c.PodID(),

libpod/events/config.go

@@ -41,6 +41,10 @@ type Event struct {
 	Type Type
 	// Health status of the current container
 	HealthStatus string `json:"health_status,omitempty"`
+	// Healthcheck log of the current container
+	HealthLog string `json:"health_log,omitempty"`
+	// HealthFailingStreak log of the current container
+	HealthFailingStreak int `json:"health_failing_streak,omitempty"`
 	// Error code for certain events involving errors.
 	Error string `json:"error,omitempty"`
 

libpod/events/events.go

@@ -76,8 +76,10 @@ func (e *Event) ToHumanReadable(truncate bool) string {
 		if e.PodID != "" {
 			humanFormat += fmt.Sprintf(", pod_id=%s", e.PodID)
 		}
-		if e.HealthStatus != "" {
+		if e.Status == HealthStatus {
 			humanFormat += fmt.Sprintf(", health_status=%s", e.HealthStatus)
+			humanFormat += fmt.Sprintf(", health_failing_streak=%d", e.HealthFailingStreak)
+			humanFormat += fmt.Sprintf(", health_log=%s", e.HealthLog)
 		}
 		// check if the container has labels and add it to the output
 		if len(e.Attributes) > 0 {

libpod/events/journal_linux.go

@@ -65,8 +65,13 @@ func (e EventJournalD) Write(ee Event) error {
 			}
 			m["PODMAN_LABELS"] = string(b)
 		}
+		if ee.Status == HealthStatus {
 			m["PODMAN_HEALTH_STATUS"] = ee.HealthStatus
-
+			if ee.HealthLog != "" {
+				m["PODMAN_HEALTH_LOG"] = ee.HealthLog
+			}
+			m["PODMAN_HEALTH_FAILING_STREAK"] = strconv.Itoa(ee.HealthFailingStreak)
+		}
 		if len(ee.Details.ContainerInspectData) > 0 {
 			m["PODMAN_CONTAINER_INSPECT_DATA"] = ee.Details.ContainerInspectData
 		}
@@ -225,6 +230,15 @@ func newEventFromJournalEntry(entry *sdjournal.JournalEntry) (*Event, error) {
 			}
 		}
 		newEvent.HealthStatus = entry.Fields["PODMAN_HEALTH_STATUS"]
+		if log, ok := entry.Fields["PODMAN_HEALTH_LOG"]; ok {
+			newEvent.HealthLog = log
+		}
+		if FailingStreak, ok := entry.Fields["PODMAN_HEALTH_FAILING_STREAK"]; ok {
+			FailingStreakInt, err := strconv.Atoi(FailingStreak)
+			if err == nil {
+				newEvent.HealthFailingStreak = FailingStreakInt
+			}
+		}
 		newEvent.Details.ContainerInspectData = entry.Fields["PODMAN_CONTAINER_INSPECT_DATA"]
 	case Network:
 		newEvent.ID = entry.Fields["PODMAN_ID"]

libpod/healthcheck.go

@@ -19,14 +19,6 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-const (
-	// MaxHealthCheckNumberLogs is the maximum number of attempts we keep
-	// in the healthcheck history file
-	MaxHealthCheckNumberLogs int = 5
-	// MaxHealthCheckLogLength in characters
-	MaxHealthCheckLogLength = 500
-)
-
 // HealthCheck verifies the state and validity of the healthcheck configuration
 // on the container and then executes the healthcheck
 func (r *Runtime) HealthCheck(ctx context.Context, name string) (define.HealthCheckStatus, error) {
@@ -143,8 +135,8 @@ func (c *Container) runHealthCheck(ctx context.Context, isStartup bool) (define.
 	}
 
 	eventLog := output.String()
-	if len(eventLog) > MaxHealthCheckLogLength {
-		eventLog = eventLog[:MaxHealthCheckLogLength]
+	if c.config.HealthMaxLogSize != 0 && len(eventLog) > int(c.config.HealthMaxLogSize) {
+		eventLog = eventLog[:c.config.HealthMaxLogSize]
 	}
 
 	if timeEnd.Sub(timeStart) > c.HealthCheckConfig().Timeout {
@@ -154,21 +146,22 @@ func (c *Container) runHealthCheck(ctx context.Context, isStartup bool) (define.
 	}
 
 	hcl := newHealthCheckLog(timeStart, timeEnd, returnCode, eventLog)
-	logStatus, err := c.updateHealthCheckLog(hcl, inStartPeriod, isStartup)
+
+	healthCheckResult, err := c.updateHealthCheckLog(hcl, inStartPeriod, isStartup)
 	if err != nil {
-		return hcResult, "", fmt.Errorf("unable to update health check log %s for %s: %w", c.healthCheckLogPath(), c.ID(), err)
+		return hcResult, "", fmt.Errorf("unable to update health check log %s for %s: %w", c.config.HealthLogDestination, c.ID(), err)
 	}
 
 	// Write HC event with appropriate status as the last thing before we
 	// return.
 	if hcResult == define.HealthCheckNotDefined || hcResult == define.HealthCheckInternalError {
-		return hcResult, logStatus, hcErr
+		return hcResult, healthCheckResult.Status, hcErr
 	}
 	if c.runtime.config.Engine.HealthcheckEvents {
-		c.newContainerHealthCheckEvent(logStatus)
+		c.newContainerHealthCheckEvent(healthCheckResult)
 	}
 
-	return hcResult, logStatus, hcErr
+	return hcResult, healthCheckResult.Status, hcErr
 }
 
 func (c *Container) processHealthCheckStatus(status string) error {
@@ -340,16 +333,12 @@ func newHealthCheckLog(start, end time.Time, exitCode int, log string) define.He
 // updateHealthStatus updates the health status of the container
 // in the healthcheck log
 func (c *Container) updateHealthStatus(status string) error {
-	healthCheck, err := c.getHealthCheckLog()
+	healthCheck, err := c.readHealthCheckLog()
 	if err != nil {
 		return err
 	}
 	healthCheck.Status = status
-	newResults, err := json.Marshal(healthCheck)
-	if err != nil {
-		return fmt.Errorf("unable to marshall healthchecks for writing status: %w", err)
-	}
-	return os.WriteFile(c.healthCheckLogPath(), newResults, 0700)
+	return c.writeHealthCheckLog(healthCheck)
 }
 
 // isUnhealthy returns true if the current health check status is unhealthy.
@@ -357,7 +346,7 @@ func (c *Container) isUnhealthy() (bool, error) {
 	if !c.HasHealthCheck() {
 		return false, nil
 	}
-	healthCheck, err := c.getHealthCheckLog()
+	healthCheck, err := c.readHealthCheckLog()
 	if err != nil {
 		return false, err
 	}
@@ -365,7 +354,7 @@ func (c *Container) isUnhealthy() (bool, error) {
 }
 
 // UpdateHealthCheckLog parses the health check results and writes the log
-func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPeriod, isStartup bool) (string, error) {
+func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPeriod, isStartup bool) (define.HealthCheckResults, error) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
@@ -373,12 +362,12 @@ func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPerio
 	// both failing and succeeding cases to match kube behavior.
 	// So don't update the health check log till the start period is over
 	if _, ok := c.config.Spec.Annotations[define.KubeHealthCheckAnnotation]; ok && inStartPeriod && !isStartup {
-		return "", nil
+		return define.HealthCheckResults{}, nil
 	}
 
-	healthCheck, err := c.getHealthCheckLog()
+	healthCheck, err := c.readHealthCheckLog()
 	if err != nil {
-		return "", err
+		return define.HealthCheckResults{}, err
 	}
 	if hcl.ExitCode == 0 {
 		//	set status to healthy, reset failing state to 0
@@ -398,28 +387,48 @@ func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPerio
 		}
 	}
 	healthCheck.Log = append(healthCheck.Log, hcl)
-	if len(healthCheck.Log) > MaxHealthCheckNumberLogs {
+	if c.config.HealthMaxLogCount != 0 && len(healthCheck.Log) > int(c.config.HealthMaxLogCount) {
 		healthCheck.Log = healthCheck.Log[1:]
 	}
-	newResults, err := json.Marshal(healthCheck)
+	return healthCheck, c.writeHealthCheckLog(healthCheck)
+}
+
+func (c *Container) witeToFileHealthCheckResults(path string, result define.HealthCheckResults) error {
+	newResults, err := json.Marshal(result)
 	if err != nil {
-		return "", fmt.Errorf("unable to marshall healthchecks for writing: %w", err)
+		return fmt.Errorf("unable to marshall healthchecks for writing: %w", err)
 	}
-	return healthCheck.Status, os.WriteFile(c.healthCheckLogPath(), newResults, 0700)
+	return os.WriteFile(path, newResults, 0700)
 }
 
-// HealthCheckLogPath returns the path for where the health check log is
-func (c *Container) healthCheckLogPath() string {
-	return filepath.Join(filepath.Dir(c.state.RunDir), "healthcheck.log")
+func (c *Container) getHealthCheckLogDestination() string {
+	var destination string
+	switch c.config.HealthLogDestination {
+	case define.DefaultHealthCheckLocalDestination, define.HealthCheckEventsLoggerDestination, "":
+		destination = filepath.Join(filepath.Dir(c.state.RunDir), "healthcheck.log")
+	default:
+		destination = filepath.Join(c.config.HealthLogDestination, c.ID()+"-healthcheck.log")
+	}
+	return destination
 }
 
-// getHealthCheckLog returns HealthCheck results by reading the container's
+func (c *Container) writeHealthCheckLog(result define.HealthCheckResults) error {
+	return c.witeToFileHealthCheckResults(c.getHealthCheckLogDestination(), result)
+}
+
+// readHealthCheckLog read HealthCheck logs from the path or events_logger
+// The caller should lock the container before this function is called.
+func (c *Container) readHealthCheckLog() (define.HealthCheckResults, error) {
+	return c.readFromFileHealthCheckLog(c.getHealthCheckLogDestination())
+}
+
+// readFromFileHealthCheckLog returns HealthCheck results by reading the container's
 // health check log file.  If the health check log file does not exist, then
 // an empty healthcheck struct is returned
 // The caller should lock the container before this function is called.
-func (c *Container) getHealthCheckLog() (define.HealthCheckResults, error) {
+func (c *Container) readFromFileHealthCheckLog(path string) (define.HealthCheckResults, error) {
 	var healthCheck define.HealthCheckResults
-	b, err := os.ReadFile(c.healthCheckLogPath())
+	b, err := os.ReadFile(path)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			// If the file does not exists just return empty healthcheck and no error.
@@ -428,7 +437,7 @@ func (c *Container) getHealthCheckLog() (define.HealthCheckResults, error) {
 		return healthCheck, fmt.Errorf("failed to read health check log file: %w", err)
 	}
 	if err := json.Unmarshal(b, &healthCheck); err != nil {
-		return healthCheck, fmt.Errorf("failed to unmarshal existing healthcheck results in %s: %w", c.healthCheckLogPath(), err)
+		return healthCheck, fmt.Errorf("failed to unmarshal existing healthcheck results in %s: %w", path, err)
 	}
 	return healthCheck, nil
 }
@@ -454,7 +463,7 @@ func (c *Container) healthCheckStatus() (string, error) {
 		return "", err
 	}
 
-	results, err := c.getHealthCheckLog()
+	results, err := c.readHealthCheckLog()
 	if err != nil {
 		return "", fmt.Errorf("unable to get healthcheck log for %s: %w", c.ID(), err)
 	}

libpod/options.go

@@ -6,6 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"time"
@@ -1513,6 +1515,57 @@ func WithHealthCheck(healthCheck *manifest.Schema2HealthConfig) CtrCreateOption
 	}
 }
 
+// WithHealthCheckLogDestination adds the healthLogDestination to the container config
+func WithHealthCheckLogDestination(destination string) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+		switch destination {
+		case define.HealthCheckEventsLoggerDestination, define.DefaultHealthCheckLocalDestination:
+			ctr.config.HealthLogDestination = destination
+		default:
+			fileInfo, err := os.Stat(destination)
+			if err != nil {
+				return fmt.Errorf("HealthCheck Log '%s' destination error: %w", destination, err)
+			}
+			mode := fileInfo.Mode()
+			if !mode.IsDir() {
+				return fmt.Errorf("HealthCheck Log '%s' destination must be directory", destination)
+			}
+
+			absPath, err := filepath.Abs(destination)
+			if err != nil {
+				return err
+			}
+			ctr.config.HealthLogDestination = absPath
+		}
+		return nil
+	}
+}
+
+// WithHealthCheckMaxLogCount adds the healthMaxLogCount to the container config
+func WithHealthCheckMaxLogCount(maxLogCount uint) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+		ctr.config.HealthMaxLogCount = maxLogCount
+		return nil
+	}
+}
+
+// WithHealthCheckMaxLogSize adds the healthMaxLogSize to the container config
+func WithHealthCheckMaxLogSize(maxLogSize uint) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+		ctr.config.HealthMaxLogSize = maxLogSize
+		return nil
+	}
+}
+
 // WithHealthCheckOnFailureAction adds an on-failure action to health-check config
 func WithHealthCheckOnFailureAction(action define.HealthCheckOnFailureAction) CtrCreateOption {
 	return func(ctr *Container) error {

libpod/runtime_ctr.go

@@ -585,7 +585,7 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai
 	}
 
 	if ctr.runtime.config.Engine.EventsContainerCreateInspectData {
-		if err := ctr.newContainerEventWithInspectData(events.Create, "", true); err != nil {
+		if err := ctr.newContainerEventWithInspectData(events.Create, define.HealthCheckResults{}, true); err != nil {
 			return nil, err
 		}
 	} else {

pkg/api/handlers/compat/containers_create.go

@@ -477,6 +477,9 @@ func cliOpts(cc handlers.CreateContainerConfig, rtc *config.Config) (*entities.C
 		HealthRetries:        define.DefaultHealthCheckRetries,
 		HealthTimeout:        define.DefaultHealthCheckTimeout,
 		HealthStartPeriod:    define.DefaultHealthCheckStartPeriod,
+		HealthLogDestination: define.DefaultHealthCheckLocalDestination,
+		HealthMaxLogCount:    define.DefaultHealthMaxLogCount,
+		HealthMaxLogSize:     define.DefaultHealthMaxLogSize,
 	}
 	if !rootless.IsRootless() {
 		var ulimits []string

pkg/api/handlers/libpod/containers_create.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 
 	"github.com/containers/podman/v5/libpod"
+	"github.com/containers/podman/v5/libpod/define"
 	"github.com/containers/podman/v5/pkg/api/handlers/utils"
 	api "github.com/containers/podman/v5/pkg/api/types"
 	"github.com/containers/podman/v5/pkg/domain/entities"
@@ -42,6 +43,9 @@ func CreateContainer(w http.ResponseWriter, r *http.Request) {
 			Umask:      conf.Containers.Umask,
 			Privileged: &privileged,
 		},
+		ContainerHealthCheckConfig: specgen.ContainerHealthCheckConfig{
+			HealthLogDestination: define.DefaultHealthCheckLocalDestination,
+		},
 	}
 
 	if err := json.NewDecoder(r.Body).Decode(&sg); err != nil {

pkg/domain/entities/pods.go

@@ -171,6 +171,9 @@ type ContainerCreateOptions struct {
 	HealthCmd            string
 	HealthInterval       string
 	HealthRetries        uint
+	HealthLogDestination string
+	HealthMaxLogCount    uint
+	HealthMaxLogSize     uint
 	HealthStartPeriod    string
 	HealthTimeout        string
 	HealthOnFailure      string

pkg/domain/infra/abi/containers.go

@@ -1736,6 +1736,10 @@ func (ic *ContainerEngine) ContainerClone(ctx context.Context, ctrCloneOpts enti
 		spec.Name = generate.CheckName(ic.Libpod, n, true)
 	}
 
+	spec.HealthLogDestination = define.DefaultHealthCheckLocalDestination
+	spec.HealthMaxLogCount = define.DefaultHealthMaxLogCount
+	spec.HealthMaxLogSize = define.DefaultHealthMaxLogSize
+
 	rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), ic.Libpod, spec, true, c)
 	if err != nil {
 		return nil, err

pkg/domain/infra/abi/play.go

@@ -86,6 +86,9 @@ func (ic *ContainerEngine) createServiceContainer(ctx context.Context, name stri
 		// No need to spin up slirp etc.
 		Net:                  &entities.NetOptions{Network: specgen.Namespace{NSMode: specgen.NoNetwork}},
 		StopTimeout:          rtc.Engine.StopTimeout,
+		HealthLogDestination: define.DefaultHealthCheckLocalDestination,
+		HealthMaxLogCount:    define.DefaultHealthMaxLogCount,
+		HealthMaxLogSize:     define.DefaultHealthMaxLogSize,
 	}
 
 	// Create and fill out the runtime spec.

pkg/specgen/generate/container.go

@@ -444,6 +444,10 @@ func ConfigToSpec(rt *libpod.Runtime, specg *specgen.SpecGenerator, containerID
 		}
 	}
 
+	specg.HealthLogDestination = conf.HealthLogDestination
+	specg.HealthMaxLogCount = conf.HealthMaxLogCount
+	specg.HealthMaxLogSize = conf.HealthMaxLogSize
+
 	specg.IDMappings = &conf.IDMappings
 	specg.ContainerCreateCommand = conf.CreateCommand
 	if len(specg.Rootfs) == 0 {

pkg/specgen/generate/container_create.go

@@ -642,6 +642,10 @@ func createContainerOptions(rt *libpod.Runtime, s *specgen.SpecGenerator, pod *l
 		options = append(options, libpod.WithHealthCheckOnFailureAction(s.ContainerHealthCheckConfig.HealthCheckOnFailureAction))
 	}
 
+	options = append(options, libpod.WithHealthCheckLogDestination(s.ContainerHealthCheckConfig.HealthLogDestination))
+	options = append(options, libpod.WithHealthCheckMaxLogCount(s.ContainerHealthCheckConfig.HealthMaxLogCount))
+	options = append(options, libpod.WithHealthCheckMaxLogSize(s.ContainerHealthCheckConfig.HealthMaxLogSize))
+
 	if s.SdNotifyMode == define.SdNotifyModeHealthy && !healthCheckSet {
 		return nil, fmt.Errorf("%w: sdnotify policy %q requires a healthcheck to be set", define.ErrInvalidArg, s.SdNotifyMode)
 	}

pkg/specgen/generate/kube/kube.go

@@ -438,6 +438,9 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener
 	}
 
 	s.Annotations[define.KubeHealthCheckAnnotation] = "true"
+	s.HealthLogDestination = define.DefaultHealthCheckLocalDestination
+	s.HealthMaxLogCount = define.DefaultHealthMaxLogCount
+	s.HealthMaxLogSize = define.DefaultHealthMaxLogSize
 
 	// Environment Variables
 	envs := map[string]string{}

pkg/specgen/generate/pod_create.go

@@ -85,6 +85,12 @@ func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (_ *libpod.Pod, finalErr e
 		// make sure of that here.
 		p.PodSpecGen.InfraContainerSpec.ResourceLimits = nil
 		p.PodSpecGen.InfraContainerSpec.WeightDevice = nil
+
+		// Set default for HealthCheck
+		p.PodSpecGen.InfraContainerSpec.HealthLogDestination = define.DefaultHealthCheckLocalDestination
+		p.PodSpecGen.InfraContainerSpec.HealthMaxLogCount = define.DefaultHealthMaxLogCount
+		p.PodSpecGen.InfraContainerSpec.HealthMaxLogSize = define.DefaultHealthMaxLogSize
+
 		rtSpec, spec, opts, err := MakeContainer(context.Background(), rt, p.PodSpecGen.InfraContainerSpec, false, nil)
 		if err != nil {
 			return nil, err

pkg/specgen/specgen.go

@@ -599,6 +599,14 @@ type ContainerHealthCheckConfig struct {
 	// Requires that HealthConfig be set.
 	// Optional.
 	StartupHealthConfig *define.StartupHealthCheck `json:"startupHealthConfig,omitempty"`
+	// HealthLogDestination defines the destination where the log is stored
+	HealthLogDestination string `json:"healthLogDestination,omitempty"`
+	// HealthMaxLogCount is maximum number of attempts in the HealthCheck log file.
+	// ('0' value means an infinite number of attempts in the log file)
+	HealthMaxLogCount uint `json:"healthMaxLogCount,omitempty"`
+	// HealthMaxLogSize is the maximum length in characters of stored HealthCheck log
+	// ("0" value means an infinite log length)
+	HealthMaxLogSize uint `json:"healthMaxLogSize,omitempty"`
 }
 
 // SpecGenerator creates an OCI spec and Libpod configuration options to create
@@ -671,13 +679,25 @@ func NewSpecGenerator(arg string, rootfs bool) *SpecGenerator {
 	}
 	return &SpecGenerator{
 		ContainerStorageConfig: csc,
+		ContainerHealthCheckConfig: ContainerHealthCheckConfig{
+			HealthLogDestination: define.DefaultHealthCheckLocalDestination,
+			HealthMaxLogCount:    define.DefaultHealthMaxLogCount,
+			HealthMaxLogSize:     define.DefaultHealthMaxLogSize,
+		},
 	}
 }
 
 // NewSpecGenerator returns a SpecGenerator struct given one of two mandatory inputs
 func NewSpecGeneratorWithRootfs(rootfs string) *SpecGenerator {
 	csc := ContainerStorageConfig{Rootfs: rootfs}
-	return &SpecGenerator{ContainerStorageConfig: csc}
+	return &SpecGenerator{
+		ContainerStorageConfig: csc,
+		ContainerHealthCheckConfig: ContainerHealthCheckConfig{
+			HealthLogDestination: define.DefaultHealthCheckLocalDestination,
+			HealthMaxLogCount:    define.DefaultHealthMaxLogCount,
+			HealthMaxLogSize:     define.DefaultHealthMaxLogSize,
+		},
+	}
 }
 
 func StringSlicesEqual(a, b []string) bool {

pkg/specgenutil/specgen.go

@@ -370,6 +370,12 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 	}
 	s.HealthCheckOnFailureAction = onFailureAction
 
+	s.HealthLogDestination = c.HealthLogDestination
+
+	s.HealthMaxLogCount = c.HealthMaxLogCount
+
+	s.HealthMaxLogSize = c.HealthMaxLogSize
+
 	if c.StartupHCCmd != "" {
 		if c.NoHealthCheck {
 			return errors.New("cannot specify both --no-healthcheck and --health-startup-cmd")

pkg/systemd/quadlet/quadlet.go

@@ -89,6 +89,9 @@ const (
 	KeyGroupAdd              = "GroupAdd"
 	KeyHealthCmd             = "HealthCmd"
 	KeyHealthInterval        = "HealthInterval"
+	KeyHealthLogDestination  = "HealthLogDestination"
+	KeyHealthMaxLogCount     = "HealthMaxLogCount"
+	KeyHealthMaxLogSize      = "HealthMaxLogSize"
 	KeyHealthOnFailure       = "HealthOnFailure"
 	KeyHealthRetries         = "HealthRetries"
 	KeyHealthStartPeriod     = "HealthStartPeriod"
@@ -201,6 +204,9 @@ var (
 		KeyHealthCmd:             true,
 		KeyHealthInterval:        true,
 		KeyHealthOnFailure:       true,
+		KeyHealthLogDestination:  true,
+		KeyHealthMaxLogCount:     true,
+		KeyHealthMaxLogSize:      true,
 		KeyHealthRetries:         true,
 		KeyHealthStartPeriod:     true,
 		KeyHealthStartupCmd:      true,
@@ -1879,6 +1885,9 @@ func handleHealth(unitFile *parser.UnitFile, groupName string, podman *PodmanCmd
 		{KeyHealthCmd, "cmd"},
 		{KeyHealthInterval, "interval"},
 		{KeyHealthOnFailure, "on-failure"},
+		{KeyHealthLogDestination, "log-destination"},
+		{KeyHealthMaxLogCount, "max-log-count"},
+		{KeyHealthMaxLogSize, "max-log-size"},
 		{KeyHealthRetries, "retries"},
 		{KeyHealthStartPeriod, "start-period"},
 		{KeyHealthTimeout, "timeout"},

test/system/220-healthcheck.bats

@@ -255,4 +255,185 @@ Log[-1].Output   | \"Uh-oh on stdout!\\\nUh-oh on stderr!\\\n\"
     done
 }
 
+function _create_container_with_health_log_settings {
+    local ctrname="$1"
+    local msg="$2"
+    local format="$3"
+    local flag="$4"
+    local expect="$5"
+    local expect_msg="$6"
+
+    run_podman run -d --name $ctrname   \
+               --health-cmd "echo $msg" \
+               $flag                    \
+               $IMAGE /home/podman/pause
+    cid="$output"
+
+    run_podman inspect $ctrname --format $format
+    is "$output" "$expect" "$expect_msg"
+
+    output=$cid
+}
+
+function _check_health_log {
+    local ctrname="$1"
+    local expect_msg="$2"
+    local comparison=$3
+    local expect_count="$4"
+
+    run_podman inspect $ctrname --format "{{.State.Health.Log}}"
+    count=$(grep -co "$expect_msg" <<< "$output")
+    assert "$count" $comparison $expect_count "Number of matching health log messages"
+}
+
+@test "podman healthcheck --health-max-log-count default value (5)" {
+    local msg="healthmsg-$(random_string)"
+    local ctrname="c-h-$(safename)"
+    _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthMaxLogCount}}" "" "5" "HealthMaxLogCount is the expected default"
+
+    for i in $(seq 1 10);
+    do
+        run_podman healthcheck run $ctrname
+        is "$output" "" "unexpected output from podman healthcheck run (pass $i)"
+    done
+
+    _check_health_log $ctrname $msg -eq 5
+
+    run_podman rm -t 0 -f $ctrname
+}
+
+@test "podman healthcheck --health-max-log-count infinite value (0)" {
+    local repeat_count=10
+    local msg="healthmsg-$(random_string)"
+    local ctrname="c-h-$(safename)"
+    _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthMaxLogCount}}" "--health-max-log-count 0" "0" "HealthMaxLogCount"
+
+    # This is run one more time than repeat_count to check that the cap is working.
+    for i in $(seq 1 $(($repeat_count + 1)));
+    do
+        run_podman healthcheck run $ctrname
+        is "$output" "" "unexpected output from podman healthcheck run (pass $i)"
+    done
+
+    # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd.
+    # And since `run_podman healthcheck run` is also run manually, it will result in two runs.
+    _check_health_log $ctrname $msg -ge 11
+
+    run_podman rm -t 0 -f $ctrname
+}
+
+
+@test "podman healthcheck --health-max-log-count 10" {
+    local repeat_count=10
+    local msg="healthmsg-$(random_string)"
+    local ctrname="c-h-$(safename)"
+    _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthMaxLogCount}}" "--health-max-log-count  $repeat_count" "$repeat_count" "HealthMaxLogCount"
+
+    # This is run one more time than repeat_count to check that the cap is working.
+    for i in $(seq 1 $(($repeat_count + 1)));
+    do
+        run_podman healthcheck run $ctrname
+        is "$output" "" "unexpected output from podman healthcheck run (pass $i)"
+    done
+
+    _check_health_log $ctrname $msg -eq $repeat_count
+
+    run_podman rm -t 0 -f $ctrname
+}
+
+@test "podman healthcheck --health-max-log-size 10" {
+    local msg="healthmsg-$(random_string)"
+    local ctrname="c-h-$(safename)"
+    _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthMaxLogSize}}" "--health-max-log-size 10" "10" "HealthMaxLogSize"
+
+    run_podman healthcheck run $ctrname
+    is "$output" "" "output from 'podman healthcheck run'"
+
+    local substr=${msg:0:10}
+    _check_health_log $ctrname "$substr}]\$" -eq 1
+
+    run_podman rm -t 0 -f $ctrname
+}
+
+@test "podman healthcheck --health-max-log-size infinite value (0)" {
+    local s=$(printf "healthmsg-%1000s")
+    local long_msg=${s// /$(random_string)}
+    local ctrname="c-h-$(safename)"
+    _create_container_with_health_log_settings $ctrname $long_msg "{{.Config.HealthMaxLogSize}}" "--health-max-log-size 0" "0" "HealthMaxLogSize"
+
+    run_podman healthcheck run $ctrname
+    is "$output" "" "output from 'podman healthcheck run'"
+
+    # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd.
+    # And since `run_podman healthcheck run` is also run manually, it will result in two runs.
+    _check_health_log $ctrname "$long_msg" -ge 1
+
+    run_podman rm -t 0 -f $ctrname
+}
+
+@test "podman healthcheck --health-max-log-size default value (500)" {
+    local s=$(printf "healthmsg-%1000s")
+    local long_msg=${s// /$(random_string)}
+    local ctrname="c-h-$(safename)"
+    _create_container_with_health_log_settings $ctrname $long_msg "{{.Config.HealthMaxLogSize}}" "" "500" "HealthMaxLogSize is the expected default"
+
+    run_podman healthcheck run $ctrname
+    is "$output" "" "output from 'podman healthcheck run'"
+
+    local expect_msg="${long_msg:0:500}"
+    _check_health_log $ctrname "$expect_msg}]\$" -eq 1
+
+    run_podman rm -t 0 -f $ctrname
+}
+
+
+@test "podman healthcheck --health-log-destination file" {
+    local TMP_DIR_HEALTHCHECK="$PODMAN_TMPDIR/healthcheck"
+    mkdir $TMP_DIR_HEALTHCHECK
+    local ctrname="c-h-$(safename)"
+    local msg="healthmsg-$(random_string)"
+    _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthLogDestination}}" "--health-log-destination $TMP_DIR_HEALTHCHECK" "$TMP_DIR_HEALTHCHECK" "HealthLogDestination"
+    cid="$output"
+
+    run_podman healthcheck run $ctrname
+    is "$output" "" "output from 'podman healthcheck run'"
+
+    healthcheck_log_path="${TMP_DIR_HEALTHCHECK}/${cid}-healthcheck.log"
+    # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd.
+    # And since `run_podman healthcheck run` is also run manually, it will result in two runs.
+    count=$(grep -co "$msg" $healthcheck_log_path)
+    assert "$count" -ge 1 "Number of matching health log messages"
+
+    run_podman rm -t 0 -f $ctrname
+}
+
+
+@test "podman healthcheck --health-log-destination journal" {
+    skip_if_remote "We cannot read journalctl over remote."
+
+    # We can't use journald on RHEL as rootless, either: rhbz#1895105
+    skip_if_journald_unavailable
+
+    local ctrname="c-h-$(safename)"
+    local msg="healthmsg-$(random_string)"
+    _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthLogDestination}}" "--health-log-destination events_logger" "events_logger" "HealthLogDestination"
+    cid="$output"
+
+    run_podman healthcheck run $ctrname
+    is "$output" "" "output from 'podman healthcheck run'"
+
+    cmd="journalctl --output cat --output-fields=PODMAN_HEALTH_LOG PODMAN_ID=$cid"
+    echo "$_LOG_PROMPT $cmd"
+    run $cmd
+    echo "$output"
+    assert "$status" -eq 0 "exit status of journalctl"
+
+    # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd.
+    # And since `run_podman healthcheck run` is also run manually, it will result in two runs.
+    count=$(grep -co "$msg" <<< "$output")
+    assert "$count" -ge 1 "Number of matching health log messages"
+
+    run_podman rm -t 0 -f $ctrname
+}
+
 # vim: filetype=sh