Merge pull request #15687 from vrothberg/RUN-1639

health check: add on-failure actions

cmd/podman/common/completion.go

@@ -1641,3 +1641,8 @@ func AutocompleteSSH(cmd *cobra.Command, args []string, toComplete string) ([]st
 	}
 	return []string{string(ssh.GolangMode), string(ssh.NativeMode)}, cobra.ShellCompDirectiveNoFileComp
 }
+
+// AutocompleteHealthOnFailure - action to take once the container turns unhealthy.
+func AutocompleteHealthOnFailure(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	return define.SupportedHealthCheckOnFailureActions, cobra.ShellCompDirectiveNoFileComp
+}

cmd/podman/common/create.go

@@ -208,6 +208,14 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
 		)
 		_ = cmd.RegisterFlagCompletionFunc(healthTimeoutFlagName, completion.AutocompleteNone)
 
+		healthOnFailureFlagName := "health-on-failure"
+		createFlags.StringVar(
+			&cf.HealthOnFailure,
+			healthOnFailureFlagName, "none",
+			"action to take once the container turns unhealthy",
+		)
+		_ = cmd.RegisterFlagCompletionFunc(healthOnFailureFlagName, AutocompleteHealthOnFailure)
+
 		createFlags.BoolVar(
 			&cf.HTTPProxy,
 			"http-proxy", containerConfig.Containers.HTTPProxy,

docs/source/markdown/options/health-on-failure.md

@@ -0,0 +1,8 @@
+#### **--health-on-failure**=*action*
+
+Action to take once the container transitions to an unhealthy state.  The default is **none**.
+
+- **none**: Take no action.
+- **kill**: Kill the container.
+- **restart**: Restart the container.  Do not combine the `restart` action with the `--restart` flag.  When running inside of a systemd unit, consider using the `kill` or `stop` action instead to make use of systemd's restart policy.
+- **stop**: Stop the container.

docs/source/markdown/podman-create.1.md.in

@@ -185,6 +185,8 @@ Read in a line delimited file of environment variables. See **Environment** note
 
 @@option health-interval
 
+@@option health-on-failure
+
 @@option health-retries
 
 @@option health-start-period

docs/source/markdown/podman-run.1.md.in

@@ -221,6 +221,8 @@ Read in a line delimited file of environment variables. See **Environment** note
 
 @@option health-interval
 
+@@option health-on-failure
+
 @@option health-retries
 
 @@option health-start-period

libpod/container_config.go

@@ -7,6 +7,7 @@ import (
 	"github.com/containers/common/libnetwork/types"
 	"github.com/containers/common/pkg/secrets"
 	"github.com/containers/image/v5/manifest"
+	"github.com/containers/podman/v4/libpod/define"
 	"github.com/containers/podman/v4/pkg/namespaces"
 	"github.com/containers/podman/v4/pkg/specgen"
 	"github.com/containers/storage"
@@ -392,6 +393,8 @@ type ContainerMiscConfig struct {
 	Systemd *bool `json:"systemd,omitempty"`
 	// HealthCheckConfig has the health check command and related timings
 	HealthCheckConfig *manifest.Schema2HealthConfig `json:"healthcheck"`
+	// HealthCheckOnFailureAction defines an action to take once the container turns unhealthy.
+	HealthCheckOnFailureAction define.HealthCheckOnFailureAction `json:"healthcheck_on_failure_action"`
 	// PreserveFDs is a number of additional file descriptors (in addition
 	// to 0, 1, 2) that will be passed to the executed process. The total FDs
 	// passed will be 3 + PreserveFDs.

libpod/container_inspect.go

@@ -390,6 +390,8 @@ func (c *Container) generateInspectContainerConfig(spec *spec.Spec) *define.Insp
 	// leak.
 	ctrConfig.Healthcheck = c.config.HealthCheckConfig
 
+	ctrConfig.HealthcheckOnFailureAction = c.config.HealthCheckOnFailureAction.String()
+
 	ctrConfig.CreateCommand = c.config.CreateCommand
 
 	ctrConfig.Timezone = c.config.Timezone

libpod/container_validate.go

@@ -137,5 +137,9 @@ func (c *Container) validate() error {
 	if c.config.SdNotifyMode == define.SdNotifyModeIgnore && len(c.config.SdNotifySocket) > 0 {
 		return fmt.Errorf("cannot set sd-notify socket %q with sd-notify mode %q", c.config.SdNotifySocket, c.config.SdNotifyMode)
 	}
+
+	if c.config.HealthCheckOnFailureAction != define.HealthCheckOnFailureActionNone && c.config.HealthCheckConfig == nil {
+		return fmt.Errorf("cannot set on-failure action to %s without a health check", c.config.HealthCheckOnFailureAction.String())
+	}
 	return nil
 }

libpod/define/container_inspect.go

@@ -55,6 +55,8 @@ type InspectContainerConfig struct {
 	StopSignal uint `json:"StopSignal"`
 	// Configured healthcheck for the container
 	Healthcheck *manifest.Schema2HealthConfig `json:"Healthcheck,omitempty"`
+	// HealthcheckOnFailureAction defines an action to take once the container turns unhealthy.
+	HealthcheckOnFailureAction string `json:"HealthcheckOnFailureAction,omitempty"`
 	// CreateCommand is the full command plus arguments of the process the
 	// container has been created with.
 	CreateCommand []string `json:"CreateCommand,omitempty"`

libpod/define/healthchecks.go

@@ -1,5 +1,10 @@
 package define
 
+import (
+	"fmt"
+	"strings"
+)
+
 const (
 	// HealthCheckHealthy describes a healthy container
 	HealthCheckHealthy string = "healthy"
@@ -57,3 +62,72 @@ const (
 	// HealthConfigTestCmdShell runs commands with the system's default shell
 	HealthConfigTestCmdShell = "CMD-SHELL"
 )
+
+// HealthCheckOnFailureAction defines how Podman reacts when a container's health
+// status turns unhealthy.
+type HealthCheckOnFailureAction int
+
+// Healthcheck on-failure actions.
+const (
+	// HealthCheckOnFailureActionNonce instructs Podman to not react on an unhealthy status.
+	HealthCheckOnFailureActionNone = iota // Must be first iota for backwards compatibility
+	// HealthCheckOnFailureActionInvalid denotes an invalid on-failure policy.
+	HealthCheckOnFailureActionInvalid = iota
+	// HealthCheckOnFailureActionNonce instructs Podman to kill the container on an unhealthy status.
+	HealthCheckOnFailureActionKill = iota
+	// HealthCheckOnFailureActionNonce instructs Podman to restart the container on an unhealthy status.
+	HealthCheckOnFailureActionRestart = iota
+	// HealthCheckOnFailureActionNonce instructs Podman to stop the container on an unhealthy status.
+	HealthCheckOnFailureActionStop = iota
+)
+
+// String representations for on-failure actions.
+const (
+	strHealthCheckOnFailureActionNone    = "none"
+	strHealthCheckOnFailureActionInvalid = "invalid"
+	strHealthCheckOnFailureActionKill    = "kill"
+	strHealthCheckOnFailureActionRestart = "restart"
+	strHealthCheckOnFailureActionStop    = "stop"
+)
+
+// SupportedHealthCheckOnFailureActions lists all supported healthcheck restart policies.
+var SupportedHealthCheckOnFailureActions = []string{
+	strHealthCheckOnFailureActionNone,
+	strHealthCheckOnFailureActionKill,
+	strHealthCheckOnFailureActionRestart,
+	strHealthCheckOnFailureActionStop,
+}
+
+// String returns the string representation of the HealthCheckOnFailureAction.
+func (h HealthCheckOnFailureAction) String() string {
+	switch h {
+	case HealthCheckOnFailureActionNone:
+		return strHealthCheckOnFailureActionNone
+	case HealthCheckOnFailureActionKill:
+		return strHealthCheckOnFailureActionKill
+	case HealthCheckOnFailureActionRestart:
+		return strHealthCheckOnFailureActionRestart
+	case HealthCheckOnFailureActionStop:
+		return strHealthCheckOnFailureActionStop
+	default:
+		return strHealthCheckOnFailureActionInvalid
+	}
+}
+
+// ParseHealthCheckOnFailureAction parses the specified string into a HealthCheckOnFailureAction.
+// An error is returned for an invalid input.
+func ParseHealthCheckOnFailureAction(s string) (HealthCheckOnFailureAction, error) {
+	switch s {
+	case "", strHealthCheckOnFailureActionNone:
+		return HealthCheckOnFailureActionNone, nil
+	case strHealthCheckOnFailureActionKill:
+		return HealthCheckOnFailureActionKill, nil
+	case strHealthCheckOnFailureActionRestart:
+		return HealthCheckOnFailureActionRestart, nil
+	case strHealthCheckOnFailureActionStop:
+		return HealthCheckOnFailureActionStop, nil
+	default:
+		err := fmt.Errorf("invalid on-failure action %q for health check: supported actions are %s", s, strings.Join(SupportedHealthCheckOnFailureActions, ","))
+		return HealthCheckOnFailureActionInvalid, err
+	}
+}

libpod/healthcheck.go

@@ -2,6 +2,7 @@ package libpod
 
 import (
 	"bufio"
+	"context"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -12,6 +13,7 @@ import (
 
 	"github.com/containers/podman/v4/libpod/define"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -29,9 +31,14 @@ func (r *Runtime) HealthCheck(name string) (define.HealthCheckStatus, error) {
 	if err != nil {
 		return define.HealthCheckContainerNotFound, fmt.Errorf("unable to look up %s to perform a health check: %w", name, err)
 	}
+
 	hcStatus, err := checkHealthCheckCanBeRun(container)
 	if err == nil {
-		return container.runHealthCheck()
+		hcStatus, err := container.runHealthCheck()
+		if err := container.processHealthCheckStatus(hcStatus); err != nil {
+			return hcStatus, err
+		}
+		return hcStatus, err
 	}
 	return hcStatus, err
 }
@@ -127,13 +134,45 @@ func (c *Container) runHealthCheck() (define.HealthCheckStatus, error) {
 		hcResult = define.HealthCheckFailure
 		hcErr = fmt.Errorf("healthcheck command exceeded timeout of %s", c.HealthCheckConfig().Timeout.String())
 	}
+
 	hcl := newHealthCheckLog(timeStart, timeEnd, returnCode, eventLog)
 	if err := c.updateHealthCheckLog(hcl, inStartPeriod); err != nil {
 		return hcResult, fmt.Errorf("unable to update health check log %s for %s: %w", c.healthCheckLogPath(), c.ID(), err)
 	}
+
 	return hcResult, hcErr
 }
 
+func (c *Container) processHealthCheckStatus(status define.HealthCheckStatus) error {
+	if status == define.HealthCheckSuccess {
+		return nil
+	}
+
+	switch c.config.HealthCheckOnFailureAction {
+	case define.HealthCheckOnFailureActionNone: // Nothing to do
+
+	case define.HealthCheckOnFailureActionKill:
+		if err := c.Kill(uint(unix.SIGKILL)); err != nil {
+			return fmt.Errorf("killing container health-check turned unhealthy: %w", err)
+		}
+
+	case define.HealthCheckOnFailureActionRestart:
+		if err := c.RestartWithTimeout(context.Background(), c.config.StopTimeout); err != nil {
+			return fmt.Errorf("restarting container after health-check turned unhealthy: %w", err)
+		}
+
+	case define.HealthCheckOnFailureActionStop:
+		if err := c.Stop(); err != nil {
+			return fmt.Errorf("stopping container after health-check turned unhealthy: %w", err)
+		}
+
+	default: // Should not happen but better be safe than sorry
+		return fmt.Errorf("unsupported on-failure action %d", c.config.HealthCheckOnFailureAction)
+	}
+
+	return nil
+}
+
 func checkHealthCheckCanBeRun(c *Container) (define.HealthCheckStatus, error) {
 	cstate, err := c.State()
 	if err != nil {

libpod/options.go

@@ -1473,6 +1473,17 @@ func WithHealthCheck(healthCheck *manifest.Schema2HealthConfig) CtrCreateOption
 	}
 }
 
+// WithHealthCheckOnFailureAction adds an on-failure action to health-check config
+func WithHealthCheckOnFailureAction(action define.HealthCheckOnFailureAction) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+		ctr.config.HealthCheckOnFailureAction = action
+		return nil
+	}
+}
+
 // WithPreserveFDs forwards from the process running Libpod into the container
 // the given number of extra FDs (starting after the standard streams) to the created container
 func WithPreserveFDs(fd uint) CtrCreateOption {

pkg/domain/entities/pods.go

@@ -212,6 +212,7 @@ type ContainerCreateOptions struct {
 	HealthRetries     uint
 	HealthStartPeriod string
 	HealthTimeout     string
+	HealthOnFailure   string
 	Hostname          string `json:"hostname,omitempty"`
 	HTTPProxy         bool
 	HostUsers         []string

pkg/specgen/generate/container_create.go

@@ -515,6 +515,10 @@ func createContainerOptions(rt *libpod.Runtime, s *specgen.SpecGenerator, pod *l
 		logrus.Debugf("New container has a health check")
 	}
 
+	if s.ContainerHealthCheckConfig.HealthCheckOnFailureAction != define.HealthCheckOnFailureActionNone {
+		options = append(options, libpod.WithHealthCheckOnFailureAction(s.ContainerHealthCheckConfig.HealthCheckOnFailureAction))
+	}
+
 	if len(s.Secrets) != 0 {
 		manager, err := rt.SecretsManager()
 		if err != nil {

pkg/specgen/specgen.go

@@ -9,6 +9,7 @@ import (
 	"github.com/containers/common/libimage"
 	nettypes "github.com/containers/common/libnetwork/types"
 	"github.com/containers/image/v5/manifest"
+	"github.com/containers/podman/v4/libpod/define"
 	"github.com/containers/storage/types"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -533,7 +534,8 @@ type ContainerResourceConfig struct {
 // ContainerHealthCheckConfig describes a container healthcheck with attributes
 // like command, retries, interval, start period, and timeout.
 type ContainerHealthCheckConfig struct {
-	HealthConfig *manifest.Schema2HealthConfig `json:"healthconfig,omitempty"`
+	HealthConfig               *manifest.Schema2HealthConfig     `json:"healthconfig,omitempty"`
+	HealthCheckOnFailureAction define.HealthCheckOnFailureAction `json:"health_check_on_failure_action,omitempty"`
 }
 
 // SpecGenerator creates an OCI spec and Libpod configuration options to create

pkg/specgenutil/specgen.go

@@ -265,6 +265,13 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 			Test: []string{"NONE"},
 		}
 	}
+
+	onFailureAction, err := define.ParseHealthCheckOnFailureAction(c.HealthOnFailure)
+	if err != nil {
+		return err
+	}
+	s.HealthCheckOnFailureAction = onFailureAction
+
 	if err := setNamespaces(s, c); err != nil {
 		return err
 	}

test/system/220-healthcheck.bats

@@ -20,44 +20,8 @@ function _check_health {
     done
 }
 
-
 @test "podman healthcheck" {
-    # Create an image with a healthcheck script; said script will
-    # pass until the file /uh-oh gets created (by us, via exec)
-    cat >${PODMAN_TMPDIR}/healthcheck <<EOF
-#!/bin/sh
-
-if test -e /uh-oh; then
-    echo "Uh-oh on stdout!"
-    echo "Uh-oh on stderr!" >&2
-    exit 1
-else
-    echo "Life is Good on stdout"
-    echo "Life is Good on stderr" >&2
-    exit 0
-fi
-EOF
-
-    cat >${PODMAN_TMPDIR}/entrypoint <<EOF
-#!/bin/sh
-
-while :; do
-    sleep 1
-done
-EOF
-
-    cat >${PODMAN_TMPDIR}/Containerfile <<EOF
-FROM $IMAGE
-
-COPY healthcheck /healthcheck
-COPY entrypoint  /entrypoint
-
-RUN  chmod 755 /healthcheck /entrypoint
-
-CMD ["/entrypoint"]
-EOF
-
-    run_podman build -t healthcheck_i ${PODMAN_TMPDIR}
+    _build_health_check_image healthcheck_i
 
     # Run that healthcheck image.
     run_podman run -d --name healthcheck_c \
@@ -66,6 +30,9 @@ EOF
                --health-retries 3          \
                healthcheck_i
 
+    run_podman inspect healthcheck_c --format "{{.Config.HealthcheckOnFailureAction}}"
+    is "$output" "none" "default on-failure action is none"
+
     # We can't check for 'starting' because a 1-second interval is too
     # short; it could run healthcheck before we get to our first check.
     #
@@ -109,4 +76,59 @@ Log[-1].Output   | \"Uh-oh on stdout!\\\nUh-oh on stderr!\"
     run_podman rmi   healthcheck_i
 }
 
+@test "podman healthcheck --health-on-failure" {
+    run_podman 125 create --health-on-failure=kill $IMAGE
+    is "$output" "Error: cannot set on-failure action to kill without a health check"
+
+    ctr="healthcheck_c"
+    img="healthcheck_i"
+
+    for policy in none kill restart stop;do
+	if [[ $policy == "none" ]];then
+	    # Do not remove the /uh-oh file for `none` as we want to
+	    # demonstrate that no action was taken
+            _build_health_check_image $img
+        else
+            _build_health_check_image $img cleanfile
+        fi
+
+        # Run that healthcheck image.
+        run_podman run -d --name $ctr      \
+               --health-cmd /healthcheck   \
+               --health-on-failure=$policy \
+               $img
+
+        # healthcheck should succeed
+        run_podman healthcheck run $ctr
+
+        # Now cause the healthcheck to fail
+        run_podman exec $ctr touch /uh-oh
+
+        # healthcheck should now fail, with exit status 1 and 'unhealthy' output
+        run_podman 1 healthcheck run $ctr
+	# FIXME: #15691 - `healthcheck run` may emit an error log that the timer already exists
+        is "$output" ".*unhealthy.*" "output from 'podman healthcheck run'"
+
+        run_podman inspect $ctr --format "{{.State.Status}} {{.Config.HealthcheckOnFailureAction}}"
+	if [[ $policy == "restart" ]];then
+	    # Container has been restarted and health check works again
+            is "$output" "running $policy" "container has been restarted"
+            run_podman healthcheck run $ctr
+        elif [[ $policy == "none" ]];then
+            # Container is still running and health check still broken
+            is "$output" "running $policy" "container continued running"
+            run_podman 1 healthcheck run $ctr
+	    # FIXME: #15691 - `healthcheck run` may emit an error log that the timer already exists
+            is "$output" ".*unhealthy.*" "output from 'podman healthcheck run'"
+	else
+	    # kill and stop yield the container into a non-running state
+            is "$output" ".* $policy" "container was stopped/killed"
+            assert "$output" != "running $policy"
+        fi
+
+        run_podman rm -f -t0 $ctr
+        run_podman rmi -f $img
+    done
+}
+
 # vim: filetype=sh

test/system/250-systemd.bats

@@ -304,6 +304,57 @@ LISTEN_FDNAMES=listen_fdnames" | sort)
     run_podman network rm -f $netname
 }
 
+@test "podman create --health-on-failure=kill" {
+    img="healthcheck_i"
+    _build_health_check_image $img
+
+    cname=$(random_string)
+    run_podman create --name $cname      \
+               --health-cmd /healthcheck \
+               --health-on-failure=kill  \
+               --restart=on-failure      \
+               $img
+
+    # run container in systemd unit
+    service_setup
+
+    run_podman container inspect $cname --format "{{.ID}}"
+    oldID="$output"
+
+    run_podman healthcheck run $cname
+
+    # Now cause the healthcheck to fail
+    run_podman exec $cname touch /uh-oh
+
+    # healthcheck should now fail, with exit status 1 and 'unhealthy' output
+    run_podman 1 healthcheck run $cname
+    is "$output" "unhealthy" "output from 'podman healthcheck run'"
+
+    # What is expected to happen now:
+    #  1) The container gets killed as the health check has failed
+    #  2) Systemd restarts the service as the restart policy is set to "on-failure"
+    #  3) The /uh-oh file is gone and $cname has another ID
+
+    # Wait at most 10 seconds for the service to be restarted
+    local timeout=10
+    while [[ $timeout -gt 1 ]]; do
+        run_podman '?' container inspect $cname
+        if [[ $status == 0 ]]; then
+            if [[ "$output" != "$oldID" ]]; then
+                break
+            fi
+        fi
+        sleep 1
+        let timeout=$timeout-1
+    done
+
+    run_podman healthcheck run $cname
+
+    # stop systemd container
+    service_cleanup
+    run_podman rmi -f $img
+}
+
 @test "podman-kube@.service template" {
     install_kube_template
     # Create the YAMl file

test/system/helpers.bash

@@ -894,5 +894,59 @@ function _podman_commands() {
     awk '/^Available Commands:/{ok=1;next}/^Options:/{ok=0}ok { print $1 }' <<<"$output" | grep .
 }
 
+###############################
+#  _build_health_check_image  #  Builds a container image with a configured health check
+###############################
+#
+# The health check will fail once the /uh-oh file exists.
+#
+# First argument is the desired name of the image
+# Second argument, if present and non-null, forces removal of the /uh-oh file once the check failed; this way the container can be restarted
+#
+
+function _build_health_check_image {
+    local imagename="$1"
+    local cleanfile=""
+
+    if [[ ! -z "$2" ]]; then
+        cleanfile="rm -f /uh-oh"
+    fi
+    # Create an image with a healthcheck script; said script will
+    # pass until the file /uh-oh gets created (by us, via exec)
+    cat >${PODMAN_TMPDIR}/healthcheck <<EOF
+#!/bin/sh
+
+if test -e /uh-oh; then
+    echo "Uh-oh on stdout!"
+    echo "Uh-oh on stderr!" >&2
+    ${cleanfile}
+    exit 1
+else
+    echo "Life is Good on stdout"
+    echo "Life is Good on stderr" >&2
+    exit 0
+fi
+EOF
+
+    cat >${PODMAN_TMPDIR}/entrypoint <<EOF
+#!/bin/sh
+
+trap 'echo Received SIGTERM, finishing; exit' SIGTERM; echo WAITING; while :; do sleep 0.1; done
+EOF
+
+    cat >${PODMAN_TMPDIR}/Containerfile <<EOF
+FROM $IMAGE
+
+COPY healthcheck /healthcheck
+COPY entrypoint  /entrypoint
+
+RUN  chmod 755 /healthcheck /entrypoint
+
+CMD ["/entrypoint"]
+EOF
+
+    run_podman build -t $imagename ${PODMAN_TMPDIR}
+}
+
 # END   miscellaneous tools
 ###############################################################################