libpod: Implement --log-opt label=LABEL=Value

This allows things like compose project names to be associated with log
messages and later used in log processing and analysis.

Signed-off-by: Povilas Kanapickas <povilas@radix.lt>

cmd/podman/common/completion.go

@@ -1169,7 +1169,7 @@ func AutocompleteLogDriver(_ *cobra.Command, _ []string, _ string) ([]string, co
 // AutocompleteLogOpt - Autocomplete log-opt options.
 // -> "path=", "tag="
 func AutocompleteLogOpt(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	logOptions := []string{"path=", "tag=", "max-size="}
+	logOptions := []string{"path=", "tag=", "max-size=", "label="}
 	if strings.HasPrefix(toComplete, "path=") {
 		return nil, cobra.ShellCompDirectiveDefault
 	}

docs/source/markdown/options/log-opt.md

@@ -18,3 +18,9 @@ Set custom logging configuration. The following *name*s are supported:
     (e.g. **--log-opt tag="{{.ImageName}}"**.
 It supports the same keys as **podman inspect --format**.
 This option is currently supported only by the **journald** log driver.
+
+**label**: specify a custom log label for the container
+    (e.g. **--log-opt label="CONTAINER_IMAGE={{.ImageName}}"**.
+It supports the same keys as **podman inspect --format**.
+This option can be repeated multiple times.
+This option is currently supported only by the **journald** log driver.

libpod/container.go

@@ -676,6 +676,11 @@ func (c *Container) LogSizeMax() int64 {
 	return c.runtime.config.Containers.LogSizeMax
 }
 
+// LogLabels returns the labels added to the container's log file
+func (c *Container) LogLabels() map[string]string {
+	return c.config.LogLabels
+}
+
 // RestartPolicy returns the container's restart policy.
 func (c *Container) RestartPolicy() string {
 	return c.config.RestartPolicy

libpod/container_config.go

@@ -369,6 +369,8 @@ type ContainerMiscConfig struct {
 	LogPath string `json:"logPath"`
 	// LogTag is the tag used for logging
 	LogTag string `json:"logTag"`
+	// LogLabels is a set of key-value pairs used to label log messages
+	LogLabels map[string]string `json:"logLabels,omitempty"`
 	// LogSize is the maximum size of the container's log file
 	LogSize int64 `json:"logSize"`
 	// LogDriver driver for logs

libpod/oci_conmon_common.go

@@ -36,6 +36,7 @@ import (
 	"go.podman.io/common/pkg/resize"
 	"go.podman.io/common/pkg/version"
 	"go.podman.io/storage/pkg/idtools"
+	"go.podman.io/storage/pkg/regexp"
 	"golang.org/x/sys/unix"
 )
 
@@ -923,26 +924,51 @@ func waitPidStop(pid int, timeout time.Duration) error {
 	}
 }
 
-func (r *ConmonOCIRuntime) getLogTag(ctr *Container) (string, error) {
+func (r *ConmonOCIRuntime) getLogData(ctr *Container) (string, map[string]string, error) {
 	logTag := ctr.LogTag()
-	if logTag == "" {
-		return "", nil
+	logLabels := ctr.LogLabels()
+
+	// inspectLocked is expensive, skip it if possible
+	if logTag == "" && len(logLabels) == 0 {
+		return "", nil, nil
 	}
+
 	data, err := ctr.inspectLocked(false)
 	if err != nil {
 		// FIXME: this error should probably be returned
-		return "", nil //nolint: nilerr
+		return "", nil, nil //nolint: nilerr
 	}
-	tmpl, err := template.New("container").Parse(logTag)
-	if err != nil {
-		return "", fmt.Errorf("template parsing error %s: %w", logTag, err)
+
+	var parsedLogTag string
+	if logTag != "" {
+		tmpl, err := template.New("container").Parse(logTag)
+		if err != nil {
+			return "", nil, fmt.Errorf("template parsing error %s: %w", logTag, err)
+		}
+		var b bytes.Buffer
+		err = tmpl.Execute(&b, data)
+		if err != nil {
+			return "", nil, err
+		}
+		parsedLogTag = b.String()
 	}
-	var b bytes.Buffer
-	err = tmpl.Execute(&b, data)
-	if err != nil {
-		return "", err
+
+	parsedLogLabels := make(map[string]string)
+	for labelKey, labelValue := range logLabels {
+		tmpl, err := template.New("container").Parse(labelValue)
+		if err != nil {
+			return "", nil, fmt.Errorf("template parsing error %s (label %s): %w", labelValue, labelKey, err)
+		}
+
+		var b bytes.Buffer
+		err = tmpl.Execute(&b, data)
+		if err != nil {
+			return "", nil, err
+		}
+		parsedLogLabels[labelKey] = b.String()
 	}
-	return b.String(), nil
+
+	return parsedLogTag, parsedLogLabels, nil
 }
 
 func getPreserveFdExtraFiles(preserveFD []uint, preserveFDs uint) (uint, []*os.File, []*os.File, error) {
@@ -1000,7 +1026,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
 		ociLog = filepath.Join(ctr.state.RunDir, "oci-log")
 	}
 
-	logTag, err := r.getLogTag(ctr)
+	logTag, logLabels, err := r.getLogData(ctr)
 	if err != nil {
 		return 0, err
 	}
@@ -1017,7 +1043,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
 	}
 
 	persistDir := filepath.Join(r.persistDir, ctr.ID())
-	args, err := r.sharedConmonArgs(ctr, ctr.ID(), ctr.bundlePath(), pidfile, ctr.LogPath(), r.exitsDir, persistDir, ociLog, ctr.LogDriver(), logTag)
+	args, err := r.sharedConmonArgs(ctr, ctr.ID(), ctr.bundlePath(), pidfile, ctr.LogPath(), r.exitsDir, persistDir, ociLog, ctr.LogDriver(), logTag, logLabels)
 	if err != nil {
 		return 0, err
 	}
@@ -1283,9 +1309,15 @@ func (r *ConmonOCIRuntime) configureConmonEnv() ([]string, error) {
 	return res, nil
 }
 
+var journaldFieldNameRegexp = regexp.Delayed(`^[A-Z0-9_]+$`)
+
+func validJournaldFieldName(s string) bool {
+	return journaldFieldNameRegexp.MatchString(s)
+}
+
 // sharedConmonArgs takes common arguments for exec and create/restore and formats them for the conmon CLI
-// func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, pidPath, logPath, exitDir, persistDir, ociLogPath, logDriver, logTag string) ([]string, error) {
-func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, pidPath, logPath, exitDir, persistDir, ociLogPath, logDriver, logTag string) ([]string, error) {
+// func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, pidPath, logPath, exitDir, persistDir, ociLogPath, logDriver, logTag string, logLabels map[string]string) ([]string, error) {
+func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, pidPath, logPath, exitDir, persistDir, ociLogPath, logDriver, logTag string, logLabels map[string]string) ([]string, error) {
 	// Make the persists directory for the container after the ctr ID is appended to it in the caller
 	// This is needed as conmon writes the exit and oom file in the given persist directory path as just "exit" and "oom"
 	// So creating a directory with the container ID under the persist dir will help keep track of which container the
@@ -1360,6 +1392,17 @@ func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, p
 	if logTag != "" {
 		args = append(args, "--log-tag", logTag)
 	}
+	if logDriverArg == define.JournaldLogging {
+		for label, value := range logLabels {
+			if !validJournaldFieldName(label) {
+				return nil, fmt.Errorf("log label %q contains invalid characters, only uppercase letters, digits, and underscores are allowed", label)
+			}
+			args = append(args, "--log-label", fmt.Sprintf("%s=%s", label, value))
+		}
+	} else if len(logLabels) > 0 {
+		return nil, fmt.Errorf("log labels can only be used with journald log driver")
+	}
+
 	if ctr.config.NoCgroups {
 		logrus.Debugf("Running with no Cgroups")
 		args = append(args, "--runtime-arg", "--cgroup-manager", "--runtime-arg", "disabled")

libpod/oci_conmon_exec_common.go

@@ -399,7 +399,7 @@ func (r *ConmonOCIRuntime) startExec(c *Container, sessionID string, options *Ex
 	}
 	defer processFile.Close()
 
-	args, err := r.sharedConmonArgs(c, sessionID, c.execBundlePath(sessionID), c.execPidPath(sessionID), c.execLogPath(sessionID), c.execExitFileDir(sessionID), c.execPersistDir(sessionID), ociLog, define.NoLogging, c.config.LogTag)
+	args, err := r.sharedConmonArgs(c, sessionID, c.execBundlePath(sessionID), c.execPidPath(sessionID), c.execLogPath(sessionID), c.execExitFileDir(sessionID), c.execPersistDir(sessionID), ociLog, define.NoLogging, c.config.LogTag, nil)
 	if err != nil {
 		return nil, nil, err
 	}

libpod/options.go

@@ -1046,6 +1046,19 @@ func WithLogTag(tag string) CtrCreateOption {
 	}
 }
 
+// WithLogLabels sets the labels to the log file.
+func WithLogLabels(logLabels map[string]string) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+
+		ctr.config.LogLabels = logLabels
+
+		return nil
+	}
+}
+
 // WithCgroupsMode disables the creation of Cgroups for the conmon process.
 func WithCgroupsMode(mode string) CtrCreateOption {
 	return func(ctr *Container) error {

pkg/specgen/generate/container_create.go

@@ -565,7 +565,9 @@ func createContainerOptions(rt *libpod.Runtime, s *specgen.SpecGenerator, pod *l
 		if len(s.LogConfiguration.Options) > 0 && s.LogConfiguration.Options["tag"] != "" {
 			options = append(options, libpod.WithLogTag(s.LogConfiguration.Options["tag"]))
 		}
-
+		if len(s.LogConfiguration.Labels) > 0 {
+			options = append(options, libpod.WithLogLabels(s.LogConfiguration.Labels))
+		}
 		if len(s.LogConfiguration.Driver) > 0 {
 			options = append(options, libpod.WithLogDriver(s.LogConfiguration.Driver))
 		}

pkg/specgen/generate/kube/kube.go

@@ -247,6 +247,7 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener
 	}
 
 	s.LogConfiguration.Options = make(map[string]string)
+	s.LogConfiguration.Labels = make(map[string]string)
 	for _, o := range opts.LogOptions {
 		opt, val, hasVal := strings.Cut(o, "=")
 		if !hasVal {
@@ -263,6 +264,12 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener
 				return nil, err
 			}
 			s.LogConfiguration.Size = logSize
+		case "label":
+			labelKey, labelVal, hasVal := strings.Cut(val, "=")
+			if !hasVal {
+				return nil, fmt.Errorf("invalid log label %q", o)
+			}
+			s.LogConfiguration.Labels[labelKey] = labelVal
 		default:
 			if len(val) == 0 {
 				return nil, fmt.Errorf("invalid log option: %w", define.ErrInvalidArg)

pkg/specgen/specgen.go

@@ -26,6 +26,10 @@ type LogConfig struct {
 	// Size is the maximum size of the log file
 	// Optional.
 	Size int64 `json:"size,omitempty"`
+	// A set of log labels to apply
+	// Only available if LogDriver is set to "journald".
+	// Optional
+	Labels map[string]string `json:"labels,omitempty"`
 	// A set of options to accompany the log driver.
 	// Optional.
 	Options map[string]string `json:"options,omitempty"`

pkg/specgenutil/specgen.go

@@ -849,6 +849,7 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 	}
 
 	logOpts := make(map[string]string)
+	logLabels := make(map[string]string)
 	for _, o := range c.LogOptions {
 		key, val, hasVal := strings.Cut(o, "=")
 		if !hasVal {
@@ -865,6 +866,12 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 				return err
 			}
 			s.LogConfiguration.Size = logSize
+		case "label":
+			labelKey, labelVal, hasVal := strings.Cut(val, "=")
+			if !hasVal {
+				return fmt.Errorf("invalid log label %q", o)
+			}
+			logLabels[labelKey] = labelVal
 		default:
 			logOpts[key] = val
 		}
@@ -873,6 +880,9 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 	if len(s.LogConfiguration.Options) == 0 || len(c.LogOptions) != 0 {
 		s.LogConfiguration.Options = logOpts
 	}
+	if len(s.LogConfiguration.Labels) == 0 || len(c.LogOptions) != 0 {
+		s.LogConfiguration.Labels = logLabels
+	}
 	if len(s.Name) == 0 || len(c.Name) != 0 {
 		s.Name = c.Name
 	}

test/e2e/common_test.go

@@ -29,6 +29,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/blang/semver/v4"
 	"github.com/containers/podman/v6/libpod/define"
 	"github.com/containers/podman/v6/pkg/inspect"
 	. "github.com/containers/podman/v6/test/utils"
@@ -1195,6 +1196,31 @@ func SkipIfInContainer(reason string) {
 	}
 }
 
+// SkipIfConmonVersionLessThan skips a test if the conmon version is less than
+// the specified minimum version (e.g., "2.2.0").
+func SkipIfConmonVersionLessThan(minVersion string) {
+	out, err := exec.Command(podmanTest.ConmonBinary, "--version").Output()
+	if err != nil {
+		Fail(fmt.Sprintf("[conmon]: failed to get conmon version: %v", err))
+	}
+	// Output format: "conmon version 2.2.0"
+	fields := strings.Fields(strings.TrimSpace(string(out)))
+	if len(fields) < 3 {
+		Fail(fmt.Sprintf("[conmon]: unexpected conmon --version output: %s", out))
+	}
+	current, err := semver.Parse(fields[2])
+	if err != nil {
+		Fail(fmt.Sprintf("[conmon]: failed to parse conmon version %q: %v", fields[2], err))
+	}
+	minVer, err := semver.Parse(minVersion)
+	if err != nil {
+		Fail(fmt.Sprintf("[conmon]: failed to parse minimum version %q: %v", minVersion, err))
+	}
+	if current.Compare(minVer) < 0 {
+		Skip(fmt.Sprintf("[conmon]: need conmon >= %s; have %s", minVersion, fields[2]))
+	}
+}
+
 // SkipIfNotActive skips a test if the given systemd unit is not active
 func SkipIfNotActive(unit string, reason string) {
 	checkReason(reason)

test/e2e/logs_test.go

@@ -598,6 +598,27 @@ var _ = Describe("Podman logs", func() {
 		}).Should(Succeed())
 	})
 
+	It("using journald labels", func() {
+		SkipIfJournaldUnavailable()
+		SkipIfConmonVersionLessThan("2.2.0")
+		containerName := "inside-journal"
+		logc := podmanTest.Podman([]string{"run", "--log-driver", "journald", "--log-opt", "label=CONTAINER_LABEL=LabelValue", "-d", "--name", containerName, ALPINE, "sh", "-c", "echo podman; sleep 0.1; echo podman; sleep 0.1; echo podman"})
+		logc.WaitWithDefaultTimeout()
+		Expect(logc).To(ExitCleanly())
+		cid := logc.OutputToString()
+
+		wait := podmanTest.Podman([]string{"wait", cid})
+		wait.WaitWithDefaultTimeout()
+		Expect(wait).To(ExitCleanly())
+
+		Eventually(func(g Gomega) {
+			cmd := exec.Command("journalctl", "--no-pager", "-o", "json", "--output-fields=CONTAINER_LABEL", fmt.Sprintf("CONTAINER_ID_FULL=%s", cid))
+			out, err := cmd.CombinedOutput()
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(string(out)).To(ContainSubstring("LabelValue"))
+		}).Should(Succeed())
+	})
+
 	It("podman logs with log-driver=none errors", func() {
 		ctrName := "logsctr"
 		logc := podmanTest.Podman([]string{"run", "--name", ctrName, "-d", "--log-driver", "none", ALPINE, "top"})

test/e2e/quadlet/logopt.container

@@ -1,9 +1,11 @@
 ## assert-podman-args "--log-opt" "path=/var/log/some-logs.json"
 ## assert-podman-args "--log-opt" "size=10mb"
 ## assert-podman-args "--log-opt" "tag="{{.ImageName}}""
+## assert-podman-args "--log-opt" "label=CONTAINER_LABEL=LabelValue"
 
 [Container]
 Image=localhost/imagename
 LogOpt=path=/var/log/some-logs.json
 LogOpt=size=10mb
 LogOpt=tag="{{.ImageName}}"
+LogOpt=label=CONTAINER_LABEL=LabelValue

test/e2e/quadlet/logopt.kube

@@ -1,9 +1,11 @@
 ## assert-podman-args "--log-opt" "path=/var/log/some-logs.json"
 ## assert-podman-args "--log-opt" "size=10mb"
 ## assert-podman-args "--log-opt" "tag="{{.ImageName}}""
+## assert-podman-args "--log-opt" "label=CONTAINER_LABEL=LabelValue"
 
 [Kube]
 Yaml=deployment.yml
 LogOpt=path=/var/log/some-logs.json
 LogOpt=size=10mb
 LogOpt=tag="{{.ImageName}}"
+LogOpt=label=CONTAINER_LABEL=LabelValue