Add volumes-from support using annotation in kube yaml

The reserved annotation io.podman.annotations.volumes-from is made public to let user define volumes-from to have one container mount volumes of other containers.

The annotation format is: io.podman.annotations.volumes-from/tgtCtr: "srcCtr1:mntOpts1;srcCtr2:mntOpts;..."

Fixes: containers#16819

Signed-off-by: Vikas Goel <vikas.goel@gmail.com>

docs/source/markdown/podman-kube-play.1.md.in

@@ -41,6 +41,8 @@ Note: *hostPath* volume types created by kube play is given an SELinux shared la
 
 Note: To set userns of a pod, use the **io.podman.annotations.userns** annotation in the pod/deployment definition. This can be overridden with the `--userns` flag.
 
+Note: Use the **io.podman.annotations.volumes-from** annotation to bind mount volumes of one container to another. You can mount volumes from multiple source containers to a target container. The source containers that belong to the same pod must be defined before the source container in the kube YAML. The annotation format is `io.podman.annotations.volumes-from/targetContainer: "sourceContainer1:mountOpts1;sourceContainer2:mountOpts2"`.
+
 Note: If the `:latest` tag is used, Podman attempts to pull the image from a registry. If the image was built locally with Podman or Buildah, it has `localhost` as the domain, in that case, Podman uses the image from the local store even if it has the `:latest` tag.
 
 Note: The command `podman play kube` is an alias of `podman kube play`, and performs the same function.

libpod/container_inspect.go

@@ -53,7 +53,7 @@ func (c *Container) volumesFrom() ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	if ctrs, ok := ctrSpec.Annotations[define.InspectAnnotationVolumesFrom]; ok {
+	if ctrs, ok := ctrSpec.Annotations[define.VolumesFromAnnotation]; ok {
 		return strings.Split(ctrs, ";"), nil
 	}
 	return nil, nil
@@ -510,7 +510,7 @@ func (c *Container) generateInspectContainerHostConfig(ctrSpec *spec.Spec, named
 		if ctrSpec.Annotations[define.InspectAnnotationAutoremove] == define.InspectResponseTrue {
 			hostConfig.AutoRemove = true
 		}
-		if ctrs, ok := ctrSpec.Annotations[define.InspectAnnotationVolumesFrom]; ok {
+		if ctrs, ok := ctrSpec.Annotations[define.VolumesFromAnnotation]; ok {
 			hostConfig.VolumesFrom = strings.Split(ctrs, ";")
 		}
 		if ctrSpec.Annotations[define.InspectAnnotationPrivileged] == define.InspectResponseTrue {

libpod/define/annotations.go

@@ -18,13 +18,6 @@ const (
 	// the two supported boolean values (InspectResponseTrue and
 	// InspectResponseFalse) it will be used in the output of Inspect().
 	InspectAnnotationAutoremove = "io.podman.annotations.autoremove"
-	// InspectAnnotationVolumesFrom is used by Inspect to identify
-	// containers whose volumes are being used by this container.
-	// It is expected to be a comma-separated list of container names and/or
-	// IDs.
-	// If an annotation with this key is found in the OCI spec, it will be
-	// used in the output of Inspect().
-	InspectAnnotationVolumesFrom = "io.podman.annotations.volumes-from"
 	// InspectAnnotationPrivileged is used by Inspect to identify containers
 	// which are privileged (IE, running with elevated privileges).
 	// It is expected to be a boolean, populated by one of
@@ -157,6 +150,12 @@ const (
 	// of the container
 	UlimitAnnotation = "io.podman.annotations.ulimit"
 
+	// VolumesFromAnnotation is used by by play kube when playing a kube
+	// yaml to specify volumes-from of the container
+	// It is expected to be a semicolon-separated list of container names and/or
+	// IDs optionally with colon separated mount options.
+	VolumesFromAnnotation = "io.podman.annotations.volumes-from"
+
 	// KubeHealthCheckAnnotation is used by kube play to tell podman that any health checks should follow
 	// the k8s behavior of waiting for the intialDelaySeconds to be over before updating the status
 	KubeHealthCheckAnnotation = "io.podman.annotations.kube.health.check"
@@ -169,7 +168,7 @@ const (
 // already reserved annotation that Podman sets during container creation.
 func IsReservedAnnotation(value string) bool {
 	switch value {
-	case InspectAnnotationCIDFile, InspectAnnotationAutoremove, InspectAnnotationVolumesFrom, InspectAnnotationPrivileged, InspectAnnotationPublishAll, InspectAnnotationInit, InspectAnnotationLabel, InspectAnnotationSeccomp, InspectAnnotationApparmor, InspectResponseTrue, InspectResponseFalse:
+	case InspectAnnotationCIDFile, InspectAnnotationAutoremove, InspectAnnotationPrivileged, InspectAnnotationPublishAll, InspectAnnotationInit, InspectAnnotationLabel, InspectAnnotationSeccomp, InspectAnnotationApparmor, InspectResponseTrue, InspectResponseFalse, VolumesFromAnnotation:
 		return true
 
 	default:

libpod/pod.go

@@ -277,7 +277,7 @@ func (p *Pod) VolumesFrom() []string {
 	if err != nil {
 		return nil
 	}
-	if ctrs, ok := infra.config.Spec.Annotations[define.InspectAnnotationVolumesFrom]; ok {
+	if ctrs, ok := infra.config.Spec.Annotations[define.VolumesFromAnnotation]; ok {
 		return strings.Split(ctrs, ";")
 	}
 	return nil

pkg/domain/infra/abi/play.go

@@ -124,6 +124,51 @@ func (ic *ContainerEngine) createServiceContainer(ctx context.Context, name stri
 	return ctr, nil
 }
 
+func prepareVolumesFrom(forContainer, podName string, ctrNames, annotations map[string]string) ([]string, error) {
+	annotationVolsFrom := define.VolumesFromAnnotation + "/" + forContainer
+
+	volsFromCtrs, ok := annotations[annotationVolsFrom]
+
+	// No volumes-from specified
+	if !ok || volsFromCtrs == "" {
+		return nil, nil
+	}
+
+	// The volumes-from string is a semicolon-separated container names
+	// optionally with respective mount options.
+	volumesFrom := strings.Split(volsFromCtrs, ";")
+	for idx, volsFromCtr := range volumesFrom {
+		// Each entry is of format "container[:mount-options]"
+		fields := strings.Split(volsFromCtr, ":")
+		if len(fields) != 1 && len(fields) != 2 {
+			return nil, fmt.Errorf("invalid annotation %s value", annotationVolsFrom)
+		}
+
+		if fields[0] == "" {
+			return nil, fmt.Errorf("from container name cannot be empty in annotation %s", annotationVolsFrom)
+		}
+
+		// Source and target containers cannot be same
+		if fields[0] == forContainer {
+			return nil, fmt.Errorf("to and from container names cannot be same in annotation %s", annotationVolsFrom)
+		}
+
+		// Update the source container name if it belongs to the pod
+		// the source container must exist before the target container
+		// in the kube yaml. Otherwise, the source container will be
+		// treated as an external container. This also helps in avoiding
+		// cyclic dependencies between containers within the pod.
+		if _, ok := ctrNames[fields[0]]; ok {
+			volumesFrom[idx] = podName + "-" + fields[0]
+			if len(fields) == 2 {
+				volumesFrom[idx] = volumesFrom[idx] + ":" + fields[1]
+			}
+		}
+	}
+
+	return volumesFrom, nil
+}
+
 // Creates the name for a k8s entity based on the provided content of a
 // K8s yaml file and a given suffix.
 func k8sName(content []byte, suffix string) string {
@@ -475,6 +520,10 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		return nil, nil, fmt.Errorf("pod does not have a name")
 	}
 
+	if _, ok := annotations[define.VolumesFromAnnotation]; ok {
+		return nil, nil, fmt.Errorf("annotation %s without target volume is reserved for internal use", define.VolumesFromAnnotation)
+	}
+
 	podOpt := entities.PodCreateOptions{
 		Infra:      true,
 		Net:        &entities.NetOptions{NoHosts: options.NoHosts},
@@ -778,6 +827,13 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			initCtrType = define.OneShotInitContainer
 		}
 
+		var volumesFrom []string
+		if list, err := prepareVolumesFrom(initCtr.Name, podName, ctrNames, annotations); err != nil {
+			return nil, nil, err
+		} else if list != nil {
+			volumesFrom = list
+		}
+
 		specgenOpts := kube.CtrSpecGenOptions{
 			Annotations:        annotations,
 			ConfigMaps:         configMaps,
@@ -798,6 +854,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			SecretsManager:     secretsManager,
 			UserNSIsHost:       p.Userns.IsHost(),
 			Volumes:            volumes,
+			VolumesFrom:        volumesFrom,
 			UtsNSIsHost:        p.UtsNs.IsHost(),
 		}
 		specGen, err := kube.ToSpecGen(ctx, &specgenOpts)
@@ -854,6 +911,13 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			labels[k] = v
 		}
 
+		var volumesFrom []string
+		if list, err := prepareVolumesFrom(container.Name, podName, ctrNames, annotations); err != nil {
+			return nil, nil, err
+		} else if list != nil {
+			volumesFrom = list
+		}
+
 		specgenOpts := kube.CtrSpecGenOptions{
 			Annotations:        annotations,
 			ConfigMaps:         configMaps,
@@ -874,6 +938,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			SecretsManager:     secretsManager,
 			UserNSIsHost:       p.Userns.IsHost(),
 			Volumes:            volumes,
+			VolumesFrom:        volumesFrom,
 			UtsNSIsHost:        p.UtsNs.IsHost(),
 		}
 

pkg/specgen/generate/kube/kube.go

@@ -140,6 +140,8 @@ type CtrSpecGenOptions struct {
 	IpcNSIsHost bool
 	// Volumes for all containers
 	Volumes map[string]*KubeVolume
+	// VolumesFrom for all containers
+	VolumesFrom []string
 	// PodID of the parent pod
 	PodID string
 	// PodName of the parent pod
@@ -566,6 +568,8 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener
 		}
 	}
 
+	s.VolumesFrom = opts.VolumesFrom
+
 	s.RestartPolicy = opts.RestartPolicy
 
 	if opts.NetNSIsHost {

pkg/specgen/generate/oci_freebsd.go

@@ -152,7 +152,7 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt
 	}
 
 	if len(s.VolumesFrom) > 0 {
-		configSpec.Annotations[define.InspectAnnotationVolumesFrom] = strings.Join(s.VolumesFrom, ";")
+		configSpec.Annotations[define.VolumesFromAnnotation] = strings.Join(s.VolumesFrom, ";")
 	}
 
 	if s.IsPrivileged() {

pkg/specgen/generate/oci_linux.go

@@ -322,7 +322,7 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt
 	}
 
 	if len(s.VolumesFrom) > 0 {
-		configSpec.Annotations[define.InspectAnnotationVolumesFrom] = strings.Join(s.VolumesFrom, ";")
+		configSpec.Annotations[define.VolumesFromAnnotation] = strings.Join(s.VolumesFrom, ";")
 	}
 
 	if s.IsPrivileged() {

test/e2e/container_inspect_test.go

@@ -68,6 +68,6 @@ var _ = Describe("Podman container inspect", func() {
 		data := podmanTest.InspectContainer(ctr2)
 		Expect(data).To(HaveLen(1))
 		Expect(data[0].HostConfig.VolumesFrom).To(Equal([]string{volsctr}))
-		Expect(data[0].Config.Annotations[define.InspectAnnotationVolumesFrom]).To(Equal(volsctr))
+		Expect(data[0].Config.Annotations[define.VolumesFromAnnotation]).To(Equal(volsctr))
 	})
 })

test/e2e/generate_kube_test.go

@@ -1696,7 +1696,7 @@ USER test1`
 		pod := new(v1.Pod)
 		err = yaml.Unmarshal(kube.Out.Contents(), pod)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(pod.Annotations).To(HaveKeyWithValue(define.InspectAnnotationVolumesFrom+"/"+ctr2, ctr1))
+		Expect(pod.Annotations).To(HaveKeyWithValue(define.VolumesFromAnnotation+"/"+ctr2, ctr1))
 	})
 
 	It("pod volumes-from annotation with semicolon as field separator", func() {
@@ -1734,7 +1734,7 @@ USER test1`
 		pod := new(v1.Pod)
 		err3 := yaml.Unmarshal(kube.Out.Contents(), pod)
 		Expect(err3).ToNot(HaveOccurred())
-		Expect(pod.Annotations).To(HaveKeyWithValue(define.InspectAnnotationVolumesFrom+"/"+tgtctr, frmopt1+";"+frmopt2))
+		Expect(pod.Annotations).To(HaveKeyWithValue(define.VolumesFromAnnotation+"/"+tgtctr, frmopt1+";"+frmopt2))
 	})
 
 	It("--podman-only on container with --rm", func() {

test/e2e/play_kube_test.go

@@ -492,6 +492,34 @@ spec:
 status: {}
 `
 
+var volumesFromPodYaml = `
+apiVersion: v1
+kind: Pod
+metadata:
+  annotations:
+    io.podman.annotations.volumes-from/tgtctr: srcctr:ro
+  name: volspod
+spec:
+    containers:
+    - name: srcctr
+      image: ` + CITEST_IMAGE + `
+      command:
+        - sleep
+        - inf
+      volumeMounts:
+      - mountPath: /mnt/vol
+        name: testing
+    - name: tgtctr
+      image: ` + CITEST_IMAGE + `
+      command:
+        - sleep
+        - inf
+    volumes:
+    - name: testing
+      persistentVolumeClaim:
+        claimName: testvol
+`
+
 var configMapYamlTemplate = `
 apiVersion: v1
 kind: ConfigMap
@@ -5909,7 +5937,7 @@ spec:
 		Expect(session).Should(Exit(125))
 	})
 
-	It("test with reserved volumes-from annotation in yaml", func() {
+	It("test pod with volumes-from annotation in yaml", func() {
 		ctr1 := "ctr1"
 		ctr2 := "ctr2"
 		ctrNameInKubePod := ctr2 + "-pod-" + ctr2
@@ -5927,7 +5955,7 @@ spec:
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
 
-		kube := podmanTest.Podman([]string{"kube", "generate", "--podman-only", "-f", outputFile, ctr2})
+		kube := podmanTest.Podman([]string{"kube", "generate", "-f", outputFile, ctr2})
 		kube.WaitWithDefaultTimeout()
 		Expect(kube).Should(ExitCleanly())
 
@@ -5952,6 +5980,114 @@ spec:
 		Expect(podrm).Should(ExitCleanly())
 	})
 
+	It("test volumes-from annotation with source containers external", func() {
+		// Assert that volumes of multiple source containers, listed in
+		// volumes-from annotation, running outside the pod are
+		// getting mounted inside the target container.
+
+		srcctr1, srcctr2, tgtctr := "srcctr1", "srcctr2", "tgtctr"
+		frmopt1, frmopt2 := srcctr1+":ro", srcctr2+":ro"
+		vol1 := filepath.Join(podmanTest.TempDir, "vol-test1")
+		vol2 := filepath.Join(podmanTest.TempDir, "vol-test2")
+
+		volsFromAnnotaton := define.VolumesFromAnnotation + "/" + tgtctr
+		volsFromValue := frmopt1 + ";" + frmopt2
+
+		err1 := os.MkdirAll(vol1, 0755)
+		Expect(err1).ToNot(HaveOccurred())
+
+		err2 := os.MkdirAll(vol2, 0755)
+		Expect(err2).ToNot(HaveOccurred())
+
+		session := podmanTest.Podman([]string{"create", "--name", srcctr1, "-v", vol1, CITEST_IMAGE})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(ExitCleanly())
+
+		session = podmanTest.Podman([]string{"create", "--name", srcctr2, "-v", vol2, CITEST_IMAGE})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(ExitCleanly())
+
+		podName := tgtctr
+		pod := getPod(
+			withPodName(podName),
+			withCtr(getCtr(withName(tgtctr))),
+			withAnnotation(volsFromAnnotaton, volsFromValue))
+
+		err3 := generateKubeYaml("pod", pod, kubeYaml)
+		Expect(err3).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"kube", "play", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(ExitCleanly())
+
+		// Assert volumes are accessible inside the target container
+		ctrNameInKubePod := podName + "-" + tgtctr
+
+		inspect := podmanTest.InspectContainer(ctrNameInKubePod)
+		Expect(inspect).To(HaveLen(1))
+
+		exec := podmanTest.Podman([]string{"exec", ctrNameInKubePod, "ls", "-d", vol1, vol2})
+		exec.WaitWithDefaultTimeout()
+		Expect(exec).Should(ExitCleanly())
+		Expect(exec.OutputToString()).To(ContainSubstring(vol1))
+		Expect(exec.OutputToString()).To(ContainSubstring(vol2))
+	})
+
+	It("test volumes-from annotation with source container in pod", func() {
+		// Assert that volume of source container, member of the pod,
+		// listed in volumes-from annotation is getting mounted inside
+		// the target container.
+
+		srcctr, tgtctr, podName := "srcctr", "tgtctr", "volspod"
+		vol := "/mnt/vol"
+
+		srcctrInKubePod := podName + "-" + srcctr
+		tgtctrInKubePod := podName + "-" + tgtctr
+
+		err := writeYaml(volumesFromPodYaml, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"kube", "play", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(ExitCleanly())
+
+		inspect := podmanTest.InspectContainer(tgtctrInKubePod)
+		Expect(inspect).To(HaveLen(1))
+
+		// Assert volume is accessible inside the target container
+		// by creating contents in the volume and accessing that from
+		// the target container.
+		volFile := filepath.Join(vol, RandomString(10)+".txt")
+
+		exec := podmanTest.Podman([]string{"exec", srcctrInKubePod, "touch", volFile})
+		exec.WaitWithDefaultTimeout()
+		Expect(exec).Should(ExitCleanly())
+
+		exec = podmanTest.Podman([]string{"exec", srcctrInKubePod, "ls", volFile})
+		exec.WaitWithDefaultTimeout()
+		Expect(exec).Should(ExitCleanly())
+		Expect(exec.OutputToString()).To(ContainSubstring(volFile))
+
+		exec = podmanTest.Podman([]string{"exec", tgtctrInKubePod, "ls", volFile})
+		exec.WaitWithDefaultTimeout()
+		Expect(exec).Should(ExitCleanly())
+		Expect(exec.OutputToString()).To(ContainSubstring(volFile))
+	})
+
+	It("test with reserved volumes-from annotation in yaml", func() {
+		// Assert that volumes-from annotation without target container
+		// errors out.
+
+		pod := getPod(withAnnotation(define.VolumesFromAnnotation, "reserved"))
+		err := generateKubeYaml("pod", pod, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"kube", "play", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(125))
+		Expect(kube.ErrorToString()).To(ContainSubstring("annotation " + define.VolumesFromAnnotation + " without target volume is reserved for internal use"))
+	})
+
 	It("test with reserved autoremove annotation in yaml", func() {
 		ctr := "ctr"
 		ctrNameInKubePod := ctr + "-pod-" + ctr