Merge pull request #22920 from cgwalters/virtiofsd-machine

machine/linux: Switch to virtiofs by default

contrib/cirrus/setup_environment.sh

@@ -394,7 +394,12 @@ case "$TEST_FLAVOR" in
         install_test_configs
         ;;
     machine-linux)
-        showrun dnf install -y podman-gvproxy*
+        showrun dnf install -y podman-gvproxy* virtiofsd
+        # Bootstrap this link if it isn't yet in the package; xref
+        # https://github.com/containers/podman/pull/22920
+        if ! test -L /usr/libexec/podman/virtiofsd; then
+            showrun ln -sfr /usr/libexec/virtiofsd /usr/libexec/podman/virtiofsd
+        fi
         remove_packaged_podman_files
         showrun make install PREFIX=/usr ETCDIR=/etc
         install_test_configs

pkg/machine/apple/apple.go

@@ -88,7 +88,7 @@ func GenerateSystemDFilesForVirtiofsMounts(mounts []machine.VirtIoFs) ([]ignitio
 		mountUnit.Add("Mount", "What", "%s")
 		mountUnit.Add("Mount", "Where", "%s")
 		mountUnit.Add("Mount", "Type", "virtiofs")
-		mountUnit.Add("Mount", "Options", "context=\"system_u:object_r:nfs_t:s0\"")
+		mountUnit.Add("Mount", "Options", fmt.Sprintf("context=\"%s\"", machine.NFSSELinuxContext))
 		mountUnit.Add("Install", "WantedBy", "multi-user.target")
 		mountUnitFile, err := mountUnit.ToString()
 		if err != nil {

pkg/machine/e2e/basic_test.go

@@ -101,6 +101,33 @@ var _ = Describe("run basic podman commands", func() {
 		Expect(runAlp).To(Exit(0))
 	})
 
+	It("Volume should be virtiofs", func() {
+		// In theory this could run on MacOS too, but we know virtiofs works for that now,
+		// this is just testing linux
+		skipIfNotVmtype(define.QemuVirt, "This is just adding coverage for virtiofs on linux")
+
+		tDir, err := filepath.Abs(GinkgoT().TempDir())
+		Expect(err).ToNot(HaveOccurred())
+
+		err = os.WriteFile(filepath.Join(tDir, "testfile"), []byte("some test contents"), 0o644)
+		Expect(err).ToNot(HaveOccurred())
+
+		name := randomString()
+		i := new(initMachine).withImage(mb.imagePath).withNow()
+
+		// Ensure that this is a volume, it may not be automatically on qemu
+		i.withVolume(tDir)
+		session, err := mb.setName(name).setCmd(i).run()
+		Expect(err).ToNot(HaveOccurred())
+		Expect(session).To(Exit(0))
+
+		ssh := new(sshMachine).withSSHCommand([]string{"findmnt", "-no", "FSTYPE", tDir})
+		findmnt, err := mb.setName(name).setCmd(ssh).run()
+		Expect(err).ToNot(HaveOccurred())
+		Expect(findmnt).To(Exit(0))
+		Expect(findmnt.outputToString()).To(ContainSubstring("virtiofs"))
+	})
+
 	It("Podman ops with port forwarding and gvproxy", func() {
 		name := randomString()
 		i := new(initMachine)

pkg/machine/provider/platform.go

@@ -32,7 +32,7 @@ func Get() (vmconfigs.VMProvider, error) {
 	logrus.Debugf("Using Podman machine with `%s` virtualization provider", resolvedVMType.String())
 	switch resolvedVMType {
 	case define.QemuVirt:
-		return new(qemu.QEMUStubber), nil
+		return qemu.NewStubber()
 	default:
 		return nil, fmt.Errorf("unsupported virtualization provider: `%s`", resolvedVMType.String())
 	}

pkg/machine/qemu/command/command.go

@@ -35,8 +35,10 @@ func NewQemuBuilder(binary string, options []string) QemuCmd {
 
 // SetMemory adds the specified amount of memory for the machine
 func (q *QemuCmd) SetMemory(m strongunits.MiB) {
-	// qemu accepts the memory in MiB
-	*q = append(*q, "-m", strconv.FormatUint(uint64(m), 10))
+	serializedMem := strconv.FormatUint(uint64(m), 10)
+	// In order to use virtiofsd, we must enable shared memory
+	*q = append(*q, "-object", fmt.Sprintf("memory-backend-memfd,id=mem,size=%sM,share=on", serializedMem))
+	*q = append(*q, "-m", serializedMem)
 }
 
 // SetCPUs adds the number of CPUs the machine will have
@@ -97,15 +99,6 @@ func (q *QemuCmd) SetSerialPort(readySocket, vmPidFile define.VMFile, name strin
 		"-pidfile", vmPidFile.GetPath())
 }
 
-// SetVirtfsMount adds a virtfs mount to the machine
-func (q *QemuCmd) SetVirtfsMount(source, tag, securityModel string, readonly bool) {
-	virtfsOptions := fmt.Sprintf("local,path=%s,mount_tag=%s,security_model=%s", source, tag, securityModel)
-	if readonly {
-		virtfsOptions += ",readonly"
-	}
-	*q = append(*q, "-virtfs", virtfsOptions)
-}
-
 // SetBootableImage specifies the image the machine will use to boot
 func (q *QemuCmd) SetBootableImage(image string) {
 	*q = append(*q, "-drive", "if=virtio,file="+image)

pkg/machine/qemu/command/qemu_command_test.go

@@ -47,12 +47,13 @@ func TestQemuCmd(t *testing.T) {
 	err = cmd.SetNetwork(vlanSocket)
 	assert.NoError(t, err)
 	cmd.SetSerialPort(*readySocket, *vmPidFile, "test-machine")
-	cmd.SetVirtfsMount("/tmp/path", "vol10", "none", true)
 	cmd.SetBootableImage(bootableImagePath)
 	cmd.SetDisplay("none")
 
 	expected := []string{
 		"/usr/bin/qemu-system-x86_64",
+		"-object",
+		"memory-backend-memfd,id=mem,size=2048M,share=on",
 		"-m", "2048",
 		"-smp", "4",
 		"-fw_cfg", fmt.Sprintf("name=opt/com.coreos/config,file=%s", ignPath),
@@ -63,7 +64,6 @@ func TestQemuCmd(t *testing.T) {
 		"-chardev", fmt.Sprintf("socket,path=%s,server=on,wait=off,id=atest-machine_ready", readySocketPath),
 		"-device", "virtserialport,chardev=atest-machine_ready,name=org.fedoraproject.port.0",
 		"-pidfile", vmPidFilePath,
-		"-virtfs", "local,path=/tmp/path,mount_tag=vol10,security_model=none,readonly",
 		"-drive", fmt.Sprintf("if=virtio,file=%s", bootableImagePath),
 		"-display", "none"}
 

pkg/machine/qemu/machine.go

@@ -24,9 +24,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-const (
-	MountType9p = "9p"
-)
+func NewStubber() (*QEMUStubber, error) {
+	return &QEMUStubber{}, nil
+}
 
 // qemuPid returns -1 or the PID of the running QEMU instance.
 func qemuPid(pidFile *define.VMFile) (int, error) {

pkg/machine/qemu/options_linux_amd64.go

@@ -10,6 +10,7 @@ func (q *QEMUStubber) addArchOptions(_ *setNewMachineCMDOpts) []string {
 	opts := []string{
 		"-accel", "kvm",
 		"-cpu", "host",
+		"-M", "memory-backend=mem",
 	}
 	return opts
 }

pkg/machine/qemu/options_linux_arm64.go

@@ -16,7 +16,7 @@ func (q *QEMUStubber) addArchOptions(_ *setNewMachineCMDOpts) []string {
 	opts := []string{
 		"-accel", "kvm",
 		"-cpu", "host",
-		"-M", "virt,gic-version=max",
+		"-M", "virt,gic-version=max,memory-backend=mem",
 		"-bios", getQemuUefiFile("QEMU_EFI.fd"),
 	}
 	return opts

pkg/machine/qemu/stubber.go

@@ -31,6 +31,9 @@ type QEMUStubber struct {
 	vmconfigs.QEMUConfig
 	// Command describes the final QEMU command line
 	Command command.QemuCmd
+
+	// virtiofsHelpers are virtiofsd child processes
+	virtiofsHelpers []virtiofsdHelperCmd
 }
 
 var (
@@ -83,13 +86,6 @@ func (q *QEMUStubber) setQEMUCommandLine(mc *vmconfigs.MachineConfig) error {
 	}
 	q.Command.SetSerialPort(*readySocket, *mc.QEMUHypervisor.QEMUPidPath, mc.Name)
 
-	// Add volumes to qemu command line
-	for _, mount := range mc.Mounts {
-		// the index provided in this case is thrown away
-		_, _, _, _, securityModel := vmconfigs.SplitVolume(0, mount.OriginalInput)
-		q.Command.SetVirtfsMount(mount.Source, mount.Tag, securityModel, mount.ReadOnly)
-	}
-
 	q.Command.SetUSBHostPassthrough(mc.Resources.USBs)
 
 	return nil
@@ -168,6 +164,24 @@ func (q *QEMUStubber) StartVM(mc *vmconfigs.MachineConfig) (func() error, func()
 	defer dnr.Close()
 	defer dnw.Close()
 
+	runtime, err := mc.RuntimeDir()
+	if err != nil {
+		return nil, nil, err
+	}
+	spawner, err := newVirtiofsdSpawner(runtime)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, hostmnt := range mc.Mounts {
+		qemuArgs, virtiofsdHelper, err := spawner.spawnForMount(hostmnt)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to init virtiofsd for mount %s: %w", hostmnt.Source, err)
+		}
+		q.Command = append(q.Command, qemuArgs...)
+		q.virtiofsHelpers = append(q.virtiofsHelpers, *virtiofsdHelper)
+	}
+
 	cmdLine := q.Command
 
 	// Disable graphic window when not in debug mode
@@ -198,8 +212,20 @@ func (q *QEMUStubber) StartVM(mc *vmconfigs.MachineConfig) (func() error, func()
 		return waitForReady(readySocket, cmd.Process.Pid, stderrBuf)
 	}
 
+	releaseFunc := func() error {
+		if err := cmd.Process.Release(); err != nil {
+			return err
+		}
+		for _, virtiofsdCmd := range q.virtiofsHelpers {
+			if err := virtiofsdCmd.command.Process.Release(); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
 	// if this is not the last line in the func, make it a defer
-	return cmd.Process.Release, readyFunc, nil
+	return releaseFunc, readyFunc, nil
 }
 
 func waitForReady(readySocket *define.VMFile, pid int, stdErrBuffer *bytes.Buffer) error {
@@ -325,27 +351,27 @@ func (q *QEMUStubber) MountVolumesToVM(mc *vmconfigs.MachineConfig, quiet bool)
 		if err != nil {
 			return err
 		}
-		switch mount.Type {
-		case MountType9p:
-			mountOptions := []string{"-t", "9p"}
-			mountOptions = append(mountOptions, []string{"-o", "trans=virtio", mount.Tag, mount.Target}...)
-			mountOptions = append(mountOptions, []string{"-o", "version=9p2000.L,msize=131072,cache=mmap"}...)
+		// NOTE: The mount type q.Type was previously serialized as 9p for older Linux versions,
+		// but we ignore it now because we want the mount type to be dynamic, not static.  Or
+		// in other words we don't want to make people unnecessarily reprovision their machines
+		// to upgrade from 9p to virtiofs.
+		mountOptions := []string{"-t", "virtiofs"}
+		mountOptions = append(mountOptions, []string{mount.Tag, mount.Target}...)
+		mountFlags := fmt.Sprintf("context=\"%s\"", machine.NFSSELinuxContext)
 		if mount.ReadOnly {
-				mountOptions = append(mountOptions, []string{"-o", "ro"}...)
+			mountFlags += ",ro"
 		}
+		mountOptions = append(mountOptions, "-o", mountFlags)
 		err = machine.CommonSSH(mc.SSH.RemoteUsername, mc.SSH.IdentityPath, mc.Name, mc.SSH.Port, append([]string{"sudo", "mount"}, mountOptions...))
 		if err != nil {
 			return err
 		}
-		default:
-			return fmt.Errorf("unknown mount type: %s", mount.Type)
-		}
 	}
 	return nil
 }
 
 func (q *QEMUStubber) MountType() vmconfigs.VolumeMountType {
-	return vmconfigs.NineP
+	return vmconfigs.VirtIOFS
 }
 
 func (q *QEMUStubber) PostStartNetworking(mc *vmconfigs.MachineConfig, noInfo bool) error {

pkg/machine/qemu/virtiofsd.go

@@ -0,0 +1,99 @@
+//go:build linux || freebsd
+
+package qemu
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/containers/common/pkg/config"
+	"github.com/containers/podman/v5/pkg/machine/define"
+	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
+	"github.com/containers/storage/pkg/fileutils"
+	"github.com/sirupsen/logrus"
+)
+
+// VirtiofsdSpawner spawns an instance of virtiofsd
+type virtiofsdSpawner struct {
+	runtimeDir *define.VMFile
+	binaryPath string
+	mountCount uint
+}
+
+func newVirtiofsdSpawner(runtimeDir *define.VMFile) (*virtiofsdSpawner, error) {
+	var binaryPath string
+	cfg, err := config.Default()
+	if err != nil {
+		return nil, err
+	}
+	binaryPath, err = cfg.FindHelperBinary("virtiofsd", true)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find virtiofsd: %w", err)
+	}
+	return &virtiofsdSpawner{binaryPath: binaryPath, runtimeDir: runtimeDir}, nil
+}
+
+// createVirtiofsCmd returns a new command instance configured to launch virtiofsd.
+func (v *virtiofsdSpawner) createVirtiofsCmd(directory, socketPath string) *exec.Cmd {
+	args := []string{"--sandbox", "none", "--socket-path", socketPath, "--shared-dir", "."}
+	// We don't need seccomp filtering; we trust our workloads. This incidentally
+	// works around issues like https://gitlab.com/virtio-fs/virtiofsd/-/merge_requests/200.
+	args = append(args, "--seccomp=none")
+	cmd := exec.Command(v.binaryPath, args...)
+	// This sets things up so that the `.` we passed in the arguments is the target directory
+	cmd.Dir = directory
+	// Quiet the daemon by default
+	cmd.Env = append(cmd.Environ(), "RUST_LOG=ERROR")
+	cmd.Stdin = nil
+	cmd.Stdout = nil
+	if logrus.IsLevelEnabled(logrus.DebugLevel) {
+		cmd.Stderr = os.Stderr
+	}
+	return cmd
+}
+
+type virtiofsdHelperCmd struct {
+	command exec.Cmd
+	socket  *define.VMFile
+}
+
+// spawnForMount returns on success a combination of qemu commandline and child process for virtiofsd
+func (v *virtiofsdSpawner) spawnForMount(hostmnt *vmconfigs.Mount) ([]string, *virtiofsdHelperCmd, error) {
+	logrus.Debugf("Initializing virtiofsd mount for %s", hostmnt.Source)
+	// By far the most common failure to spawn virtiofsd will be a typo'd source directory,
+	// so let's synchronously check that ourselves here.
+	if err := fileutils.Exists(hostmnt.Source); err != nil {
+		return nil, nil, fmt.Errorf("failed to access virtiofs source directory %s", hostmnt.Source)
+	}
+	virtiofsChar := fmt.Sprintf("virtiofschar%d", v.mountCount)
+	virtiofsCharPath, err := v.runtimeDir.AppendToNewVMFile(virtiofsChar, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	qemuCommand := []string{}
+
+	qemuCommand = append(qemuCommand, "-chardev", fmt.Sprintf("socket,id=%s,path=%s", virtiofsChar, virtiofsCharPath.Path))
+	qemuCommand = append(qemuCommand, "-device", fmt.Sprintf("vhost-user-fs-pci,queue-size=1024,chardev=%s,tag=%s", virtiofsChar, hostmnt.Tag))
+	// TODO: Honor hostmnt.readonly somehow here (add an option to virtiofsd)
+	virtiofsdCmd := v.createVirtiofsCmd(hostmnt.Source, virtiofsCharPath.Path)
+	if err := virtiofsdCmd.Start(); err != nil {
+		return nil, nil, fmt.Errorf("failed to start virtiofsd")
+	}
+	// Wait for the socket
+	for {
+		if err := fileutils.Exists(virtiofsCharPath.Path); err == nil {
+			break
+		}
+		logrus.Debugf("waiting for virtiofsd socket %q", virtiofsCharPath.Path)
+		time.Sleep(time.Millisecond * 100)
+	}
+	// Increment our count of mounts which are used to create unique names for the devices
+	v.mountCount += 1
+	return qemuCommand, &virtiofsdHelperCmd{
+		command: *virtiofsdCmd,
+		socket:  virtiofsCharPath,
+	}, nil
+}

pkg/machine/volumes.go

@@ -7,6 +7,11 @@ import (
 	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
 )
 
+// NFSSELinuxContext is what is used by NFS mounts, which is allowed
+// access by container_t.  We need to fix the Fedora selinux policy
+// to just allow access to virtiofs_t.
+const NFSSELinuxContext = "system_u:object_r:nfs_t:s0"
+
 type Volume interface {
 	Kind() VolumeKind
 }