Add krun support to podman machine

This PR adds libkrun support to podman machine.  This is an experimental feature and should not be marketed yet.  Before we unmark the experimental status on this function, we will need to have full CI support and a full podman point release has pased.

This work relies on the fact that vfkit and libkrun share a reasonably (if not perfectly) same API.  The --log-level debug option will not show a GUI screen for boots as krun is not capable of this.

Signed-off-by: Brent Baude <bbaude@redhat.com>
This commit is contained in:
Brent Baude
2024-04-24 08:26:53 -05:00
committed by Brent Baude
parent 0b9bc253a2
commit d2c1de5993
16 changed files with 613 additions and 401 deletions

347
pkg/machine/apple/apple.go Normal file
View File

@ -0,0 +1,347 @@
//go:build darwin
package apple
import (
"context"
"errors"
"fmt"
"net"
"os"
"syscall"
"time"
"github.com/containers/common/pkg/config"
"github.com/containers/common/pkg/strongunits"
gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
"github.com/containers/podman/v5/pkg/machine"
"github.com/containers/podman/v5/pkg/machine/define"
"github.com/containers/podman/v5/pkg/machine/ignition"
"github.com/containers/podman/v5/pkg/machine/sockets"
"github.com/containers/podman/v5/pkg/machine/vmconfigs"
"github.com/containers/podman/v5/pkg/systemd/parser"
vfConfig "github.com/crc-org/vfkit/pkg/config"
"github.com/sirupsen/logrus"
)
const applehvMACAddress = "5a:94:ef:e4:0c:ee"
var (
gvProxyWaitBackoff = 500 * time.Millisecond
gvProxyMaxBackoffAttempts = 6
ignitionSocketName = "ignition.sock"
)
// ResizeDisk uses os truncate to resize (only larger) a raw disk. the input size
// is assumed GiB
func ResizeDisk(mc *vmconfigs.MachineConfig, newSize strongunits.GiB) error {
logrus.Debugf("resizing %s to %d bytes", mc.ImagePath.GetPath(), newSize.ToBytes())
return os.Truncate(mc.ImagePath.GetPath(), int64(newSize.ToBytes()))
}
func SetProviderAttrs(mc *vmconfigs.MachineConfig, opts define.SetOptions, state define.Status) error {
if state != define.Stopped {
return errors.New("unable to change settings unless vm is stopped")
}
if opts.DiskSize != nil {
if err := ResizeDisk(mc, *opts.DiskSize); err != nil {
return err
}
}
if opts.Rootful != nil && mc.HostUser.Rootful != *opts.Rootful {
if err := mc.SetRootful(*opts.Rootful); err != nil {
return err
}
}
if opts.USBs != nil {
return fmt.Errorf("changing USBs not supported for applehv machines")
}
// VFKit does not require saving memory, disk, or cpu
return nil
}
func GenerateSystemDFilesForVirtiofsMounts(mounts []machine.VirtIoFs) ([]ignition.Unit, error) {
// mounting in fcos with virtiofs is a bit of a dance. we need a unit file for the mount, a unit file
// for automatic mounting on boot, and a "preparatory" service file that disables FCOS security, performs
// the mkdir of the mount point, and then re-enables security. This must be done for each mount.
unitFiles := make([]ignition.Unit, 0, len(mounts))
for _, mnt := range mounts {
// Here we are looping the mounts and for each mount, we are adding two unit files
// for virtiofs. One unit file is the mount itself and the second is to automount it
// on boot.
autoMountUnit := parser.NewUnitFile()
autoMountUnit.Add("Automount", "Where", "%s")
autoMountUnit.Add("Install", "WantedBy", "multi-user.target")
autoMountUnit.Add("Unit", "Description", "Mount virtiofs volume %s")
autoMountUnitFile, err := autoMountUnit.ToString()
if err != nil {
return nil, err
}
mountUnit := parser.NewUnitFile()
mountUnit.Add("Mount", "What", "%s")
mountUnit.Add("Mount", "Where", "%s")
mountUnit.Add("Mount", "Type", "virtiofs")
mountUnit.Add("Mount", "Options", "context=\"system_u:object_r:nfs_t:s0\"")
mountUnit.Add("Install", "WantedBy", "multi-user.target")
mountUnitFile, err := mountUnit.ToString()
if err != nil {
return nil, err
}
virtiofsAutomount := ignition.Unit{
Enabled: ignition.BoolToPtr(true),
Name: fmt.Sprintf("%s.automount", mnt.Tag),
Contents: ignition.StrToPtr(fmt.Sprintf(autoMountUnitFile, mnt.Target, mnt.Target)),
}
virtiofsMount := ignition.Unit{
Enabled: ignition.BoolToPtr(true),
Name: fmt.Sprintf("%s.mount", mnt.Tag),
Contents: ignition.StrToPtr(fmt.Sprintf(mountUnitFile, mnt.Tag, mnt.Target)),
}
// This "unit" simulates something like systemctl enable virtiofs-mount-prepare@
enablePrep := ignition.Unit{
Enabled: ignition.BoolToPtr(true),
Name: fmt.Sprintf("virtiofs-mount-prepare@%s.service", mnt.Tag),
}
unitFiles = append(unitFiles, virtiofsAutomount, virtiofsMount, enablePrep)
}
// mount prep is a way to workaround the FCOS limitation of creating directories
// at the rootfs / and then mounting to them.
mountPrep := parser.NewUnitFile()
mountPrep.Add("Unit", "Description", "Allow virtios to mount to /")
mountPrep.Add("Unit", "DefaultDependencies", "no")
mountPrep.Add("Unit", "ConditionPathExists", "!%f")
mountPrep.Add("Service", "Type", "oneshot")
mountPrep.Add("Service", "ExecStartPre", "chattr -i /")
mountPrep.Add("Service", "ExecStart", "mkdir -p '%f'")
mountPrep.Add("Service", "ExecStopPost", "chattr +i /")
mountPrep.Add("Install", "WantedBy", "remote-fs.target")
mountPrepFile, err := mountPrep.ToString()
if err != nil {
return nil, err
}
virtioFSChattr := ignition.Unit{
Contents: ignition.StrToPtr(mountPrepFile),
Name: "virtiofs-mount-prepare@.service",
}
unitFiles = append(unitFiles, virtioFSChattr)
return unitFiles, nil
}
// StartGenericAppleVM is wrappered by apple provider methods and starts the vm
func StartGenericAppleVM(mc *vmconfigs.MachineConfig, cmdBinary string, bootloader vfConfig.Bootloader, endpoint string) (func() error, func() error, error) {
var (
ignitionSocket *define.VMFile
)
// Add networking
netDevice, err := vfConfig.VirtioNetNew(applehvMACAddress)
if err != nil {
return nil, nil, err
}
// Set user networking with gvproxy
gvproxySocket, err := mc.GVProxySocket()
if err != nil {
return nil, nil, err
}
// Wait on gvproxy to be running and aware
if err := sockets.WaitForSocketWithBackoffs(gvProxyMaxBackoffAttempts, gvProxyWaitBackoff, gvproxySocket.GetPath(), "gvproxy"); err != nil {
return nil, nil, err
}
netDevice.SetUnixSocketPath(gvproxySocket.GetPath())
// create a one-time virtual machine for starting because we dont want all this information in the
// machineconfig if possible. the preference was to derive this stuff
vm := vfConfig.NewVirtualMachine(uint(mc.Resources.CPUs), uint64(mc.Resources.Memory), bootloader)
defaultDevices, readySocket, err := GetDefaultDevices(mc)
if err != nil {
return nil, nil, err
}
vm.Devices = append(vm.Devices, defaultDevices...)
vm.Devices = append(vm.Devices, netDevice)
mounts, err := VirtIOFsToVFKitVirtIODevice(mc.Mounts)
if err != nil {
return nil, nil, err
}
vm.Devices = append(vm.Devices, mounts...)
// To start the VM, we need to call vfkit
cfg, err := config.Default()
if err != nil {
return nil, nil, err
}
cmdBinaryPath, err := cfg.FindHelperBinary(cmdBinary, true)
if err != nil {
return nil, nil, err
}
logrus.Debugf("helper binary path is: %s", cmdBinaryPath)
cmd, err := vm.Cmd(cmdBinaryPath)
if err != nil {
return nil, nil, err
}
if logrus.IsLevelEnabled(logrus.DebugLevel) {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
}
endpointArgs, err := GetVfKitEndpointCMDArgs(endpoint)
if err != nil {
return nil, nil, err
}
machineDataDir, err := mc.DataDir()
if err != nil {
return nil, nil, err
}
cmd.Args = append(cmd.Args, endpointArgs...)
firstBoot, err := mc.IsFirstBoot()
if err != nil {
return nil, nil, err
}
if logrus.IsLevelEnabled(logrus.DebugLevel) {
debugDevArgs, err := GetDebugDevicesCMDArgs()
if err != nil {
return nil, nil, err
}
cmd.Args = append(cmd.Args, debugDevArgs...)
cmd.Args = append(cmd.Args, "--gui") // add command line switch to pop the gui open
}
if firstBoot {
// If this is the first boot of the vm, we need to add the vsock
// device to vfkit so we can inject the ignition file
socketName := fmt.Sprintf("%s-%s", mc.Name, ignitionSocketName)
ignitionSocket, err = machineDataDir.AppendToNewVMFile(socketName, &socketName)
if err != nil {
return nil, nil, err
}
if err := ignitionSocket.Delete(); err != nil {
logrus.Errorf("unable to delete ignition socket: %q", err)
}
ignitionVsockDeviceCLI, err := GetIgnitionVsockDeviceAsCLI(ignitionSocket.GetPath())
if err != nil {
return nil, nil, err
}
cmd.Args = append(cmd.Args, ignitionVsockDeviceCLI...)
logrus.Debug("first boot detected")
logrus.Debugf("serving ignition file over %s", ignitionSocket.GetPath())
go func() {
if err := ServeIgnitionOverSock(ignitionSocket, mc); err != nil {
logrus.Error(err)
}
logrus.Debug("ignition vsock server exited")
}()
}
logrus.Debugf("listening for ready on: %s", readySocket.GetPath())
if err := readySocket.Delete(); err != nil {
logrus.Warnf("unable to delete previous ready socket: %q", err)
}
readyListen, err := net.Listen("unix", readySocket.GetPath())
if err != nil {
return nil, nil, err
}
logrus.Debug("waiting for ready notification")
readyChan := make(chan error)
go sockets.ListenAndWaitOnSocket(readyChan, readyListen)
logrus.Debugf("helper command-line: %v", cmd.Args)
if err := cmd.Start(); err != nil {
return nil, nil, err
}
returnFunc := func() error {
processErrChan := make(chan error)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
defer close(processErrChan)
for {
select {
case <-ctx.Done():
return
default:
}
if err := CheckProcessRunning(cmdBinary, cmd.Process.Pid); err != nil {
processErrChan <- err
return
}
// lets poll status every half second
time.Sleep(500 * time.Millisecond)
}
}()
// wait for either socket or to be ready or process to have exited
select {
case err := <-processErrChan:
if err != nil {
return err
}
case err := <-readyChan:
if err != nil {
return err
}
logrus.Debug("ready notification received")
}
return nil
}
return cmd.Process.Release, returnFunc, nil
}
// CheckProcessRunning checks non blocking if the pid exited
// returns nil if process is running otherwise an error if not
func CheckProcessRunning(processName string, pid int) error {
var status syscall.WaitStatus
pid, err := syscall.Wait4(pid, &status, syscall.WNOHANG, nil)
if err != nil {
return fmt.Errorf("failed to read %s process status: %w", processName, err)
}
if pid > 0 {
// child exited
return fmt.Errorf("%s exited unexpectedly with exit code %d", processName, status.ExitStatus())
}
return nil
}
// StartGenericNetworking is wrappered by apple provider methods
func StartGenericNetworking(mc *vmconfigs.MachineConfig, cmd *gvproxy.GvproxyCommand) error {
gvProxySock, err := mc.GVProxySocket()
if err != nil {
return err
}
// make sure it does not exist before gvproxy is called
if err := gvProxySock.Delete(); err != nil {
logrus.Error(err)
}
cmd.AddVfkitSocket(fmt.Sprintf("unixgram://%s", gvProxySock.GetPath()))
return nil
}

View File

@ -1,6 +1,6 @@
//go:build darwin
package applehv
package apple
import (
"net"
@ -11,9 +11,9 @@ import (
"github.com/sirupsen/logrus"
)
// serveIgnitionOverSock allows podman to open a small httpd instance on the vsock between the host
// ServeIgnitionOverSock allows podman to open a small httpd instance on the vsock between the host
// and guest to inject the ignitionfile into fcos
func serveIgnitionOverSock(ignitionSocket *define.VMFile, mc *vmconfigs.MachineConfig) error {
func ServeIgnitionOverSock(ignitionSocket *define.VMFile, mc *vmconfigs.MachineConfig) error {
ignitionFile, err := mc.IgnitionFile()
if err != nil {
return err

View File

@ -1,14 +1,17 @@
//go:build darwin
package applehv
package apple
import (
"errors"
"github.com/containers/podman/v5/pkg/machine/define"
"github.com/containers/podman/v5/pkg/machine/vmconfigs"
vfConfig "github.com/crc-org/vfkit/pkg/config"
"github.com/crc-org/vfkit/pkg/rest"
)
func getDefaultDevices(mc *vmconfigs.MachineConfig) ([]vfConfig.VirtioDevice, *define.VMFile, error) {
func GetDefaultDevices(mc *vmconfigs.MachineConfig) ([]vfConfig.VirtioDevice, *define.VMFile, error) {
var devices []vfConfig.VirtioDevice
disk, err := vfConfig.VirtioBlkNew(mc.ImagePath.GetPath())
@ -42,7 +45,7 @@ func getDefaultDevices(mc *vmconfigs.MachineConfig) ([]vfConfig.VirtioDevice, *d
return devices, readySocket, nil
}
func getDebugDevices() ([]vfConfig.VirtioDevice, error) {
func GetDebugDevices() ([]vfConfig.VirtioDevice, error) {
var devices []vfConfig.VirtioDevice
gpu, err := vfConfig.VirtioGPUNew()
if err != nil {
@ -59,11 +62,11 @@ func getDebugDevices() ([]vfConfig.VirtioDevice, error) {
return append(devices, gpu, mouse, kb), nil
}
func getIgnitionVsockDevice(path string) (vfConfig.VirtioDevice, error) {
func GetIgnitionVsockDevice(path string) (vfConfig.VirtioDevice, error) {
return vfConfig.VirtioVsockNew(1024, path, true)
}
func virtIOFsToVFKitVirtIODevice(mounts []*vmconfigs.Mount) ([]vfConfig.VirtioDevice, error) {
func VirtIOFsToVFKitVirtIODevice(mounts []*vmconfigs.Mount) ([]vfConfig.VirtioDevice, error) {
virtioDevices := make([]vfConfig.VirtioDevice, 0, len(mounts))
for _, vol := range mounts {
virtfsDevice, err := vfConfig.VirtioFsNew(vol.Source, vol.Tag)
@ -74,3 +77,48 @@ func virtIOFsToVFKitVirtIODevice(mounts []*vmconfigs.Mount) ([]vfConfig.VirtioDe
}
return virtioDevices, nil
}
// GetVfKitEndpointCMDArgs converts the vfkit endpoint to a cmdline format
func GetVfKitEndpointCMDArgs(endpoint string) ([]string, error) {
if len(endpoint) == 0 {
return nil, errors.New("endpoint cannot be empty")
}
restEndpoint, err := rest.NewEndpoint(endpoint)
if err != nil {
return nil, err
}
return restEndpoint.ToCmdLine()
}
// GetIgnitionVsockDeviceAsCLI retrieves the ignition vsock device and converts
// it to a cmdline format
func GetIgnitionVsockDeviceAsCLI(ignitionSocketPath string) ([]string, error) {
ignitionVsockDevice, err := GetIgnitionVsockDevice(ignitionSocketPath)
if err != nil {
return nil, err
}
// Convert the device into cli args
ignitionVsockDeviceCLI, err := ignitionVsockDevice.ToCmdLine()
if err != nil {
return nil, err
}
return ignitionVsockDeviceCLI, nil
}
// GetDebugDevicesCMDArgs retrieves the debug devices and converts them to a
// cmdline format
func GetDebugDevicesCMDArgs() ([]string, error) {
args := []string{}
debugDevices, err := GetDebugDevices()
if err != nil {
return nil, err
}
for _, debugDevice := range debugDevices {
debugCli, err := debugDevice.ToCmdLine()
if err != nil {
return nil, err
}
args = append(args, debugCli...)
}
return args, nil
}

View File

@ -24,7 +24,7 @@ const (
version = "/version"
)
func (vf *VfkitHelper) get(endpoint string, payload io.Reader) (*http.Response, error) {
func (vf *Helper) get(endpoint string, payload io.Reader) (*http.Response, error) {
client := &http.Client{}
req, err := http.NewRequest(http.MethodGet, endpoint, payload)
if err != nil {
@ -33,7 +33,7 @@ func (vf *VfkitHelper) get(endpoint string, payload io.Reader) (*http.Response,
return client.Do(req)
}
func (vf *VfkitHelper) post(endpoint string, payload io.Reader) (*http.Response, error) {
func (vf *Helper) post(endpoint string, payload io.Reader) (*http.Response, error) {
client := &http.Client{}
req, err := http.NewRequest(http.MethodPost, endpoint, payload)
if err != nil {
@ -43,7 +43,7 @@ func (vf *VfkitHelper) post(endpoint string, payload io.Reader) (*http.Response,
}
// getRawState asks vfkit for virtual machine state unmodified (see state())
func (vf *VfkitHelper) getRawState() (define.Status, error) {
func (vf *Helper) getRawState() (define.Status, error) {
var response rest.VMState
endPoint := vf.Endpoint + state
serverResponse, err := vf.get(endPoint, nil)
@ -66,7 +66,7 @@ func (vf *VfkitHelper) getRawState() (define.Status, error) {
// state asks vfkit for the virtual machine state. in case the vfkit
// service is not responding, we assume the service is not running
// and return a stopped status
func (vf *VfkitHelper) State() (define.Status, error) {
func (vf *Helper) State() (define.Status, error) {
vmState, err := vf.getRawState()
if err == nil {
return vmState, nil
@ -77,7 +77,7 @@ func (vf *VfkitHelper) State() (define.Status, error) {
return "", err
}
func (vf *VfkitHelper) stateChange(newState rest.StateChange) error {
func (vf *Helper) stateChange(newState rest.StateChange) error {
b, err := json.Marshal(rest.VMState{State: string(newState)})
if err != nil {
return err
@ -88,7 +88,7 @@ func (vf *VfkitHelper) stateChange(newState rest.StateChange) error {
return err
}
func (vf *VfkitHelper) Stop(force, wait bool) error {
func (vf *Helper) Stop(force, wait bool) error {
waitDuration := time.Millisecond * 10
// TODO Add ability to wait until stopped
if force {
@ -118,10 +118,10 @@ func (vf *VfkitHelper) Stop(force, wait bool) error {
return waitErr
}
// VfkitHelper describes the use of vfkit: cmdline and endpoint
type VfkitHelper struct {
LogLevel logrus.Level
Endpoint string
VfkitBinaryPath *define.VMFile
VirtualMachine *config.VirtualMachine
// Helper describes the use of vfkit: cmdline and endpoint
type Helper struct {
LogLevel logrus.Level
Endpoint string
BinaryPath *define.VMFile
VirtualMachine *config.VirtualMachine
}

View File

@ -3,66 +3,14 @@
package applehv
import (
"fmt"
"os"
"syscall"
"github.com/containers/common/pkg/strongunits"
"github.com/containers/podman/v5/pkg/machine"
"github.com/containers/podman/v5/pkg/machine/define"
"github.com/containers/podman/v5/pkg/machine/ignition"
"github.com/containers/podman/v5/pkg/machine/vmconfigs"
"github.com/containers/podman/v5/pkg/systemd/parser"
vfRest "github.com/crc-org/vfkit/pkg/rest"
"github.com/sirupsen/logrus"
)
func (a *AppleHVStubber) Remove(mc *vmconfigs.MachineConfig) ([]string, func() error, error) {
return []string{}, func() error { return nil }, nil
}
// getIgnitionVsockDeviceAsCLI retrieves the ignition vsock device and converts
// it to a cmdline format
func getIgnitionVsockDeviceAsCLI(ignitionSocketPath string) ([]string, error) {
ignitionVsockDevice, err := getIgnitionVsockDevice(ignitionSocketPath)
if err != nil {
return nil, err
}
// Convert the device into cli args
ignitionVsockDeviceCLI, err := ignitionVsockDevice.ToCmdLine()
if err != nil {
return nil, err
}
return ignitionVsockDeviceCLI, nil
}
// getDebugDevicesCMDArgs retrieves the debug devices and converts them to a
// cmdline format
func getDebugDevicesCMDArgs() ([]string, error) {
args := []string{}
debugDevices, err := getDebugDevices()
if err != nil {
return nil, err
}
for _, debugDevice := range debugDevices {
debugCli, err := debugDevice.ToCmdLine()
if err != nil {
return nil, err
}
args = append(args, debugCli...)
}
return args, nil
}
// getVfKitEndpointCMDArgs converts the vfkit endpoint to a cmdline format
func getVfKitEndpointCMDArgs(endpoint string) ([]string, error) {
restEndpoint, err := vfRest.NewEndpoint(endpoint)
if err != nil {
return nil, err
}
return restEndpoint.ToCmdLine()
}
func (a *AppleHVStubber) State(mc *vmconfigs.MachineConfig, _ bool) (define.Status, error) {
vmStatus, err := mc.AppleHypervisor.Vfkit.State()
if err != nil {
@ -74,102 +22,3 @@ func (a *AppleHVStubber) State(mc *vmconfigs.MachineConfig, _ bool) (define.Stat
func (a *AppleHVStubber) StopVM(mc *vmconfigs.MachineConfig, _ bool) error {
return mc.AppleHypervisor.Vfkit.Stop(false, true)
}
// checkProcessRunning checks non blocking if the pid exited
// returns nil if process is running otherwise an error if not
func checkProcessRunning(processName string, pid int) error {
var status syscall.WaitStatus
pid, err := syscall.Wait4(pid, &status, syscall.WNOHANG, nil)
if err != nil {
return fmt.Errorf("failed to read %s process status: %w", processName, err)
}
if pid > 0 {
// child exited
return fmt.Errorf("%s exited unexpectedly with exit code %d", processName, status.ExitStatus())
}
return nil
}
// resizeDisk uses os truncate to resize (only larger) a raw disk. the input size
// is assumed GiB
func resizeDisk(mc *vmconfigs.MachineConfig, newSize strongunits.GiB) error {
logrus.Debugf("resizing %s to %d bytes", mc.ImagePath.GetPath(), newSize.ToBytes())
return os.Truncate(mc.ImagePath.GetPath(), int64(newSize.ToBytes()))
}
func generateSystemDFilesForVirtiofsMounts(mounts []machine.VirtIoFs) []ignition.Unit {
// mounting in fcos with virtiofs is a bit of a dance. we need a unit file for the mount, a unit file
// for automatic mounting on boot, and a "preparatory" service file that disables FCOS security, performs
// the mkdir of the mount point, and then re-enables security. This must be done for each mount.
unitFiles := make([]ignition.Unit, 0, len(mounts))
for _, mnt := range mounts {
// Here we are looping the mounts and for each mount, we are adding two unit files
// for virtiofs. One unit file is the mount itself and the second is to automount it
// on boot.
autoMountUnit := parser.NewUnitFile()
autoMountUnit.Add("Automount", "Where", "%s")
autoMountUnit.Add("Install", "WantedBy", "multi-user.target")
autoMountUnit.Add("Unit", "Description", "Mount virtiofs volume %s")
autoMountUnitFile, err := autoMountUnit.ToString()
if err != nil {
logrus.Warnf(err.Error())
}
mountUnit := parser.NewUnitFile()
mountUnit.Add("Mount", "What", "%s")
mountUnit.Add("Mount", "Where", "%s")
mountUnit.Add("Mount", "Type", "virtiofs")
mountUnit.Add("Mount", "Options", "context=\"system_u:object_r:nfs_t:s0\"")
mountUnit.Add("Install", "WantedBy", "multi-user.target")
mountUnitFile, err := mountUnit.ToString()
if err != nil {
logrus.Warnf(err.Error())
}
virtiofsAutomount := ignition.Unit{
Enabled: ignition.BoolToPtr(true),
Name: fmt.Sprintf("%s.automount", mnt.Tag),
Contents: ignition.StrToPtr(fmt.Sprintf(autoMountUnitFile, mnt.Target, mnt.Target)),
}
virtiofsMount := ignition.Unit{
Enabled: ignition.BoolToPtr(true),
Name: fmt.Sprintf("%s.mount", mnt.Tag),
Contents: ignition.StrToPtr(fmt.Sprintf(mountUnitFile, mnt.Tag, mnt.Target)),
}
// This "unit" simulates something like systemctl enable virtiofs-mount-prepare@
enablePrep := ignition.Unit{
Enabled: ignition.BoolToPtr(true),
Name: fmt.Sprintf("virtiofs-mount-prepare@%s.service", mnt.Tag),
}
unitFiles = append(unitFiles, virtiofsAutomount, virtiofsMount, enablePrep)
}
// mount prep is a way to workaround the FCOS limitation of creating directories
// at the rootfs / and then mounting to them.
mountPrep := parser.NewUnitFile()
mountPrep.Add("Unit", "Description", "Allow virtios to mount to /")
mountPrep.Add("Unit", "DefaultDependencies", "no")
mountPrep.Add("Unit", "ConditionPathExists", "!%f")
mountPrep.Add("Service", "Type", "oneshot")
mountPrep.Add("Service", "ExecStartPre", "chattr -i /")
mountPrep.Add("Service", "ExecStart", "mkdir -p '%f'")
mountPrep.Add("Service", "ExecStopPost", "chattr +i /")
mountPrep.Add("Install", "WantedBy", "remote-fs.target")
mountPrepFile, err := mountPrep.ToString()
if err != nil {
logrus.Warnf(err.Error())
}
virtioFSChattr := ignition.Unit{
Contents: ignition.StrToPtr(mountPrepFile),
Name: "virtiofs-mount-prepare@.service",
}
unitFiles = append(unitFiles, virtioFSChattr)
return unitFiles
}

View File

@ -3,36 +3,26 @@
package applehv
import (
"context"
"errors"
"fmt"
"net"
"os"
"strconv"
"time"
"github.com/containers/common/pkg/config"
gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
"github.com/containers/podman/v5/pkg/machine"
"github.com/containers/podman/v5/pkg/machine/applehv/vfkit"
"github.com/containers/podman/v5/pkg/machine/apple"
"github.com/containers/podman/v5/pkg/machine/apple/vfkit"
"github.com/containers/podman/v5/pkg/machine/define"
"github.com/containers/podman/v5/pkg/machine/ignition"
"github.com/containers/podman/v5/pkg/machine/shim/diskpull"
"github.com/containers/podman/v5/pkg/machine/sockets"
"github.com/containers/podman/v5/pkg/machine/vmconfigs"
"github.com/containers/podman/v5/utils"
vfConfig "github.com/crc-org/vfkit/pkg/config"
"github.com/sirupsen/logrus"
)
// applehcMACAddress is a pre-defined mac address that vfkit recognizes
// and is required for network flow
const applehvMACAddress = "5a:94:ef:e4:0c:ee"
var (
vfkitCommand = "vfkit"
gvProxyWaitBackoff = 500 * time.Millisecond
gvProxyMaxBackoffAttempts = 6
vfkitCommand = "vfkit"
)
type AppleHVStubber struct {
@ -53,7 +43,7 @@ func (a AppleHVStubber) RequireExclusiveActive() bool {
func (a AppleHVStubber) CreateVM(opts define.CreateVMOpts, mc *vmconfigs.MachineConfig, ignBuilder *ignition.IgnitionBuilder) error {
mc.AppleHypervisor = new(vmconfigs.AppleHVConfig)
mc.AppleHypervisor.Vfkit = vfkit.VfkitHelper{}
mc.AppleHypervisor.Vfkit = vfkit.Helper{}
bl := vfConfig.NewEFIBootloader(fmt.Sprintf("%s/efi-bl-%s", opts.Dirs.DataDir.GetPath(), opts.Name), true)
mc.AppleHypervisor.Vfkit.VirtualMachine = vfConfig.NewVirtualMachine(uint(mc.Resources.CPUs), uint64(mc.Resources.Memory), bl)
@ -69,9 +59,13 @@ func (a AppleHVStubber) CreateVM(opts define.CreateVMOpts, mc *vmconfigs.Machine
}
// Populate the ignition file with virtiofs stuff
ignBuilder.WithUnit(generateSystemDFilesForVirtiofsMounts(virtiofsMounts)...)
virtIOIgnitionMounts, err := apple.GenerateSystemDFilesForVirtiofsMounts(virtiofsMounts)
if err != nil {
return err
}
ignBuilder.WithUnit(virtIOIgnitionMounts...)
return resizeDisk(mc, mc.Resources.DiskSize)
return apple.ResizeDisk(mc, mc.Resources.DiskSize)
}
func (a AppleHVStubber) Exists(name string) (bool, error) {
@ -97,220 +91,20 @@ func (a AppleHVStubber) SetProviderAttrs(mc *vmconfigs.MachineConfig, opts defin
if err != nil {
return err
}
if state != define.Stopped {
return errors.New("unable to change settings unless vm is stopped")
}
if opts.DiskSize != nil {
if err := resizeDisk(mc, *opts.DiskSize); err != nil {
return err
}
}
if opts.Rootful != nil && mc.HostUser.Rootful != *opts.Rootful {
if err := mc.SetRootful(*opts.Rootful); err != nil {
return err
}
}
if opts.USBs != nil {
return fmt.Errorf("changing USBs not supported for applehv machines")
}
// VFKit does not require saving memory, disk, or cpu
return nil
return apple.SetProviderAttrs(mc, opts, state)
}
func (a AppleHVStubber) StartNetworking(mc *vmconfigs.MachineConfig, cmd *gvproxy.GvproxyCommand) error {
gvProxySock, err := mc.GVProxySocket()
if err != nil {
return err
}
// make sure it does not exist before gvproxy is called
if err := gvProxySock.Delete(); err != nil {
logrus.Error(err)
}
cmd.AddVfkitSocket(fmt.Sprintf("unixgram://%s", gvProxySock.GetPath()))
return nil
return apple.StartGenericNetworking(mc, cmd)
}
func (a AppleHVStubber) StartVM(mc *vmconfigs.MachineConfig) (func() error, func() error, error) {
var (
ignitionSocket *define.VMFile
)
if bl := mc.AppleHypervisor.Vfkit.VirtualMachine.Bootloader; bl == nil {
bl := mc.AppleHypervisor.Vfkit.VirtualMachine.Bootloader
if bl == nil {
return nil, nil, fmt.Errorf("unable to determine boot loader for this machine")
}
// Add networking
netDevice, err := vfConfig.VirtioNetNew(applehvMACAddress)
if err != nil {
return nil, nil, err
}
// Set user networking with gvproxy
gvproxySocket, err := mc.GVProxySocket()
if err != nil {
return nil, nil, err
}
// Wait on gvproxy to be running and aware
if err := sockets.WaitForSocketWithBackoffs(gvProxyMaxBackoffAttempts, gvProxyWaitBackoff, gvproxySocket.GetPath(), "gvproxy"); err != nil {
return nil, nil, err
}
netDevice.SetUnixSocketPath(gvproxySocket.GetPath())
// create a one-time virtual machine for starting because we dont want all this information in the
// machineconfig if possible. the preference was to derive this stuff
vm := vfConfig.NewVirtualMachine(uint(mc.Resources.CPUs), uint64(mc.Resources.Memory), mc.AppleHypervisor.Vfkit.VirtualMachine.Bootloader)
defaultDevices, readySocket, err := getDefaultDevices(mc)
if err != nil {
return nil, nil, err
}
vm.Devices = append(vm.Devices, defaultDevices...)
vm.Devices = append(vm.Devices, netDevice)
mounts, err := virtIOFsToVFKitVirtIODevice(mc.Mounts)
if err != nil {
return nil, nil, err
}
vm.Devices = append(vm.Devices, mounts...)
// To start the VM, we need to call vfkit
cfg, err := config.Default()
if err != nil {
return nil, nil, err
}
vfkitBinaryPath, err := cfg.FindHelperBinary(vfkitCommand, true)
if err != nil {
return nil, nil, err
}
logrus.Debugf("vfkit path is: %s", vfkitBinaryPath)
cmd, err := vm.Cmd(vfkitBinaryPath)
if err != nil {
return nil, nil, err
}
if logrus.IsLevelEnabled(logrus.DebugLevel) {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
}
vfkitEndpointArgs, err := getVfKitEndpointCMDArgs(mc.AppleHypervisor.Vfkit.Endpoint)
if err != nil {
return nil, nil, err
}
machineDataDir, err := mc.DataDir()
if err != nil {
return nil, nil, err
}
cmd.Args = append(cmd.Args, vfkitEndpointArgs...)
firstBoot, err := mc.IsFirstBoot()
if err != nil {
return nil, nil, err
}
if logrus.IsLevelEnabled(logrus.DebugLevel) {
debugDevArgs, err := getDebugDevicesCMDArgs()
if err != nil {
return nil, nil, err
}
cmd.Args = append(cmd.Args, debugDevArgs...)
cmd.Args = append(cmd.Args, "--gui") // add command line switch to pop the gui open
}
if firstBoot {
// If this is the first boot of the vm, we need to add the vsock
// device to vfkit so we can inject the ignition file
socketName := fmt.Sprintf("%s-%s", mc.Name, ignitionSocketName)
ignitionSocket, err = machineDataDir.AppendToNewVMFile(socketName, &socketName)
if err != nil {
return nil, nil, err
}
if err := ignitionSocket.Delete(); err != nil {
logrus.Errorf("unable to delete ignition socket: %q", err)
}
ignitionVsockDeviceCLI, err := getIgnitionVsockDeviceAsCLI(ignitionSocket.GetPath())
if err != nil {
return nil, nil, err
}
cmd.Args = append(cmd.Args, ignitionVsockDeviceCLI...)
logrus.Debug("first boot detected")
logrus.Debugf("serving ignition file over %s", ignitionSocket.GetPath())
go func() {
if err := serveIgnitionOverSock(ignitionSocket, mc); err != nil {
logrus.Error(err)
}
logrus.Debug("ignition vsock server exited")
}()
}
logrus.Debugf("listening for ready on: %s", readySocket.GetPath())
if err := readySocket.Delete(); err != nil {
logrus.Warnf("unable to delete previous ready socket: %q", err)
}
readyListen, err := net.Listen("unix", readySocket.GetPath())
if err != nil {
return nil, nil, err
}
logrus.Debug("waiting for ready notification")
readyChan := make(chan error)
go sockets.ListenAndWaitOnSocket(readyChan, readyListen)
logrus.Debugf("vfkit command-line: %v", cmd.Args)
if err := cmd.Start(); err != nil {
return nil, nil, err
}
returnFunc := func() error {
processErrChan := make(chan error)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
defer close(processErrChan)
for {
select {
case <-ctx.Done():
return
default:
}
if err := checkProcessRunning("vfkit", cmd.Process.Pid); err != nil {
processErrChan <- err
return
}
// lets poll status every half second
time.Sleep(500 * time.Millisecond)
}
}()
// wait for either socket or to be ready or process to have exited
select {
case err := <-processErrChan:
if err != nil {
return err
}
case err := <-readyChan:
if err != nil {
return err
}
logrus.Debug("ready notification received")
}
return nil
}
return cmd.Process.Release, returnFunc, nil
return apple.StartGenericAppleVM(mc, vfkitCommand, bl, mc.AppleHypervisor.Vfkit.Endpoint)
}
func (a AppleHVStubber) StopHostNetworking(_ *vmconfigs.MachineConfig, _ define.VMType) error {

View File

@ -12,6 +12,7 @@ const (
WSLVirt
AppleHvVirt
HyperVVirt
LibKrun
UnknownVirt
)
@ -22,6 +23,7 @@ const (
qemu = "qemu"
appleHV = "applehv"
hyperV = "hyperv"
libkrun = "libkrun"
)
func (v VMType) String() string {
@ -32,6 +34,23 @@ func (v VMType) String() string {
return appleHV
case HyperVVirt:
return hyperV
case LibKrun:
return libkrun
}
return qemu
}
// DiskType returns a string representation that matches the OCI artifact
// type on the container image registry
func (v VMType) DiskType() string {
switch v {
case WSLVirt:
return wsl
// Both AppleHV and Libkrun use same raw disk flavor
case AppleHvVirt, LibKrun:
return appleHV
case HyperVVirt:
return hyperV
}
return qemu
}
@ -44,6 +63,8 @@ func (v VMType) ImageFormat() ImageFormat {
return Raw
case HyperVVirt:
return Vhdx
case LibKrun:
return Raw
}
return Qcow
}
@ -56,6 +77,8 @@ func ParseVMType(input string, emptyFallback VMType) (VMType, error) {
return WSLVirt, nil
case appleHV:
return AppleHvVirt, nil
case libkrun:
return LibKrun, nil
case hyperV:
return HyperVVirt, nil
case "":

View File

@ -23,7 +23,7 @@ func CreateReadyUnitFile(provider define.VMType, opts *ReadyUnitOpts) (string, e
readyUnit.Add("Unit", "Requires", "dev-virtio\\x2dports-vport1p1.device")
readyUnit.Add("Unit", "After", "systemd-user-sessions.service")
readyUnit.Add("Service", "ExecStart", "/bin/sh -c '/usr/bin/echo Ready >/dev/vport1p1'")
case define.AppleHvVirt:
case define.AppleHvVirt, define.LibKrun:
readyUnit.Add("Unit", "Requires", "dev-virtio\\x2dports-vsock.device")
readyUnit.Add("Service", "ExecStart", "/bin/sh -c '/usr/bin/echo Ready | socat - VSOCK-CONNECT:2:1025'")
case define.HyperVVirt:

View File

@ -0,0 +1,141 @@
//go:build darwin
package libkrun
import (
"fmt"
"strconv"
gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
"github.com/containers/podman/v5/pkg/machine"
"github.com/containers/podman/v5/pkg/machine/apple"
"github.com/containers/podman/v5/pkg/machine/apple/vfkit"
"github.com/containers/podman/v5/pkg/machine/define"
"github.com/containers/podman/v5/pkg/machine/ignition"
"github.com/containers/podman/v5/pkg/machine/shim/diskpull"
"github.com/containers/podman/v5/pkg/machine/vmconfigs"
"github.com/containers/podman/v5/utils"
vfConfig "github.com/crc-org/vfkit/pkg/config"
)
const (
krunkitBinary = "krunkit"
localhostURI = "http://localhost"
)
type LibKrunStubber struct {
vmconfigs.AppleHVConfig
}
func (l LibKrunStubber) CreateVM(opts define.CreateVMOpts, mc *vmconfigs.MachineConfig, builder *ignition.IgnitionBuilder) error {
mc.LibKrunHypervisor = new(vmconfigs.LibKrunConfig)
mc.LibKrunHypervisor.KRun = vfkit.Helper{}
bl := vfConfig.NewEFIBootloader(fmt.Sprintf("%s/efi-bl-%s", opts.Dirs.DataDir.GetPath(), opts.Name), true)
mc.LibKrunHypervisor.KRun.VirtualMachine = vfConfig.NewVirtualMachine(uint(mc.Resources.CPUs), uint64(mc.Resources.Memory), bl)
randPort, err := utils.GetRandomPort()
if err != nil {
return err
}
mc.LibKrunHypervisor.KRun.Endpoint = localhostURI + ":" + strconv.Itoa(randPort)
virtiofsMounts := make([]machine.VirtIoFs, 0, len(mc.Mounts))
for _, mnt := range mc.Mounts {
virtiofsMounts = append(virtiofsMounts, machine.MountToVirtIOFs(mnt))
}
// Populate the ignition file with virtiofs stuff
virtIOIgnitionMounts, err := apple.GenerateSystemDFilesForVirtiofsMounts(virtiofsMounts)
if err != nil {
return err
}
builder.WithUnit(virtIOIgnitionMounts...)
return apple.ResizeDisk(mc, mc.Resources.DiskSize)
}
func (l LibKrunStubber) GetDisk(userInputPath string, dirs *define.MachineDirs, mc *vmconfigs.MachineConfig) error {
return diskpull.GetDisk(userInputPath, dirs, mc.ImagePath, l.VMType(), mc.Name)
}
func (l LibKrunStubber) PrepareIgnition(mc *vmconfigs.MachineConfig, ignBuilder *ignition.IgnitionBuilder) (*ignition.ReadyUnitOpts, error) {
return nil, nil
}
func (l LibKrunStubber) Exists(name string) (bool, error) {
// not applicable for libkrun (same as applehv)
return false, nil
}
func (l LibKrunStubber) MountType() vmconfigs.VolumeMountType {
return vmconfigs.VirtIOFS
}
func (l LibKrunStubber) MountVolumesToVM(mc *vmconfigs.MachineConfig, quiet bool) error {
return nil
}
func (l LibKrunStubber) Remove(mc *vmconfigs.MachineConfig) ([]string, func() error, error) {
return []string{}, func() error { return nil }, nil
}
func (l LibKrunStubber) RemoveAndCleanMachines(dirs *define.MachineDirs) error {
return nil
}
func (l LibKrunStubber) SetProviderAttrs(mc *vmconfigs.MachineConfig, opts define.SetOptions) error {
state, err := l.State(mc, false)
if err != nil {
return err
}
return apple.SetProviderAttrs(mc, opts, state)
}
func (l LibKrunStubber) StartNetworking(mc *vmconfigs.MachineConfig, cmd *gvproxy.GvproxyCommand) error {
return apple.StartGenericNetworking(mc, cmd)
}
func (l LibKrunStubber) PostStartNetworking(mc *vmconfigs.MachineConfig, noInfo bool) error {
return nil
}
func (l LibKrunStubber) StartVM(mc *vmconfigs.MachineConfig) (func() error, func() error, error) {
bl := mc.LibKrunHypervisor.KRun.VirtualMachine.Bootloader
if bl == nil {
return nil, nil, fmt.Errorf("unable to determine boot loader for this machine")
}
return apple.StartGenericAppleVM(mc, krunkitBinary, bl, mc.LibKrunHypervisor.KRun.Endpoint)
}
func (l LibKrunStubber) State(mc *vmconfigs.MachineConfig, bypass bool) (define.Status, error) {
return mc.LibKrunHypervisor.KRun.State()
}
func (l LibKrunStubber) StopVM(mc *vmconfigs.MachineConfig, hardStop bool) error {
return mc.LibKrunHypervisor.KRun.Stop(hardStop, true)
}
func (l LibKrunStubber) StopHostNetworking(mc *vmconfigs.MachineConfig, vmType define.VMType) error {
return nil
}
func (l LibKrunStubber) VMType() define.VMType {
return define.LibKrun
}
func (l LibKrunStubber) UserModeNetworkEnabled(mc *vmconfigs.MachineConfig) bool {
return true
}
func (l LibKrunStubber) UseProviderNetworkSetup() bool {
return false
}
func (l LibKrunStubber) RequireExclusiveActive() bool {
return true
}
func (l LibKrunStubber) UpdateSSHPort(mc *vmconfigs.MachineConfig, port int) error {
return nil
}

View File

@ -87,7 +87,7 @@ func NewOCIArtifactPull(ctx context.Context, dirs *define.MachineDirs, endpoint
diskOpts := DiskArtifactOpts{
arch: arch,
diskType: vmType.String(),
diskType: vmType.DiskType(),
os: machineOS,
}

View File

@ -7,6 +7,7 @@ import (
"github.com/containers/common/pkg/config"
"github.com/containers/podman/v5/pkg/machine/applehv"
"github.com/containers/podman/v5/pkg/machine/define"
"github.com/containers/podman/v5/pkg/machine/libkrun"
"github.com/containers/podman/v5/pkg/machine/vmconfigs"
"github.com/sirupsen/logrus"
)
@ -29,6 +30,8 @@ func Get() (vmconfigs.VMProvider, error) {
switch resolvedVMType {
case define.AppleHvVirt:
return new(applehv.AppleHVStubber), nil
case define.LibKrun:
return new(libkrun.LibKrunStubber), nil
default:
return nil, fmt.Errorf("unsupported virtualization provider: `%s`", resolvedVMType.String())
}

View File

@ -33,10 +33,11 @@ type MachineConfig struct {
ImagePath *define.VMFile // Temporary only until a proper image struct is worked out
// Provider stuff
AppleHypervisor *AppleHVConfig `json:",omitempty"`
QEMUHypervisor *QEMUConfig `json:",omitempty"`
HyperVHypervisor *HyperVConfig `json:",omitempty"`
WSLHypervisor *WSLConfig `json:",omitempty"`
AppleHypervisor *AppleHVConfig `json:",omitempty"`
HyperVHypervisor *HyperVConfig `json:",omitempty"`
LibKrunHypervisor *LibKrunConfig `json:",omitempty"`
QEMUHypervisor *QEMUConfig `json:",omitempty"`
WSLHypervisor *WSLConfig `json:",omitempty"`
lock *lockfile.LockFile //nolint:unused

View File

@ -19,6 +19,7 @@ type QEMUConfig struct {
// Stubs
type AppleHVConfig struct{}
type HyperVConfig struct{}
type LibKrunConfig struct{}
type WSLConfig struct{}
func getHostUID() int {

View File

@ -3,12 +3,16 @@ package vmconfigs
import (
"os"
"github.com/containers/podman/v5/pkg/machine/applehv/vfkit"
"github.com/containers/podman/v5/pkg/machine/apple/vfkit"
)
type AppleHVConfig struct {
// The VFKit endpoint where we can interact with the VM
Vfkit vfkit.VfkitHelper
Vfkit vfkit.Helper
}
type LibKrunConfig struct {
KRun vfkit.Helper
}
// Stubs

View File

@ -18,8 +18,9 @@ type WSLConfig struct {
}
// Stubs
type QEMUConfig struct{}
type AppleHVConfig struct{}
type LibKrunConfig struct{}
type QEMUConfig struct{}
func getHostUID() int {
return 1000