Implement pod-network-reload

This adds a new command, 'podman network reload', to reload the
networks of existing containers, forcing recreation of firewall
rules after e.g. `firewall-cmd --reload` wipes them out.

Under the hood, this works by calling CNI to tear down the
existing network, then recreate it using identical settings. We
request that CNI preserve the old IP and MAC address in most
cases (where the container only had 1 IP/MAC), but there will be
some downtime inherent to the teardown/bring-up approach. The
architecture of CNI doesn't really make doing this without
downtime easy (or maybe even possible...).

At present, this only works for root Podman, and only locally.
I don't think there is much of a point to adding remote support
(this is very much a local debugging command), but I think adding
rootless support (to kill/recreate slirp4netns) could be
valuable.

Signed-off-by: Matthew Heon <matthew.heon@pm.me>
Signed-off-by: Paul Holzinger <paul.holzinger@web.de>

cmd/podman/networks/reload.go

@@ -0,0 +1,69 @@
+package network
+
+import (
+	"fmt"
+
+	"github.com/containers/podman/v2/cmd/podman/common"
+	"github.com/containers/podman/v2/cmd/podman/registry"
+	"github.com/containers/podman/v2/cmd/podman/utils"
+	"github.com/containers/podman/v2/cmd/podman/validate"
+	"github.com/containers/podman/v2/pkg/domain/entities"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+var (
+	networkReloadDescription = `reload container networks, recreating firewall rules`
+	networkReloadCommand     = &cobra.Command{
+		Use:   "reload [options] [CONTAINER...]",
+		Short: "Reload firewall rules for one or more containers",
+		Long:  networkReloadDescription,
+		RunE:  networkReload,
+		Args: func(cmd *cobra.Command, args []string) error {
+			return validate.CheckAllLatestAndCIDFile(cmd, args, false, false)
+		},
+		ValidArgsFunction: common.AutocompleteContainers,
+		Example: `podman network reload --latest
+  podman network reload 3c13ef6dd843
+  podman network reload test1 test2`,
+		Annotations: map[string]string{
+			registry.ParentNSRequired: "",
+		},
+	}
+)
+
+var (
+	reloadOptions entities.NetworkReloadOptions
+)
+
+func reloadFlags(flags *pflag.FlagSet) {
+	flags.BoolVarP(&reloadOptions.All, "all", "a", false, "Reload network configuration of all containers")
+}
+
+func init() {
+	registry.Commands = append(registry.Commands, registry.CliCommand{
+		Mode:    []entities.EngineMode{entities.ABIMode},
+		Command: networkReloadCommand,
+		Parent:  networkCmd,
+	})
+	reloadFlags(networkReloadCommand.Flags())
+	validate.AddLatestFlag(networkReloadCommand, &reloadOptions.Latest)
+}
+
+func networkReload(cmd *cobra.Command, args []string) error {
+	responses, err := registry.ContainerEngine().NetworkReload(registry.Context(), args, reloadOptions)
+	if err != nil {
+		return err
+	}
+
+	var errs utils.OutputErrors
+	for _, r := range responses {
+		if r.Err == nil {
+			fmt.Println(r.Id)
+		} else {
+			errs = append(errs, r.Err)
+		}
+	}
+
+	return errs.PrintErrors()
+}

docs/source/markdown/podman-network-reload.1.md

@@ -0,0 +1,62 @@
+% podman-network-reload(1)
+
+## NAME
+podman\-network\-reload - Reload network configuration for containers
+
+## SYNOPSIS
+**podman network reload** [*options*] [*container...*]
+
+## DESCRIPTION
+Reload one or more container network configurations.
+
+Rootful Podman relies on iptables rules in order to provide network connectivity. If the iptables rules are deleted,
+this happens for example with `firewall-cmd --reload`, the container loses network connectivity. This command restores
+the network connectivity.
+
+This command is not available for rootless users since rootless containers are not affected by such connectivity problems.
+
+## OPTIONS
+#### **--all**, **-a**
+
+Reload network configuration of all containers.
+
+#### **--latest**, **-l**
+
+Instead of providing the container name or ID, use the last created container. If you use methods other than Podman
+to run containers such as CRI-O, the last started container could be from either of those methods.
+
+The latest option is not supported on the remote client.
+
+## EXAMPLE
+
+Reload the network configuration after a firewall reload.
+
+```
+# podman run -p 80:80 -d nginx
+b1b538e8bc4078fc3ee1c95b666ebc7449b9a97bacd15bcbe464a29e1be59c1c
+# curl 127.0.0.1
+works
+# sudo firewall-cmd --reload
+success
+# curl 127.0.0.1
+hangs
+# podman network reload b1b538e8bc40
+b1b538e8bc4078fc3ee1c95b666ebc7449b9a97bacd15bcbe464a29e1be59c1c
+# curl 127.0.0.1
+works
+```
+
+Reload the network configuration for all containers.
+
+```
+# podman network reload --all
+b1b538e8bc4078fc3ee1c95b666ebc7449b9a97bacd15bcbe464a29e1be59c1c
+fe7e8eca56f844ec33af10f0aa3b31b44a172776e3277b9550a623ed5d96e72b
+```
+
+
+## SEE ALSO
+podman(1), podman-network(1)
+
+## HISTORY
+December 2020, Originally compiled by Paul Holzinger <paul.holzinger@web.de>

docs/source/markdown/podman-network.1.md

@@ -12,12 +12,13 @@ The network command manages CNI networks for Podman. It is not supported for roo
 ## COMMANDS
 
 | Command    | Man Page                                                       | Description                                                         |
-| -------  | --------------------------------------------------- | ---------------------------------------------------------------------------- |
+| ---------- | -------------------------------------------------------------- | ------------------------------------------------------------------- |
 | connect    | [podman-network-connect(1)](podman-network-connect.1.md)       | Connect a container to a network                                    |
 | create     | [podman-network-create(1)](podman-network-create.1.md)         | Create a Podman CNI network                                         |
 | disconnect | [podman-network-disconnect(1)](podman-network-disconnect.1.md) | Disconnect a container from a network                               |
 | inspect    | [podman-network-inspect(1)](podman-network-inspect.1.md)       | Displays the raw CNI network configuration for one or more networks |
 | ls         | [podman-network-ls(1)](podman-network-ls.1.md)                 | Display a summary of CNI networks                                   |
+| reload     | [podman-network-reload(1)](podman-network-reload.1.md)         | Reload network configuration for containers                         |
 | rm         | [podman-network-rm(1)](podman-network-rm.1.md)                 | Remove one or more CNI networks                                     |
 
 ## SEE ALSO

docs/source/network.rst

@@ -11,4 +11,6 @@ Network
 
 :doc:`ls <markdown/podman-network-ls.1>` network list
 
+:doc:`reload <markdown/podman-network-reload.1>` network reload
+
 :doc:`rm <markdown/podman-network-rm.1>` network rm

libpod/container_api.go

@@ -639,6 +639,32 @@ func (c *Container) Sync() error {
 	return nil
 }
 
+// ReloadNetwork reconfigures the container's network.
+// Technically speaking, it will tear down and then reconfigure the container's
+// network namespace, which will result in all firewall rules being recreated.
+// It is mostly intended to be used in cases where the system firewall has been
+// reloaded, and existing rules have been wiped out. It is expected that some
+// downtime will result, as the rules are destroyed as part of this process.
+// At present, this only works on root containers; it may be expanded to restart
+// slirp4netns in the future to work with rootless containers as well.
+// Requires that the container must be running or created.
+func (c *Container) ReloadNetwork() error {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return err
+		}
+	}
+
+	if !c.ensureState(define.ContainerStateCreated, define.ContainerStateRunning) {
+		return errors.Wrapf(define.ErrCtrStateInvalid, "cannot reload network unless container network has been configured")
+	}
+
+	return c.reloadNetwork()
+}
+
 // Refresh is DEPRECATED and REMOVED.
 func (c *Container) Refresh(ctx context.Context) error {
 	// This has been deprecated for a long while, and is in the process of

libpod/container_internal_linux.go

@@ -230,6 +230,19 @@ func (c *Container) cleanupNetwork() error {
 	return nil
 }
 
+// reloadNetwork reloads the network for the given container, recreating
+// firewall rules.
+func (c *Container) reloadNetwork() error {
+	result, err := c.runtime.reloadContainerNetwork(c)
+	if err != nil {
+		return err
+	}
+
+	c.state.NetworkStatus = result
+
+	return c.save()
+}
+
 func (c *Container) getUserOverrides() *lookup.Overrides {
 	var hasPasswdFile, hasGroupFile bool
 	overrides := lookup.Overrides{}

libpod/container_internal_unsupported.go

@@ -50,6 +50,10 @@ func (c *Container) cleanupOverlayMounts() error {
 	return nil
 }
 
+func (c *Container) reloadNetwork() error {
+	return define.ErrNotImplemented
+}
+
 func (c *Container) getUserOverrides() *lookup.Overrides {
 	return nil
 }

libpod/networking_linux.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"syscall"
@@ -740,8 +741,9 @@ func (r *Runtime) closeNetNS(ctr *Container) error {
 	return nil
 }
 
-// Tear down a network namespace, undoing all state associated with it.
-func (r *Runtime) teardownNetNS(ctr *Container) error {
+// Tear down a container's CNI network configuration, but do not tear down the
+// namespace itself.
+func (r *Runtime) teardownCNI(ctr *Container) error {
 	if ctr.state.NetNS == nil {
 		// The container has no network namespace, we're set
 		return nil
@@ -780,6 +782,19 @@ func (r *Runtime) teardownNetNS(ctr *Container) error {
 			return errors.Wrapf(err, "error tearing down CNI namespace configuration for container %s", ctr.ID())
 		}
 	}
+	return nil
+}
+
+// Tear down a network namespace, undoing all state associated with it.
+func (r *Runtime) teardownNetNS(ctr *Container) error {
+	if err := r.teardownCNI(ctr); err != nil {
+		return err
+	}
+
+	networks, _, err := ctr.networks()
+	if err != nil {
+		return err
+	}
 
 	// CNI-in-slirp4netns
 	if rootless.IsRootless() && len(networks) != 0 {
@@ -820,6 +835,68 @@ func getContainerNetNS(ctr *Container) (string, error) {
 	return "", nil
 }
 
+// Reload only works with containers with a configured network.
+// It will tear down, and then reconfigure, the network of the container.
+// This is mainly used when a reload of firewall rules wipes out existing
+// firewall configuration.
+// Efforts will be made to preserve MAC and IP addresses, but this only works if
+// the container only joined a single CNI network, and was only assigned a
+// single MAC or IP.
+// Only works on root containers at present, though in the future we could
+// extend this to stop + restart slirp4netns
+func (r *Runtime) reloadContainerNetwork(ctr *Container) ([]*cnitypes.Result, error) {
+	if ctr.state.NetNS == nil {
+		return nil, errors.Wrapf(define.ErrCtrStateInvalid, "container %s network is not configured, refusing to reload", ctr.ID())
+	}
+	if rootless.IsRootless() || ctr.config.NetMode.IsSlirp4netns() {
+		return nil, errors.Wrapf(define.ErrRootless, "network reload only supported for root containers")
+	}
+
+	logrus.Infof("Going to reload container %s network", ctr.ID())
+
+	var requestedIP net.IP
+	var requestedMAC net.HardwareAddr
+	// Set requested IP and MAC address, if possible.
+	if len(ctr.state.NetworkStatus) == 1 {
+		result := ctr.state.NetworkStatus[0]
+		if len(result.IPs) == 1 {
+			resIP := result.IPs[0]
+
+			requestedIP = resIP.Address.IP
+			ctr.requestedIP = requestedIP
+			logrus.Debugf("Going to preserve container %s IP address %s", ctr.ID(), ctr.requestedIP.String())
+
+			if resIP.Interface != nil && *resIP.Interface < len(result.Interfaces) && *resIP.Interface >= 0 {
+				var err error
+				requestedMAC, err = net.ParseMAC(result.Interfaces[*resIP.Interface].Mac)
+				if err != nil {
+					return nil, errors.Wrapf(err, "error parsing container %s MAC address %s", ctr.ID(), result.Interfaces[*resIP.Interface].Mac)
+				}
+				ctr.requestedMAC = requestedMAC
+				logrus.Debugf("Going to preserve container %s MAC address %s", ctr.ID(), ctr.requestedMAC.String())
+			}
+		}
+	}
+
+	err := r.teardownCNI(ctr)
+	if err != nil {
+		// teardownCNI will error if the iptables rules do not exists and this is the case after
+		// a firewall reload. The purpose of network reload is to recreate the rules if they do
+		// not exists so we should not log this specific error as error. This would confuse users otherwise.
+		b, rerr := regexp.MatchString("Couldn't load target `CNI-[a-f0-9]{24}':No such file or directory", err.Error())
+		if rerr == nil && !b {
+			logrus.Error(err)
+		} else {
+			logrus.Info(err)
+		}
+	}
+
+	// teardownCNI will clean the requested IP and MAC so we need to set them again
+	ctr.requestedIP = requestedIP
+	ctr.requestedMAC = requestedMAC
+	return r.configureNetNS(ctr, ctr.state.NetNS)
+}
+
 func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) {
 	var netStats *netlink.LinkStatistics
 	// rootless v2 cannot seem to resolve its network connection to
@@ -983,12 +1060,12 @@ func resultToBasicNetworkConfig(result *cnitypes.Result) (define.InspectBasicNet
 			config.IPAddress = ctrIP.Address.IP.String()
 			config.IPPrefixLen = size
 			config.Gateway = ctrIP.Gateway.String()
-			if ctrIP.Interface != nil && *ctrIP.Interface < len(result.Interfaces) && *ctrIP.Interface > 0 {
+			if ctrIP.Interface != nil && *ctrIP.Interface < len(result.Interfaces) && *ctrIP.Interface >= 0 {
 				config.MacAddress = result.Interfaces[*ctrIP.Interface].Mac
 			}
 		case ctrIP.Version == "4" && config.IPAddress != "":
 			config.SecondaryIPAddresses = append(config.SecondaryIPAddresses, ctrIP.Address.String())
-			if ctrIP.Interface != nil && *ctrIP.Interface < len(result.Interfaces) && *ctrIP.Interface > 0 {
+			if ctrIP.Interface != nil && *ctrIP.Interface < len(result.Interfaces) && *ctrIP.Interface >= 0 {
 				config.AdditionalMacAddresses = append(config.AdditionalMacAddresses, result.Interfaces[*ctrIP.Interface].Mac)
 			}
 		case ctrIP.Version == "6" && config.IPAddress == "":

libpod/networking_unsupported.go

@@ -2,7 +2,10 @@
 
 package libpod
 
-import "github.com/containers/podman/v2/libpod/define"
+import (
+	cnitypes "github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containers/podman/v2/libpod/define"
+)
 
 func (r *Runtime) setupRootlessNetNS(ctr *Container) error {
 	return define.ErrNotImplemented
@@ -28,6 +31,10 @@ func (c *Container) getContainerNetworkInfo() (*define.InspectNetworkSettings, e
 	return nil, define.ErrNotImplemented
 }
 
+func (r *Runtime) reloadContainerNetwork(ctr *Container) ([]*cnitypes.Result, error) {
+	return nil, define.ErrNotImplemented
+}
+
 func getCNINetworksDir() (string, error) {
 	return "", define.ErrNotImplemented
 }

pkg/domain/entities/engine_container.go

@@ -55,6 +55,7 @@ type ContainerEngine interface {
 	NetworkDisconnect(ctx context.Context, networkname string, options NetworkDisconnectOptions) error
 	NetworkInspect(ctx context.Context, namesOrIds []string, options InspectOptions) ([]NetworkInspectReport, []error, error)
 	NetworkList(ctx context.Context, options NetworkListOptions) ([]*NetworkListReport, error)
+	NetworkReload(ctx context.Context, names []string, options NetworkReloadOptions) ([]*NetworkReloadReport, error)
 	NetworkRm(ctx context.Context, namesOrIds []string, options NetworkRmOptions) ([]*NetworkRmReport, error)
 	PlayKube(ctx context.Context, path string, opts PlayKubeOptions) (*PlayKubeReport, error)
 	PodCreate(ctx context.Context, opts PodCreateOptions) (*PodCreateReport, error)

pkg/domain/entities/network.go

@@ -22,6 +22,19 @@ type NetworkListReport struct {
 // NetworkInspectReport describes the results from inspect networks
 type NetworkInspectReport map[string]interface{}
 
+// NetworkReloadOptions describes options for reloading container network
+// configuration.
+type NetworkReloadOptions struct {
+	All    bool
+	Latest bool
+}
+
+// NetworkReloadReport describes the results of reloading a container network.
+type NetworkReloadReport struct {
+	Id  string
+	Err error
+}
+
 // NetworkRmOptions describes options for removing networks
 type NetworkRmOptions struct {
 	Force bool

pkg/domain/infra/abi/network.go

@@ -60,6 +60,26 @@ func (ic *ContainerEngine) NetworkInspect(ctx context.Context, namesOrIds []stri
 	return rawCNINetworks, errs, nil
 }
 
+func (ic *ContainerEngine) NetworkReload(ctx context.Context, names []string, options entities.NetworkReloadOptions) ([]*entities.NetworkReloadReport, error) {
+	ctrs, err := getContainersByContext(options.All, options.Latest, names, ic.Libpod)
+	if err != nil {
+		return nil, err
+	}
+
+	reports := make([]*entities.NetworkReloadReport, 0, len(ctrs))
+	for _, ctr := range ctrs {
+		report := new(entities.NetworkReloadReport)
+		report.Id = ctr.ID()
+		report.Err = ctr.ReloadNetwork()
+		if options.All && errors.Cause(report.Err) == define.ErrCtrStateInvalid {
+			continue
+		}
+		reports = append(reports, report)
+	}
+
+	return reports, nil
+}
+
 func (ic *ContainerEngine) NetworkRm(ctx context.Context, namesOrIds []string, options entities.NetworkRmOptions) ([]*entities.NetworkRmReport, error) {
 	reports := []*entities.NetworkRmReport{}
 

pkg/domain/infra/tunnel/network.go

@@ -35,6 +35,10 @@ func (ic *ContainerEngine) NetworkInspect(ctx context.Context, namesOrIds []stri
 	return reports, errs, nil
 }
 
+func (ic *ContainerEngine) NetworkReload(ctx context.Context, names []string, options entities.NetworkReloadOptions) ([]*entities.NetworkReloadReport, error) {
+	return nil, errors.New("not implemented")
+}
+
 func (ic *ContainerEngine) NetworkRm(ctx context.Context, namesOrIds []string, options entities.NetworkRmOptions) ([]*entities.NetworkRmReport, error) {
 	reports := make([]*entities.NetworkRmReport, 0, len(namesOrIds))
 	for _, name := range namesOrIds {

test/system/500-networking.bats

@@ -116,4 +116,68 @@ load helpers
     fi
 }
 
+@test "podman network reload" {
+    skip_if_remote "podman network reload does not have remote support"
+    skip_if_rootless "podman network reload does not work rootless"
+
+    random_1=$(random_string 30)
+    HOST_PORT=12345
+    SERVER=http://127.0.0.1:$HOST_PORT
+
+    # Create a test file with random content
+    INDEX1=$PODMAN_TMPDIR/hello.txt
+    echo $random_1 > $INDEX1
+
+    # Bind-mount this file with a different name to a container running httpd
+    run_podman run -d --name myweb -p "$HOST_PORT:80" \
+               -v $INDEX1:/var/www/index.txt \
+               -w /var/www \
+               $IMAGE /bin/busybox-extras httpd -f -p 80
+    cid=$output
+
+    run_podman inspect $cid --format "{{.NetworkSettings.IPAddress}}"
+    ip="$output"
+    run_podman inspect $cid --format "{{.NetworkSettings.MacAddress}}"
+    mac="$output"
+
+    # Verify http contents: curl from localhost
+    run curl -s $SERVER/index.txt
+    is "$output" "$random_1" "curl 127.0.0.1:/index.txt"
+
+    # flush the CNI iptables here
+    run iptables -t nat -F CNI-HOSTPORT-DNAT
+
+    # check that we cannot curl (timeout after 5 sec)
+    run timeout 5 curl -s $SERVER/index.txt
+    if [ "$status" -ne 124 ]; then
+        die "curl did not timeout, status code: $status"
+    fi
+
+    # reload the network to recreate the iptables rules
+    run_podman network reload $cid
+    is "$output" "$cid" "Output does not match container ID"
+
+    # check that we still have the same mac and ip
+    run_podman inspect $cid --format "{{.NetworkSettings.IPAddress}}"
+    is "$output" "$ip" "IP address changed after podman network reload"
+    run_podman inspect $cid --format "{{.NetworkSettings.MacAddress}}"
+    is "$output" "$mac" "MAC address changed after podman network reload"
+
+    # check that we can still curl
+    run curl -s $SERVER/index.txt
+    is "$output" "$random_1" "curl 127.0.0.1:/index.txt"
+
+    # make sure --all is working and that this
+    # cmd also works if the iptables still exists
+    run_podman network reload --all
+    is "$output" "$cid" "Output does not match container ID"
+
+    # check that we can still curl
+    run curl -s $SERVER/index.txt
+    is "$output" "$random_1" "curl 127.0.0.1:/index.txt"
+
+    # cleanup the container
+    run_podman rm -f $cid
+}
+
 # vim: filetype=sh