Merge pull request #16880 from ygalblum/kube-publish-ports

Kube Play - allow setting and overriding published host ports

cmd/podman/kube/play.go

@@ -151,6 +151,10 @@ func playFlags(cmd *cobra.Command) {
 	replaceFlagName := "replace"
 	flags.BoolVar(&playOptions.Replace, replaceFlagName, false, "Delete and recreate pods defined in the YAML file")
 
+	publishPortsFlagName := "publish"
+	flags.StringSliceVar(&playOptions.PublishPorts, publishPortsFlagName, []string{}, "Publish a container's port, or a range of ports, to the host")
+	_ = cmd.RegisterFlagCompletionFunc(publishPortsFlagName, completion.AutocompleteNone)
+
 	if !registry.IsRemote() {
 		certDirFlagName := "cert-dir"
 		flags.StringVar(&playOptions.CertDir, certDirFlagName, "", "`Pathname` of a directory containing TLS certificates and keys")

docs/source/markdown/podman-kube-play.1.md.in

@@ -170,6 +170,13 @@ When no network option is specified and *host* network mode is not configured in
 
 This option conflicts with host added in the Kubernetes YAML.
 
+#### **--publish**=*[[ip:][hostPort]:]containerPort[/protocol]*
+
+Define or override a port definition in the YAML file.
+
+The lists of ports in the YAML file and the command line are merged. Matching is done by using the **containerPort** field.
+If **containerPort** exists in both the YAML file and the option, the latter takes precedence.
+
 #### **--quiet**, **-q**
 
 Suppress output information when pulling images

pkg/api/handlers/libpod/kube.go

@@ -19,15 +19,16 @@ func KubePlay(w http.ResponseWriter, r *http.Request) {
 	runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime)
 	decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
 	query := struct {
-		Annotations map[string]string `schema:"annotations"`
+		Annotations  map[string]string `schema:"annotations"`
-		Network     []string          `schema:"network"`
+		Network      []string          `schema:"network"`
-		TLSVerify   bool              `schema:"tlsVerify"`
+		TLSVerify    bool              `schema:"tlsVerify"`
-		LogDriver   string            `schema:"logDriver"`
+		LogDriver    string            `schema:"logDriver"`
-		LogOptions  []string          `schema:"logOptions"`
+		LogOptions   []string          `schema:"logOptions"`
-		Start       bool              `schema:"start"`
+		Start        bool              `schema:"start"`
-		StaticIPs   []string          `schema:"staticIPs"`
+		StaticIPs    []string          `schema:"staticIPs"`
-		StaticMACs  []string          `schema:"staticMACs"`
+		StaticMACs   []string          `schema:"staticMACs"`
-		NoHosts     bool              `schema:"noHosts"`
+		NoHosts      bool              `schema:"noHosts"`
+		PublishPorts []string          `schema:"publishPorts"`
 	}{
 		TLSVerify: true,
 		Start:     true,
@@ -82,18 +83,19 @@ func KubePlay(w http.ResponseWriter, r *http.Request) {
 
 	containerEngine := abi.ContainerEngine{Libpod: runtime}
 	options := entities.PlayKubeOptions{
-		Annotations: query.Annotations,
+		Annotations:  query.Annotations,
-		Authfile:    authfile,
+		Authfile:     authfile,
-		Username:    username,
+		Username:     username,
-		Password:    password,
+		Password:     password,
-		Networks:    query.Network,
+		Networks:     query.Network,
-		NoHosts:     query.NoHosts,
+		NoHosts:      query.NoHosts,
-		Quiet:       true,
+		Quiet:        true,
-		LogDriver:   logDriver,
+		LogDriver:    logDriver,
-		LogOptions:  query.LogOptions,
+		LogOptions:   query.LogOptions,
-		StaticIPs:   staticIPs,
+		StaticIPs:    staticIPs,
-		StaticMACs:  staticMACs,
+		StaticMACs:   staticMACs,
-		IsRemote:    true,
+		IsRemote:     true,
+		PublishPorts: query.PublishPorts,
 	}
 	if _, found := r.URL.Query()["tlsVerify"]; found {
 		options.SkipTLSVerify = types.NewOptionalBool(!query.TLSVerify)

pkg/bindings/kube/types.go

@@ -48,6 +48,8 @@ type PlayOptions struct {
 	Userns *string
 	// Force - remove volumes on --down
 	Force *bool
+	// PublishPorts - configure how to expose ports configured inside the K8S YAML file
+	PublishPorts []string
 }
 
 // ApplyOptions are optional options for applying kube YAML files to a k8s cluster

pkg/bindings/kube/types_play_options.go

@@ -302,3 +302,18 @@ func (o *PlayOptions) GetForce() bool {
 	}
 	return *o.Force
 }
+
+// WithPublishPorts set field PublishPorts to given value
+func (o *PlayOptions) WithPublishPorts(value []string) *PlayOptions {
+	o.PublishPorts = value
+	return o
+}
+
+// GetPublishPorts returns value of field PublishPorts
+func (o *PlayOptions) GetPublishPorts() []string {
+	if o.PublishPorts == nil {
+		var z []string
+		return z
+	}
+	return o.PublishPorts
+}

pkg/domain/entities/play.go

@@ -62,6 +62,8 @@ type PlayKubeOptions struct {
 	IsRemote bool
 	// Force - remove volumes on --down
 	Force bool
+	// PublishPorts - configure how to expose ports configured inside the K8S YAML file
+	PublishPorts []string
 }
 
 // PlayKubePod represents a single pod and associated containers created by play kube

pkg/domain/infra/abi/play.go

@@ -465,6 +465,14 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 	}
 	*ipIndex++
 
+	if len(options.PublishPorts) > 0 {
+		publishPorts, err := specgenutil.CreatePortBindings(options.PublishPorts)
+		if err != nil {
+			return nil, nil, err
+		}
+		mergePublishPorts(&podOpt, publishPorts)
+	}
+
 	p := specgen.NewPodSpecGenerator()
 	if err != nil {
 		return nil, nil, err
@@ -1001,6 +1009,38 @@ func (ic *ContainerEngine) playKubePVC(ctx context.Context, pvcYAML *v1.Persiste
 	return &report, nil
 }
 
+func mergePublishPorts(p *entities.PodCreateOptions, publishPortsOption []nettypes.PortMapping) {
+	for _, publishPortSpec := range p.Net.PublishPorts {
+		if !portAlreadyPublished(publishPortSpec, publishPortsOption) {
+			publishPortsOption = append(publishPortsOption, publishPortSpec)
+		}
+	}
+	p.Net.PublishPorts = publishPortsOption
+}
+
+func portAlreadyPublished(port nettypes.PortMapping, publishedPorts []nettypes.PortMapping) bool {
+	for _, publishedPort := range publishedPorts {
+		if port.ContainerPort >= publishedPort.ContainerPort &&
+			port.ContainerPort < publishedPort.ContainerPort+publishedPort.Range &&
+			isSamePortProtocol(port.Protocol, publishedPort.Protocol) {
+			return true
+		}
+	}
+	return false
+}
+
+func isSamePortProtocol(a, b string) bool {
+	if len(a) == 0 {
+		a = string(v1.ProtocolTCP)
+	}
+	if len(b) == 0 {
+		b = string(v1.ProtocolTCP)
+	}
+
+	ret := strings.EqualFold(a, b)
+	return ret
+}
+
 func (ic *ContainerEngine) importVolume(ctx context.Context, vol *libpod.Volume, tarFile *os.File) error {
 	volumeConfig, err := vol.Config()
 	if err != nil {

pkg/domain/infra/tunnel/kube.go

@@ -72,6 +72,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, opts en
 	if start := opts.Start; start != types.OptionalBoolUndefined {
 		options.WithStart(start == types.OptionalBoolTrue)
 	}
+	options.WithPublishPorts(opts.PublishPorts)
 	return play.KubeWithBody(ic.ClientCtx, body, options)
 }
 

test/e2e/play_kube_test.go

@@ -1,12 +1,15 @@
 package integration
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net"
+	"net/http"
 	"net/url"
 	"os"
 	"os/user"
@@ -850,6 +853,94 @@ spec:
 {{ end }}
 `
 
+var publishPortsPodWithoutPorts = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+spec:
+  containers:
+  - name: nginx
+    image: quay.io/libpod/alpine_nginx:latest
+`
+
+var publishPortsPodWithContainerPort = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+spec:
+  containers:
+  - name: nginx
+    image: quay.io/libpod/alpine_nginx:latest
+    ports:
+    - containerPort: 80
+`
+
+var publishPortsPodWithContainerHostPort = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+spec:
+  containers:
+  - name: nginx
+    image: quay.io/libpod/alpine_nginx:latest
+    ports:
+    - containerPort: 80
+      hostPort: 19001
+`
+
+var publishPortsEchoWithHostPortUDP = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: network-echo
+spec:
+  containers:
+  - name: udp-echo
+    image: quay.io/libpod/busybox:latest
+    command:
+    - "/bin/sh"
+    - "-c"
+    - "nc -ulk -p 19008 -e /bin/cat"
+    ports:
+    - containerPort: 19008
+      hostPort: 19009
+      protocol: udp
+  - name: tcp-echo
+    image: quay.io/libpod/busybox:latest
+    command:
+    - "/bin/sh"
+    - "-c"
+    - "nc -lk -p 19008 -e /bin/cat"
+`
+
+var publishPortsEchoWithHostPortTCP = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: network-echo
+spec:
+  containers:
+  - name: udp-echo
+    image: quay.io/libpod/busybox:latest
+    command:
+    - "/bin/sh"
+    - "-c"
+    - "nc -ulk -p 19008 -e /bin/cat"
+  - name: tcp-echo
+    image: quay.io/libpod/busybox:latest
+    command:
+    - "/bin/sh"
+    - "-c"
+    - "nc -lk -p 19008 -e /bin/cat"
+    ports:
+    - containerPort: 19008
+      hostPort: 19011
+      protocol: tcp
+`
+
 var (
 	defaultCtrName        = "testCtr"
 	defaultCtrCmd         = []string{"top"}
@@ -1569,6 +1660,108 @@ func testPodWithSecret(podmanTest *PodmanTestIntegration, podYamlString, fileNam
 	Expect(podRm).Should(Exit(0))
 }
 
+func testHTTPServer(port string, shouldErr bool, expectedResponse string) {
+	address := url.URL{
+		Scheme: "http",
+		Host:   net.JoinHostPort("localhost", port),
+	}
+
+	interval := 250 * time.Millisecond
+	var err error
+	var resp *http.Response
+	for i := 0; i < 6; i++ {
+		resp, err = http.Get(address.String())
+		if err != nil && shouldErr {
+			Expect(err.Error()).To(ContainSubstring(expectedResponse))
+			return
+		}
+		if err == nil {
+			defer resp.Body.Close()
+			break
+		}
+		time.Sleep(interval)
+		interval *= 2
+	}
+	Expect(err).To(BeNil())
+
+	body, err := io.ReadAll(resp.Body)
+	Expect(err).To(BeNil())
+	Expect(string(body)).Should(Equal(expectedResponse))
+}
+
+func testEchoServer(connection io.ReadWriter) {
+	stringToSend := "hello world"
+	var err error
+	var bytesSent int
+	interval := 250 * time.Millisecond
+	for i := 0; i < 6; i++ {
+		bytesSent, err = fmt.Fprint(connection, stringToSend)
+		if err == nil {
+			break
+		}
+		time.Sleep(interval)
+		interval *= 2
+	}
+	Expect(err).To(BeNil())
+	Expect(bytesSent).To(Equal(len(stringToSend)))
+
+	stringReceived := make([]byte, bytesSent)
+	var bytesRead int
+	interval = 250 * time.Millisecond
+	for i := 0; i < 6; i++ {
+		bytesRead, err = bufio.NewReader(connection).Read(stringReceived)
+		if err == nil {
+			break
+		}
+		time.Sleep(interval)
+		interval *= 2
+	}
+	Expect(err).To(BeNil())
+	Expect(bytesRead).To(Equal(bytesSent))
+
+	Expect(stringToSend).To(Equal(string(stringReceived)))
+}
+
+func testEchoServerUDP(address string) {
+	udpServer, err := net.ResolveUDPAddr("udp", address)
+	Expect(err).To(BeNil())
+
+	interval := 250 * time.Millisecond
+	var conn *net.UDPConn
+	for i := 0; i < 6; i++ {
+		conn, err = net.DialUDP("udp", nil, udpServer)
+		if err == nil {
+			break
+		}
+		time.Sleep(interval)
+		interval *= 2
+	}
+	Expect(err).To(BeNil())
+	defer conn.Close()
+
+	testEchoServer(conn)
+}
+
+func testEchoServerTCP(address string) {
+	tcpServer, err := net.ResolveTCPAddr("tcp", address)
+	Expect(err).To(BeNil())
+
+	interval := 250 * time.Millisecond
+	var conn *net.TCPConn
+	for i := 0; i < 6; i++ {
+		conn, err = net.DialTCP("tcp", nil, tcpServer)
+		if err == nil {
+			break
+		}
+		time.Sleep(interval)
+		interval *= 2
+	}
+	Expect(err).To(BeNil())
+	defer conn.Close()
+
+	testEchoServer(conn)
+}
+
 var _ = Describe("Podman play kube", func() {
 	var (
 		tempdir    string
@@ -4677,4 +4870,112 @@ spec:
 		Expect(exec.OutputToString()).Should(ContainSubstring("BAR"))
 		// we want to check that we can mount a subpath but not replace the entire dir
 	})
+
+	It("podman play kube without Ports - curl should fail", func() {
+		err := writeYaml(publishPortsPodWithoutPorts, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(0))
+
+		curlTest := podmanTest.Podman([]string{"run", "--network", "host", NGINX_IMAGE, "curl", "-s", "localhost:19000"})
+		curlTest.WaitWithDefaultTimeout()
+		Expect(curlTest).Should(Exit(7))
+	})
+
+	It("podman play kube without Ports, publish in command line - curl should succeed", func() {
+		err := writeYaml(publishPortsPodWithoutPorts, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19002:80", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(0))
+
+		testHTTPServer("19002", false, "podman rulez")
+	})
+
+	It("podman play kube with privileged container ports - should fail", func() {
+		SkipIfNotRootless("rootlessport can expose privileged port 80, no point in checking for failure")
+		err := writeYaml(publishPortsPodWithContainerPort, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(125))
+		// The error message is printed only on local call
+		if !IsRemote() {
+			Expect(kube.OutputToString()).Should(ContainSubstring("rootlessport cannot expose privileged port 80"))
+		}
+	})
+
+	It("podman play kube with privileged containers ports and publish in command line - curl should succeed", func() {
+		err := writeYaml(publishPortsPodWithContainerPort, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19003:80", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(0))
+
+		testHTTPServer("19003", false, "podman rulez")
+	})
+
+	It("podman play kube with Host Ports - curl should succeed", func() {
+		err := writeYaml(publishPortsPodWithContainerHostPort, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19004:80", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(0))
+
+		testHTTPServer("19004", false, "podman rulez")
+	})
+
+	It("podman play kube with Host Ports and publish in command line - curl should succeed only on overriding port", func() {
+		err := writeYaml(publishPortsPodWithContainerHostPort, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19005:80", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(0))
+
+		testHTTPServer("19001", true, "connection refused")
+		testHTTPServer("19005", false, "podman rulez")
+	})
+
+	It("podman play kube multiple publish ports", func() {
+		err := writeYaml(publishPortsPodWithoutPorts, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19006:80", "--publish", "19007:80", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(0))
+
+		testHTTPServer("19006", false, "podman rulez")
+		testHTTPServer("19007", false, "podman rulez")
+	})
+
+	It("podman play kube override with tcp should keep udp from YAML file", func() {
+		err := writeYaml(publishPortsEchoWithHostPortUDP, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19010:19008/tcp", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(0))
+
+		testEchoServerUDP(":19009")
+		testEchoServerTCP(":19010")
+	})
+
+	It("podman play kube override with udp should keep tcp from YAML file", func() {
+		err := writeYaml(publishPortsEchoWithHostPortTCP, kubeYaml)
+		Expect(err).ToNot(HaveOccurred())
+
+		kube := podmanTest.Podman([]string{"play", "kube", "--publish", "19012:19008/udp", kubeYaml})
+		kube.WaitWithDefaultTimeout()
+		Expect(kube).Should(Exit(0))
+
+		testEchoServerUDP(":19012")
+		testEchoServerTCP(":19011")
+	})
 })