Support multiple networks

This is a refresh of Dan William's PR #974 with a rebase and proper
vendoring of ocicni and containernetworking/cni.  It adds the ability
to define multiple networks as so:

podman run --network=net1,net2,foobar ...

Signed-off-by: baude <bbaude@redhat.com>

Closes: #1082
Approved by: baude
This commit is contained in:
baude
2018-07-12 09:51:31 -05:00
committed by Atomic Bot
parent e615b7d671
commit 4f699db8da
27 changed files with 881 additions and 470 deletions

View File

@ -143,18 +143,11 @@ type containerState struct {
// ExecSessions contains active exec sessions for container
// Exec session ID is mapped to PID of exec process
ExecSessions map[string]*ExecSession `json:"execSessions,omitempty"`
// IPs contains IP addresses assigned to the container
// Only populated if we created a network namespace for the container,
// and the network namespace is currently active
IPs []*cnitypes.IPConfig `json:"ipAddresses,omitempty"`
// Interfaces contains interface information about the container
// Only populated if we created a network namespace for the container,
// and the network namespace is currently active
Interfaces []*cnitypes.Interface `json:"interfaces,omitempty"`
// Routes contains network routes present in the container
// Only populated if we created a network namespace for the container,
// and the network namespace is currently active
Routes []*types.Route `json:"routes,omitempty"`
// NetworkStatus contains the configuration results for all networks
// the pod is attached to. Only populated if we created a network
// namespace for the container, and the network namespace is currently
// active
NetworkStatus []*cnitypes.Result `json:"networkResults,omitempty"`
// BindMounts contains files that will be bind-mounted into the
// container when it is mounted.
// These include /etc/hosts and /etc/resolv.conf
@ -268,6 +261,8 @@ type ContainerConfig struct {
// Hosts to add in container
// Will be appended to host's host file
HostAdd []string `json:"hostsAdd,omitempty"`
// Network names (CNI) to add container to. Empty to use default network.
Networks []string `json:"networks,omitempty"`
// Image Config
@ -773,11 +768,13 @@ func (c *Container) IPs() ([]net.IPNet, error) {
return nil, errors.Wrapf(ErrInvalidArg, "container %s network namespace is not managed by libpod")
}
ips := make([]net.IPNet, 0, len(c.state.IPs))
ips := make([]net.IPNet, 0)
for _, ip := range c.state.IPs {
for _, r := range c.state.NetworkStatus {
for _, ip := range r.IPs {
ips = append(ips, ip.Address)
}
}
return ips, nil
}
@ -799,16 +796,17 @@ func (c *Container) Routes() ([]types.Route, error) {
return nil, errors.Wrapf(ErrInvalidArg, "container %s network namespace is not managed by libpod")
}
routes := make([]types.Route, 0, len(c.state.Routes))
routes := make([]types.Route, 0)
for _, route := range c.state.Routes {
for _, r := range c.state.NetworkStatus {
for _, route := range r.Routes {
newRoute := types.Route{
Dst: route.Dst,
GW: route.GW,
}
routes = append(routes, newRoute)
}
}
return routes, nil
}

View File

@ -321,9 +321,7 @@ func resetState(state *containerState) error {
state.Mounted = false
state.State = ContainerStateConfigured
state.ExecSessions = make(map[string]*ExecSession)
state.IPs = nil
state.Interfaces = nil
state.Routes = nil
state.NetworkStatus = nil
state.BindMounts = make(map[string]string)
return nil

View File

@ -75,9 +75,7 @@ func (c *Container) cleanupNetwork() error {
}
c.state.NetNS = nil
c.state.IPs = nil
c.state.Interfaces = nil
c.state.Routes = nil
c.state.NetworkStatus = nil
if c.valid {
return c.save()

View File

@ -5,7 +5,6 @@ package libpod
import (
"crypto/rand"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
@ -24,21 +23,22 @@ import (
)
// Get an OCICNI network config
func getPodNetwork(id, name, nsPath string, ports []ocicni.PortMapping) ocicni.PodNetwork {
func getPodNetwork(id, name, nsPath string, networks []string, ports []ocicni.PortMapping) ocicni.PodNetwork {
return ocicni.PodNetwork{
Name: name,
Namespace: name, // TODO is there something else we should put here? We don't know about Kube namespaces
ID: id,
NetNS: nsPath,
PortMappings: ports,
Networks: networks,
}
}
// Create and configure a new network namespace for a container
func (r *Runtime) configureNetNS(ctr *Container, ctrNS ns.NetNS) (err error) {
podNetwork := getPodNetwork(ctr.ID(), ctr.Name(), ctrNS.Path(), ctr.config.PortMappings)
podNetwork := getPodNetwork(ctr.ID(), ctr.Name(), ctrNS.Path(), ctr.config.Networks, ctr.config.PortMappings)
result, err := r.netPlugin.SetUpPod(podNetwork)
results, err := r.netPlugin.SetUpPod(podNetwork)
if err != nil {
return errors.Wrapf(err, "error configuring network namespace for container %s", ctr.ID())
}
@ -50,28 +50,28 @@ func (r *Runtime) configureNetNS(ctr *Container, ctrNS ns.NetNS) (err error) {
}
}()
logrus.Debugf("Response from CNI plugins: %v", result.String())
resultStruct, err := cnitypes.GetResult(result)
ctr.state.NetNS = ctrNS
ctr.state.NetworkStatus = make([]*cnitypes.Result, 0)
for idx, r := range results {
logrus.Debugf("[%d] CNI result: %v", idx, r.String())
resultCurrent, err := cnitypes.GetResult(r)
if err != nil {
return errors.Wrapf(err, "error parsing result from CNI plugins")
return errors.Wrapf(err, "error parsing CNI plugin result %q: %v", r.String(), err)
}
ctr.state.NetworkStatus = append(ctr.state.NetworkStatus, resultCurrent)
}
ctr.state.NetNS = ctrNS
ctr.state.IPs = resultStruct.IPs
ctr.state.Routes = resultStruct.Routes
ctr.state.Interfaces = resultStruct.Interfaces
for _, r := range ctr.state.NetworkStatus {
// We need to temporarily use iptables to allow the container
// to resolve DNS until this issue is fixed upstream.
// https://github.com/containernetworking/plugins/pull/75
if resultStruct.IPs != nil {
for _, ip := range resultStruct.IPs {
for _, ip := range r.IPs {
if ip.Address.IP.To4() != nil {
iptablesDNS("-I", ip.Address.IP.String())
}
}
}
return nil
}
@ -148,27 +148,6 @@ func joinNetNS(path string) (ns.NetNS, error) {
return ns, nil
}
// Get a container's IP address
func (r *Runtime) getContainerIP(ctr *Container) (net.IP, error) {
if ctr.state.NetNS == nil {
return nil, errors.Wrapf(ErrInvalidArg, "container %s has no network namespace, cannot get IP", ctr.ID())
}
podNetwork := getPodNetwork(ctr.ID(), ctr.Name(), ctr.state.NetNS.Path(), ctr.config.PortMappings)
ipStr, err := r.netPlugin.GetPodNetworkStatus(podNetwork)
if err != nil {
return nil, errors.Wrapf(err, "error retrieving network status of container %s", ctr.ID())
}
ip := net.ParseIP(ipStr)
if ip == nil {
return nil, errors.Wrapf(ErrInternal, "error parsing IP address %s for container %s", ipStr, ctr.ID())
}
return ip, nil
}
// Tear down a network namespace
func (r *Runtime) teardownNetNS(ctr *Container) error {
if ctr.state.NetNS == nil {
@ -180,15 +159,17 @@ func (r *Runtime) teardownNetNS(ctr *Container) error {
// on per IP address, we also need to try to remove the iptables rule
// on cleanup. Remove when https://github.com/containernetworking/plugins/pull/75
// is merged.
for _, ip := range ctr.state.IPs {
for _, r := range ctr.state.NetworkStatus {
for _, ip := range r.IPs {
if ip.Address.IP.To4() != nil {
iptablesDNS("-D", ip.Address.IP.String())
}
}
}
logrus.Debugf("Tearing down network namespace at %s for container %s", ctr.state.NetNS.Path(), ctr.ID())
podNetwork := getPodNetwork(ctr.ID(), ctr.Name(), ctr.state.NetNS.Path(), ctr.config.PortMappings)
podNetwork := getPodNetwork(ctr.ID(), ctr.Name(), ctr.state.NetNS.Path(), ctr.config.Networks, ctr.config.PortMappings)
// The network may have already been torn down, so don't fail here, just log
if err := r.netPlugin.TearDownPod(podNetwork); err != nil {
@ -232,9 +213,11 @@ func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) {
}
func (c *Container) getContainerNetworkInfo(data *inspect.ContainerInspectData) *inspect.ContainerInspectData {
if c.state.NetNS != nil {
if c.state.NetNS != nil && len(c.state.NetworkStatus) > 0 {
// Report network settings from the first pod network
result := c.state.NetworkStatus[0]
// Go through our IP addresses
for _, ctrIP := range c.state.IPs {
for _, ctrIP := range result.IPs {
ipWithMask := ctrIP.Address.String()
splitIP := strings.Split(ipWithMask, "/")
mask, _ := strconv.Atoi(splitIP[1])
@ -253,7 +236,7 @@ func (c *Container) getContainerNetworkInfo(data *inspect.ContainerInspectData)
data.NetworkSettings.SandboxKey = c.state.NetNS.Path()
// Set MAC address of interface linked with network namespace path
for _, i := range c.state.Interfaces {
for _, i := range result.Interfaces {
if i.Sandbox == data.NetworkSettings.SandboxKey {
data.NetworkSettings.MacAddress = i.Mac
}

View File

@ -733,7 +733,7 @@ func WithDependencyCtrs(ctrs []*Container) CtrCreateOption {
// namespace with a minimal configuration.
// An optional array of port mappings can be provided.
// Conflicts with WithNetNSFrom().
func WithNetNS(portMappings []ocicni.PortMapping, postConfigureNetNS bool) CtrCreateOption {
func WithNetNS(portMappings []ocicni.PortMapping, postConfigureNetNS bool, networks []string) CtrCreateOption {
return func(ctr *Container) error {
if ctr.valid {
return ErrCtrFinalized
@ -746,6 +746,7 @@ func WithNetNS(portMappings []ocicni.PortMapping, postConfigureNetNS bool) CtrCr
ctr.config.PostConfigureNetNS = postConfigureNetNS
ctr.config.CreateNetNS = true
ctr.config.PortMappings = portMappings
ctr.config.Networks = networks
return nil
}

View File

@ -134,6 +134,9 @@ type RuntimeConfig struct {
CNIPluginDir []string `toml:"cni_plugin_dir"`
// HooksDir Path to the directory containing hooks configuration files
HooksDir string `toml:"hooks_dir"`
// CNIDefaultNetwork is the network name of the default CNI network
// to attach pods to
CNIDefaultNetwork string `toml:"cni_default_network,omitempty"`
// HooksDirNotExistFatal switches between fatal errors and non-fatal warnings if the configured HooksDir does not exist.
HooksDirNotExistFatal bool `toml:"hooks_dir_not_exist_fatal"`
// DefaultMountsFile is the path to the default mounts file for testing purposes only
@ -442,7 +445,7 @@ func makeRuntime(runtime *Runtime) (err error) {
}
// Set up the CNI net plugin
netPlugin, err := ocicni.InitCNI(runtime.config.CNIConfigDir, runtime.config.CNIPluginDir...)
netPlugin, err := ocicni.InitCNI(runtime.config.CNIDefaultNetwork, runtime.config.CNIConfigDir, runtime.config.CNIPluginDir...)
if err != nil {
return errors.Wrapf(err, "error configuring CNI network plugin")
}

View File

@ -351,9 +351,20 @@ func (c *CreateConfig) GetContainerCreateOptions(runtime *libpod.Runtime) ([]lib
// does not have one
options = append(options, libpod.WithEntrypoint(c.Entrypoint))
networks := make([]string, 0)
userNetworks := c.NetMode.UserDefined()
if userNetworks != "" {
for _, netName := range strings.Split(userNetworks, ",") {
if netName == "" {
return nil, errors.Wrapf(err, "container networks %q invalid", networks)
}
networks = append(networks, netName)
}
}
if rootless.IsRootless() {
if !c.NetMode.IsHost() && !c.NetMode.IsNone() {
options = append(options, libpod.WithNetNS(portBindings, true))
options = append(options, libpod.WithNetNS(portBindings, true, networks))
}
} else if c.NetMode.IsContainer() {
connectedCtr, err := c.Runtime.LookupContainer(c.NetMode.ConnectedContainer())
@ -363,8 +374,7 @@ func (c *CreateConfig) GetContainerCreateOptions(runtime *libpod.Runtime) ([]lib
options = append(options, libpod.WithNetNSFrom(connectedCtr))
} else if !c.NetMode.IsHost() && !c.NetMode.IsNone() {
postConfigureNetNS := (len(c.IDMappings.UIDMap) > 0 || len(c.IDMappings.GIDMap) > 0) && !c.UsernsMode.IsHost()
options = append(options, libpod.WithNetNS([]ocicni.PortMapping{}, postConfigureNetNS))
options = append(options, libpod.WithNetNS(portBindings, postConfigureNetNS))
options = append(options, libpod.WithNetNS(portBindings, postConfigureNetNS, networks))
}
if c.PidMode.IsContainer() {

View File

@ -345,11 +345,12 @@ func addNetNS(config *CreateConfig, g *generate.Generator) error {
return nil
} else if netMode.IsContainer() {
logrus.Debug("Using container netmode")
} else {
return errors.Errorf("unknown network mode")
}
} else if netMode.IsUserDefined() {
logrus.Debug("Using user defined netmode")
return nil
}
return errors.Errorf("unknown network mode")
}
func addUTSNS(config *CreateConfig, g *generate.Generator) error {
utsMode := config.UtsMode

View File

@ -8,7 +8,7 @@ github.com/boltdb/bolt master
github.com/buger/goterm 2f8dfbc7dbbff5dd1d391ed91482c24df243b2d3
github.com/containerd/cgroups 77e628511d924b13a77cebdc73b757a47f6d751b
github.com/containerd/continuity master
github.com/containernetworking/cni v0.6.0
github.com/containernetworking/cni v0.7.0-alpha1
github.com/containernetworking/plugins 1fb94a4222eafc6f948eacdca9c9f2158b427e53
github.com/containers/image 5144ced37a1b21b63c6ef605e56811e29a687528
github.com/containers/storage 90d0a58ffabb324042e074d33d1f6fbf3185c789

View File

@ -1,4 +1,5 @@
[![Build Status](https://travis-ci.org/containernetworking/cni.svg?branch=master)](https://travis-ci.org/containernetworking/cni)
[![Linux Build Status](https://travis-ci.org/containernetworking/cni.svg?branch=master)](https://travis-ci.org/containernetworking/cni)
[![Windows Build Status](https://ci.appveyor.com/api/projects/status/wtrkou8oow7x533e/branch/master?svg=true)](https://ci.appveyor.com/project/cni-bot/cni/branch/master)
[![Coverage Status](https://coveralls.io/repos/github/containernetworking/cni/badge.svg?branch=master)](https://coveralls.io/github/containernetworking/cni?branch=master)
[![Slack Status](https://cryptic-tundra-43194.herokuapp.com/badge.svg)](https://cryptic-tundra-43194.herokuapp.com/)
@ -8,7 +9,9 @@
# Community Sync Meeting
There is a community sync meeting for users and developers every 1-2 months. The next meeting will help on a Google Hangout and the link is in the [agenda](https://docs.google.com/document/d/10ECyT2mBGewsJUcmYmS8QNo1AcNgy2ZIe2xS7lShYhE/edit?usp=sharing) (Notes from previous meeting are also in this doc). The next meeting will be held on *Wednesday, June 21th* at *3:00pm UTC* [Add to Calendar]https://www.worldtimebuddy.com/?qm=1&lid=100,5,2643743,5391959&h=100&date=2017-6-21&sln=15-16).
There is a community sync meeting for users and developers every 1-2 months. The next meeting will help on a Google Hangout and the link is in the [agenda](https://docs.google.com/document/d/10ECyT2mBGewsJUcmYmS8QNo1AcNgy2ZIe2xS7lShYhE/edit?usp=sharing) (Notes from previous meeting are also in this doc).
The next meeting will be held on *Wednesday, October 4th* at *3:00pm UTC / 11:00am EDT / 8:00am PDT* [Add to Calendar](https://www.worldtimebuddy.com/?qm=1&lid=100,5,2643743,5391959&h=100&date=2017-10-04&sln=15-16).
---
@ -35,11 +38,11 @@ To avoid duplication, we think it is prudent to define a common interface betwee
## Who is using CNI?
### Container runtimes
- [rkt - container engine](https://coreos.com/blog/rkt-cni-networking.html)
- [Kurma - container runtime](http://kurma.io/)
- [Kubernetes - a system to simplify container operations](http://kubernetes.io/docs/admin/network-plugins/)
- [OpenShift - Kubernetes with additional enterprise features](https://github.com/openshift/origin/blob/master/docs/openshift_networking_requirements.md)
- [Cloud Foundry - a platform for cloud applications](https://github.com/cloudfoundry-incubator/cf-networking-release)
- [Mesos - a distributed systems kernel](https://github.com/apache/mesos/blob/master/docs/cni.md)
- [Apache Mesos - a distributed systems kernel](https://github.com/apache/mesos/blob/master/docs/cni.md)
- [Amazon ECS - a highly scalable, high performance container management service](https://aws.amazon.com/ecs/)
### 3rd party plugins
- [Project Calico - a layer 3 virtual network](https://github.com/projectcalico/calico-cni)
@ -54,6 +57,10 @@ To avoid duplication, we think it is prudent to define a common interface betwee
- [Nuage CNI - Nuage Networks SDN plugin for network policy kubernetes support ](https://github.com/nuagenetworks/nuage-cni)
- [Silk - a CNI plugin designed for Cloud Foundry](https://github.com/cloudfoundry-incubator/silk)
- [Linen - a CNI plugin designed for overlay networks with Open vSwitch and fit in SDN/OpenFlow network environment](https://github.com/John-Lin/linen-cni)
- [Vhostuser - a Dataplane network plugin - Supports OVS-DPDK & VPP](https://github.com/intel/vhost-user-net-plugin)
- [Amazon ECS CNI Plugins - a collection of CNI Plugins to configure containers with Amazon EC2 elastic network interfaces (ENIs)](https://github.com/aws/amazon-ecs-cni-plugins)
- [Bonding CNI - a Link aggregating plugin to address failover and high availability network](https://github.com/Intel-Corp/bond-cni)
- [ovn-kubernetes - an container network plugin built on Open vSwitch (OVS) and Open Virtual Networking (OVN) with support for both Linux and Windows](https://github.com/openvswitch/ovn-kubernetes)
The CNI team also maintains some [core plugins in a separate repository](https://github.com/containernetworking/plugins).

View File

@ -15,7 +15,11 @@
package libcni
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/containernetworking/cni/pkg/invoke"
@ -23,6 +27,14 @@ import (
"github.com/containernetworking/cni/pkg/version"
)
var (
CacheDir = "/var/lib/cni"
)
// A RuntimeConf holds the arguments to one invocation of a CNI plugin
// excepting the network configuration, with the nested exception that
// the `runtimeConfig` from the network configuration is included
// here.
type RuntimeConf struct {
ContainerID string
NetNS string
@ -34,6 +46,9 @@ type RuntimeConf struct {
// in this map which match the capabilities of the plugin are passed
// to the plugin
CapabilityArgs map[string]interface{}
// A cache directory in which to library data. Defaults to CacheDir
CacheDir string
}
type NetworkConfig struct {
@ -50,25 +65,38 @@ type NetworkConfigList struct {
type CNI interface {
AddNetworkList(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
GetNetworkList(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
DelNetworkList(net *NetworkConfigList, rt *RuntimeConf) error
AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
GetNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
DelNetwork(net *NetworkConfig, rt *RuntimeConf) error
}
type CNIConfig struct {
Path []string
exec invoke.Exec
}
// CNIConfig implements the CNI interface
var _ CNI = &CNIConfig{}
func buildOneConfig(list *NetworkConfigList, orig *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (*NetworkConfig, error) {
// NewCNIConfig returns a new CNIConfig object that will search for plugins
// in the given paths and use the given exec interface to run those plugins,
// or if the exec interface is not given, will use a default exec handler.
func NewCNIConfig(path []string, exec invoke.Exec) *CNIConfig {
return &CNIConfig{
Path: path,
exec: exec,
}
}
func buildOneConfig(name, cniVersion string, orig *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (*NetworkConfig, error) {
var err error
inject := map[string]interface{}{
"name": list.Name,
"cniVersion": list.CNIVersion,
"name": name,
"cniVersion": cniVersion,
}
// Add previous plugin result
if prevResult != nil {
@ -119,21 +147,37 @@ func injectRuntimeConfig(orig *NetworkConfig, rt *RuntimeConf) (*NetworkConfig,
return orig, nil
}
// AddNetworkList executes a sequence of plugins with the ADD command
func (c *CNIConfig) AddNetworkList(list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {
var prevResult types.Result
// ensure we have a usable exec if the CNIConfig was not given one
func (c *CNIConfig) ensureExec() invoke.Exec {
if c.exec == nil {
c.exec = &invoke.DefaultExec{
RawExec: &invoke.RawExec{Stderr: os.Stderr},
PluginDecoder: version.PluginDecoder{},
}
}
return c.exec
}
func (c *CNIConfig) addOrGetNetwork(command, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) {
c.ensureExec()
pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
if err != nil {
return nil, err
}
newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt)
if err != nil {
return nil, err
}
return invoke.ExecPluginWithResult(pluginPath, newConf.Bytes, c.args(command, rt), c.exec)
}
// Note that only GET requests should pass an initial prevResult
func (c *CNIConfig) addOrGetNetworkList(command string, prevResult types.Result, list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {
var err error
for _, net := range list.Plugins {
pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path)
if err != nil {
return nil, err
}
newConf, err := buildOneConfig(list, net, prevResult, rt)
if err != nil {
return nil, err
}
prevResult, err = invoke.ExecPluginWithResult(pluginPath, newConf.Bytes, c.args("ADD", rt))
prevResult, err = c.addOrGetNetwork(command, list.Name, list.CNIVersion, net, prevResult, rt)
if err != nil {
return nil, err
}
@ -142,68 +186,194 @@ func (c *CNIConfig) AddNetworkList(list *NetworkConfigList, rt *RuntimeConf) (ty
return prevResult, nil
}
func getResultCacheFilePath(netName string, rt *RuntimeConf) string {
cacheDir := rt.CacheDir
if cacheDir == "" {
cacheDir = CacheDir
}
return filepath.Join(cacheDir, "results", fmt.Sprintf("%s-%s", netName, rt.ContainerID))
}
func setCachedResult(result types.Result, netName string, rt *RuntimeConf) error {
data, err := json.Marshal(result)
if err != nil {
return err
}
fname := getResultCacheFilePath(netName, rt)
if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil {
return err
}
return ioutil.WriteFile(fname, data, 0600)
}
func delCachedResult(netName string, rt *RuntimeConf) error {
fname := getResultCacheFilePath(netName, rt)
return os.Remove(fname)
}
func getCachedResult(netName, cniVersion string, rt *RuntimeConf) (types.Result, error) {
fname := getResultCacheFilePath(netName, rt)
data, err := ioutil.ReadFile(fname)
if err != nil {
// Ignore read errors; the cached result may not exist on-disk
return nil, nil
}
// Read the version of the cached result
decoder := version.ConfigDecoder{}
resultCniVersion, err := decoder.Decode(data)
if err != nil {
return nil, err
}
// Ensure we can understand the result
result, err := version.NewResult(resultCniVersion, data)
if err != nil {
return nil, err
}
// Convert to the config version to ensure plugins get prevResult
// in the same version as the config. The cached result version
// should match the config version unless the config was changed
// while the container was running.
result, err = result.GetAsVersion(cniVersion)
if err != nil && resultCniVersion != cniVersion {
return nil, fmt.Errorf("failed to convert cached result version %q to config version %q: %v", resultCniVersion, cniVersion, err)
}
return result, err
}
// AddNetworkList executes a sequence of plugins with the ADD command
func (c *CNIConfig) AddNetworkList(list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {
result, err := c.addOrGetNetworkList("ADD", nil, list, rt)
if err != nil {
return nil, err
}
if err = setCachedResult(result, list.Name, rt); err != nil {
return nil, fmt.Errorf("failed to set network '%s' cached result: %v", list.Name, err)
}
return result, nil
}
// GetNetworkList executes a sequence of plugins with the GET command
func (c *CNIConfig) GetNetworkList(list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {
// GET was added in CNI spec version 0.4.0 and higher
if gtet, err := version.GreaterThanOrEqualTo(list.CNIVersion, "0.4.0"); err != nil {
return nil, err
} else if !gtet {
return nil, fmt.Errorf("configuration version %q does not support the GET command", list.CNIVersion)
}
cachedResult, err := getCachedResult(list.Name, list.CNIVersion, rt)
if err != nil {
return nil, fmt.Errorf("failed to get network '%s' cached result: %v", list.Name, err)
}
return c.addOrGetNetworkList("GET", cachedResult, list, rt)
}
func (c *CNIConfig) delNetwork(name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) error {
c.ensureExec()
pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
if err != nil {
return err
}
newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt)
if err != nil {
return err
}
return invoke.ExecPluginWithoutResult(pluginPath, newConf.Bytes, c.args("DEL", rt), c.exec)
}
// DelNetworkList executes a sequence of plugins with the DEL command
func (c *CNIConfig) DelNetworkList(list *NetworkConfigList, rt *RuntimeConf) error {
var cachedResult types.Result
// Cached result on DEL was added in CNI spec version 0.4.0 and higher
if gtet, err := version.GreaterThanOrEqualTo(list.CNIVersion, "0.4.0"); err != nil {
return err
} else if gtet {
cachedResult, err = getCachedResult(list.Name, list.CNIVersion, rt)
if err != nil {
return fmt.Errorf("failed to get network '%s' cached result: %v", list.Name, err)
}
}
for i := len(list.Plugins) - 1; i >= 0; i-- {
net := list.Plugins[i]
pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path)
if err != nil {
return err
}
newConf, err := buildOneConfig(list, net, nil, rt)
if err != nil {
return err
}
if err := invoke.ExecPluginWithoutResult(pluginPath, newConf.Bytes, c.args("DEL", rt)); err != nil {
if err := c.delNetwork(list.Name, list.CNIVersion, net, cachedResult, rt); err != nil {
return err
}
}
_ = delCachedResult(list.Name, rt)
return nil
}
// AddNetwork executes the plugin with the ADD command
func (c *CNIConfig) AddNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) {
pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path)
result, err := c.addOrGetNetwork("ADD", net.Network.Name, net.Network.CNIVersion, net, nil, rt)
if err != nil {
return nil, err
}
net, err = injectRuntimeConfig(net, rt)
if err != nil {
return nil, err
if err = setCachedResult(result, net.Network.Name, rt); err != nil {
return nil, fmt.Errorf("failed to set network '%s' cached result: %v", net.Network.Name, err)
}
return invoke.ExecPluginWithResult(pluginPath, net.Bytes, c.args("ADD", rt))
return result, nil
}
// GetNetwork executes the plugin with the GET command
func (c *CNIConfig) GetNetwork(net *NetworkConfig, rt *RuntimeConf) (types.Result, error) {
// GET was added in CNI spec version 0.4.0 and higher
if gtet, err := version.GreaterThanOrEqualTo(net.Network.CNIVersion, "0.4.0"); err != nil {
return nil, err
} else if !gtet {
return nil, fmt.Errorf("configuration version %q does not support the GET command", net.Network.CNIVersion)
}
cachedResult, err := getCachedResult(net.Network.Name, net.Network.CNIVersion, rt)
if err != nil {
return nil, fmt.Errorf("failed to get network '%s' cached result: %v", net.Network.Name, err)
}
return c.addOrGetNetwork("GET", net.Network.Name, net.Network.CNIVersion, net, cachedResult, rt)
}
// DelNetwork executes the plugin with the DEL command
func (c *CNIConfig) DelNetwork(net *NetworkConfig, rt *RuntimeConf) error {
pluginPath, err := invoke.FindInPath(net.Network.Type, c.Path)
if err != nil {
var cachedResult types.Result
// Cached result on DEL was added in CNI spec version 0.4.0 and higher
if gtet, err := version.GreaterThanOrEqualTo(net.Network.CNIVersion, "0.4.0"); err != nil {
return err
} else if gtet {
cachedResult, err = getCachedResult(net.Network.Name, net.Network.CNIVersion, rt)
if err != nil {
return fmt.Errorf("failed to get network '%s' cached result: %v", net.Network.Name, err)
}
}
net, err = injectRuntimeConfig(net, rt)
if err != nil {
if err := c.delNetwork(net.Network.Name, net.Network.CNIVersion, net, cachedResult, rt); err != nil {
return err
}
return invoke.ExecPluginWithoutResult(pluginPath, net.Bytes, c.args("DEL", rt))
_ = delCachedResult(net.Network.Name, rt)
return nil
}
// GetVersionInfo reports which versions of the CNI spec are supported by
// the given plugin.
func (c *CNIConfig) GetVersionInfo(pluginType string) (version.PluginInfo, error) {
pluginPath, err := invoke.FindInPath(pluginType, c.Path)
c.ensureExec()
pluginPath, err := c.exec.FindInPath(pluginType, c.Path)
if err != nil {
return nil, err
}
return invoke.GetVersionInfo(pluginPath)
return invoke.GetVersionInfo(pluginPath, c.exec)
}
// =====

View File

@ -45,6 +45,9 @@ func ConfFromBytes(bytes []byte) (*NetworkConfig, error) {
if err := json.Unmarshal(bytes, &conf.Network); err != nil {
return nil, fmt.Errorf("error parsing configuration: %s", err)
}
if conf.Network.Type == "" {
return nil, fmt.Errorf("error parsing configuration: missing 'type'")
}
return conf, nil
}

View File

@ -22,32 +22,54 @@ import (
"github.com/containernetworking/cni/pkg/types"
)
func DelegateAdd(delegatePlugin string, netconf []byte) (types.Result, error) {
if os.Getenv("CNI_COMMAND") != "ADD" {
return nil, fmt.Errorf("CNI_COMMAND is not ADD")
func delegateAddOrGet(command, delegatePlugin string, netconf []byte, exec Exec) (types.Result, error) {
if exec == nil {
exec = defaultExec
}
paths := filepath.SplitList(os.Getenv("CNI_PATH"))
pluginPath, err := FindInPath(delegatePlugin, paths)
pluginPath, err := exec.FindInPath(delegatePlugin, paths)
if err != nil {
return nil, err
}
return ExecPluginWithResult(pluginPath, netconf, ArgsFromEnv())
return ExecPluginWithResult(pluginPath, netconf, ArgsFromEnv(), exec)
}
// DelegateAdd calls the given delegate plugin with the CNI ADD action and
// JSON configuration
func DelegateAdd(delegatePlugin string, netconf []byte, exec Exec) (types.Result, error) {
if os.Getenv("CNI_COMMAND") != "ADD" {
return nil, fmt.Errorf("CNI_COMMAND is not ADD")
}
return delegateAddOrGet("ADD", delegatePlugin, netconf, exec)
}
// DelegateGet calls the given delegate plugin with the CNI GET action and
// JSON configuration
func DelegateGet(delegatePlugin string, netconf []byte, exec Exec) (types.Result, error) {
if os.Getenv("CNI_COMMAND") != "GET" {
return nil, fmt.Errorf("CNI_COMMAND is not GET")
}
return delegateAddOrGet("GET", delegatePlugin, netconf, exec)
}
// DelegateDel calls the given delegate plugin with the CNI DEL action and
// JSON configuration
func DelegateDel(delegatePlugin string, netconf []byte, exec Exec) error {
if exec == nil {
exec = defaultExec
}
func DelegateDel(delegatePlugin string, netconf []byte) error {
if os.Getenv("CNI_COMMAND") != "DEL" {
return fmt.Errorf("CNI_COMMAND is not DEL")
}
paths := filepath.SplitList(os.Getenv("CNI_PATH"))
pluginPath, err := FindInPath(delegatePlugin, paths)
pluginPath, err := exec.FindInPath(delegatePlugin, paths)
if err != nil {
return err
}
return ExecPluginWithoutResult(pluginPath, netconf, ArgsFromEnv())
return ExecPluginWithoutResult(pluginPath, netconf, ArgsFromEnv(), exec)
}

View File

@ -22,34 +22,62 @@ import (
"github.com/containernetworking/cni/pkg/version"
)
func ExecPluginWithResult(pluginPath string, netconf []byte, args CNIArgs) (types.Result, error) {
return defaultPluginExec.WithResult(pluginPath, netconf, args)
}
func ExecPluginWithoutResult(pluginPath string, netconf []byte, args CNIArgs) error {
return defaultPluginExec.WithoutResult(pluginPath, netconf, args)
}
func GetVersionInfo(pluginPath string) (version.PluginInfo, error) {
return defaultPluginExec.GetVersionInfo(pluginPath)
}
var defaultPluginExec = &PluginExec{
RawExec: &RawExec{Stderr: os.Stderr},
VersionDecoder: &version.PluginDecoder{},
}
type PluginExec struct {
RawExec interface {
// Exec is an interface encapsulates all operations that deal with finding
// and executing a CNI plugin. Tests may provide a fake implementation
// to avoid writing fake plugins to temporary directories during the test.
type Exec interface {
ExecPlugin(pluginPath string, stdinData []byte, environ []string) ([]byte, error)
}
VersionDecoder interface {
FindInPath(plugin string, paths []string) (string, error)
Decode(jsonBytes []byte) (version.PluginInfo, error)
}
// For example, a testcase could pass an instance of the following fakeExec
// object to ExecPluginWithResult() to verify the incoming stdin and environment
// and provide a tailored response:
//
//import (
// "encoding/json"
// "path"
// "strings"
//)
//
//type fakeExec struct {
// version.PluginDecoder
//}
//
//func (f *fakeExec) ExecPlugin(pluginPath string, stdinData []byte, environ []string) ([]byte, error) {
// net := &types.NetConf{}
// err := json.Unmarshal(stdinData, net)
// if err != nil {
// return nil, fmt.Errorf("failed to unmarshal configuration: %v", err)
// }
// pluginName := path.Base(pluginPath)
// if pluginName != net.Type {
// return nil, fmt.Errorf("plugin name %q did not match config type %q", pluginName, net.Type)
// }
// for _, e := range environ {
// // Check environment for forced failure request
// parts := strings.Split(e, "=")
// if len(parts) > 0 && parts[0] == "FAIL" {
// return nil, fmt.Errorf("failed to execute plugin %s", pluginName)
// }
// }
// return []byte("{\"CNIVersion\":\"0.4.0\"}"), nil
//}
//
//func (f *fakeExec) FindInPath(plugin string, paths []string) (string, error) {
// if len(paths) > 0 {
// return path.Join(paths[0], plugin), nil
// }
// return "", fmt.Errorf("failed to find plugin %s in paths %v", plugin, paths)
//}
func ExecPluginWithResult(pluginPath string, netconf []byte, args CNIArgs, exec Exec) (types.Result, error) {
if exec == nil {
exec = defaultExec
}
func (e *PluginExec) WithResult(pluginPath string, netconf []byte, args CNIArgs) (types.Result, error) {
stdoutBytes, err := e.RawExec.ExecPlugin(pluginPath, netconf, args.AsEnv())
stdoutBytes, err := exec.ExecPlugin(pluginPath, netconf, args.AsEnv())
if err != nil {
return nil, err
}
@ -64,8 +92,11 @@ func (e *PluginExec) WithResult(pluginPath string, netconf []byte, args CNIArgs)
return version.NewResult(confVersion, stdoutBytes)
}
func (e *PluginExec) WithoutResult(pluginPath string, netconf []byte, args CNIArgs) error {
_, err := e.RawExec.ExecPlugin(pluginPath, netconf, args.AsEnv())
func ExecPluginWithoutResult(pluginPath string, netconf []byte, args CNIArgs, exec Exec) error {
if exec == nil {
exec = defaultExec
}
_, err := exec.ExecPlugin(pluginPath, netconf, args.AsEnv())
return err
}
@ -73,7 +104,10 @@ func (e *PluginExec) WithoutResult(pluginPath string, netconf []byte, args CNIAr
// For recent-enough plugins, it uses the information returned by the VERSION
// command. For older plugins which do not recognize that command, it reports
// version 0.1.0
func (e *PluginExec) GetVersionInfo(pluginPath string) (version.PluginInfo, error) {
func GetVersionInfo(pluginPath string, exec Exec) (version.PluginInfo, error) {
if exec == nil {
exec = defaultExec
}
args := &Args{
Command: "VERSION",
@ -83,7 +117,7 @@ func (e *PluginExec) GetVersionInfo(pluginPath string) (version.PluginInfo, erro
Path: "dummy",
}
stdin := []byte(fmt.Sprintf(`{"cniVersion":%q}`, version.Current()))
stdoutBytes, err := e.RawExec.ExecPlugin(pluginPath, stdin, args.AsEnv())
stdoutBytes, err := exec.ExecPlugin(pluginPath, stdin, args.AsEnv())
if err != nil {
if err.Error() == "unknown CNI_COMMAND: VERSION" {
return version.PluginSupports("0.1.0"), nil
@ -91,5 +125,19 @@ func (e *PluginExec) GetVersionInfo(pluginPath string) (version.PluginInfo, erro
return nil, err
}
return e.VersionDecoder.Decode(stdoutBytes)
return exec.Decode(stdoutBytes)
}
// DefaultExec is an object that implements the Exec interface which looks
// for and executes plugins from disk.
type DefaultExec struct {
*RawExec
version.PluginDecoder
}
// DefaultExec implements the Exec interface
var _ Exec = &DefaultExec{}
var defaultExec = &DefaultExec{
RawExec: &RawExec{Stderr: os.Stderr},
}

View File

@ -57,3 +57,7 @@ func pluginErr(err error, output []byte) error {
return err
}
func (e *RawExec) FindInPath(plugin string, paths []string) (string, error) {
return FindInPath(plugin, paths)
}

View File

@ -24,9 +24,9 @@ import (
"github.com/containernetworking/cni/pkg/types/020"
)
const ImplementedSpecVersion string = "0.3.1"
const ImplementedSpecVersion string = "0.4.0"
var SupportedVersions = []string{"0.3.0", ImplementedSpecVersion}
var SupportedVersions = []string{"0.3.0", "0.3.1", ImplementedSpecVersion}
func NewResult(data []byte) (types.Result, error) {
result := &Result{}
@ -196,7 +196,7 @@ func (r *Result) Version() string {
func (r *Result) GetAsVersion(version string) (types.Result, error) {
switch version {
case "0.3.0", ImplementedSpecVersion:
case "0.3.0", "0.3.1", ImplementedSpecVersion:
r.CNIVersion = version
return r, nil
case types020.SupportedVersions[0], types020.SupportedVersions[1], types020.SupportedVersions[2]:

View File

@ -63,12 +63,14 @@ type NetConf struct {
Name string `json:"name,omitempty"`
Type string `json:"type,omitempty"`
Capabilities map[string]bool `json:"capabilities,omitempty"`
IPAM struct {
Type string `json:"type,omitempty"`
} `json:"ipam,omitempty"`
IPAM IPAM `json:"ipam,omitempty"`
DNS DNS `json:"dns"`
}
type IPAM struct {
Type string `json:"type,omitempty"`
}
// NetConfList describes an ordered list of networks.
type NetConfList struct {
CNIVersion string `json:"cniVersion,omitempty"`
@ -167,7 +169,7 @@ func (r *Route) UnmarshalJSON(data []byte) error {
return nil
}
func (r *Route) MarshalJSON() ([]byte, error) {
func (r Route) MarshalJSON() ([]byte, error) {
rt := route{
Dst: IPNet(r.Dst),
GW: r.GW,

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"fmt"
"io"
"strconv"
"strings"
)
// PluginInfo reports information about CNI versioning
@ -79,3 +81,60 @@ func (*PluginDecoder) Decode(jsonBytes []byte) (PluginInfo, error) {
}
return &info, nil
}
// ParseVersion parses a version string like "3.0.1" or "0.4.5" into major,
// minor, and micro numbers or returns an error
func ParseVersion(version string) (int, int, int, error) {
var major, minor, micro int
parts := strings.Split(version, ".")
if len(parts) == 0 || len(parts) >= 4 {
return -1, -1, -1, fmt.Errorf("invalid version %q: too many or too few parts", version)
}
major, err := strconv.Atoi(parts[0])
if err != nil {
return -1, -1, -1, fmt.Errorf("failed to convert major version part %q: %v", parts[0], err)
}
if len(parts) >= 2 {
minor, err = strconv.Atoi(parts[1])
if err != nil {
return -1, -1, -1, fmt.Errorf("failed to convert minor version part %q: %v", parts[1], err)
}
}
if len(parts) >= 3 {
micro, err = strconv.Atoi(parts[2])
if err != nil {
return -1, -1, -1, fmt.Errorf("failed to convert micro version part %q: %v", parts[2], err)
}
}
return major, minor, micro, nil
}
// GreaterThanOrEqualTo takes two string versions, parses them into major/minor/micro
// nubmers, and compares them to determine whether the first version is greater
// than or equal to the second
func GreaterThanOrEqualTo(version, otherVersion string) (bool, error) {
firstMajor, firstMinor, firstMicro, err := ParseVersion(version)
if err != nil {
return false, err
}
secondMajor, secondMinor, secondMicro, err := ParseVersion(otherVersion)
if err != nil {
return false, err
}
if firstMajor > secondMajor {
return true, nil
} else if firstMajor == secondMajor {
if firstMinor > secondMinor {
return true, nil
} else if firstMinor == secondMinor && firstMicro >= secondMicro {
return true, nil
}
}
return false, nil
}

View File

@ -24,7 +24,7 @@ import (
// Current reports the version of the CNI spec implemented by this library
func Current() string {
return "0.3.1"
return "0.4.0"
}
// Legacy PluginInfo describes a plugin that is backwards compatible with the
@ -35,7 +35,7 @@ func Current() string {
// Any future CNI spec versions which meet this definition should be added to
// this list.
var Legacy = PluginSupports("0.1.0", "0.2.0")
var All = PluginSupports("0.1.0", "0.2.0", "0.3.0", "0.3.1")
var All = PluginSupports("0.1.0", "0.2.0", "0.3.0", "0.3.1", "0.4.0")
var resultFactories = []struct {
supportedVersions []string

View File

@ -1,97 +0,0 @@
// +build ignore
// Simple tool to create an archive stream from an old and new directory
//
// By default it will stream the comparison of two temporary directories with junk files
package main
import (
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"github.com/containers/storage/pkg/archive"
"github.com/sirupsen/logrus"
)
var (
flDebug = flag.Bool("D", false, "debugging output")
flNewDir = flag.String("newdir", "", "")
flOldDir = flag.String("olddir", "", "")
log = logrus.New()
)
func main() {
flag.Usage = func() {
fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)")
fmt.Printf("%s [OPTIONS]\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
log.Out = os.Stderr
if (len(os.Getenv("DEBUG")) > 0) || *flDebug {
logrus.SetLevel(logrus.DebugLevel)
}
var newDir, oldDir string
if len(*flNewDir) == 0 {
var err error
newDir, err = ioutil.TempDir("", "storage-test-newDir")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(newDir)
if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil {
log.Fatal(err)
}
} else {
newDir = *flNewDir
}
if len(*flOldDir) == 0 {
oldDir, err := ioutil.TempDir("", "storage-test-oldDir")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(oldDir)
} else {
oldDir = *flOldDir
}
changes, err := archive.ChangesDirs(newDir, oldDir)
if err != nil {
log.Fatal(err)
}
a, err := archive.ExportChanges(newDir, changes)
if err != nil {
log.Fatal(err)
}
defer a.Close()
i, err := io.Copy(os.Stdout, a)
if err != nil && err != io.EOF {
log.Fatal(err)
}
fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i)
}
func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
fileData := []byte("fooo")
for n := 0; n < numberOfFiles; n++ {
fileName := fmt.Sprintf("file-%d", n)
if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil {
return 0, err
}
if makeLinks {
if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil {
return 0, err
}
}
}
totalSize := numberOfFiles * len(fileData)
return totalSize, nil
}

View File

@ -4,13 +4,16 @@ import (
"errors"
"fmt"
"os"
"os/exec"
"path"
"sort"
"strings"
"sync"
"github.com/containernetworking/cni/libcni"
cniinvoke "github.com/containernetworking/cni/pkg/invoke"
cnitypes "github.com/containernetworking/cni/pkg/types"
cnicurrent "github.com/containernetworking/cni/pkg/types/current"
cniversion "github.com/containernetworking/cni/pkg/version"
"github.com/fsnotify/fsnotify"
"github.com/sirupsen/logrus"
)
@ -19,14 +22,16 @@ type cniNetworkPlugin struct {
loNetwork *cniNetwork
sync.RWMutex
defaultNetwork *cniNetwork
defaultNetName string
networks map[string]*cniNetwork
nsenterPath string
pluginDir string
cniDirs []string
vendorCNIDirPrefix string
nsManager *nsManager
confDir string
binDirs []string
monitorNetDirChan chan struct{}
shutdownChan chan struct{}
watcher *fsnotify.Watcher
done *sync.WaitGroup
// The pod map provides synchronization for a given pod's network
// operations. Each pod's setup/teardown/status operations
@ -34,12 +39,17 @@ type cniNetworkPlugin struct {
// pods can proceed in parallel.
podsLock sync.Mutex
pods map[string]*podLock
// For testcases
exec cniinvoke.Exec
cacheDir string
}
type cniNetwork struct {
name string
filePath string
NetworkConfig *libcni.NetworkConfigList
CNIConfig libcni.CNI
CNIConfig *libcni.CNIConfig
}
var errMissingDefaultNetwork = errors.New("Missing CNI default network")
@ -99,110 +109,150 @@ func (plugin *cniNetworkPlugin) podUnlock(podNetwork PodNetwork) {
}
}
func (plugin *cniNetworkPlugin) monitorNetDir() {
func newWatcher(confDir string) (*fsnotify.Watcher, error) {
// Ensure plugin directory exists, because the following monitoring logic
// relies on that.
if err := os.MkdirAll(confDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create %q: %v", confDir, err)
}
watcher, err := fsnotify.NewWatcher()
if err != nil {
logrus.Errorf("could not create new watcher %v", err)
return
return nil, fmt.Errorf("could not create new watcher %v", err)
}
defer watcher.Close()
defer func() {
// Close watcher on error
if err != nil {
watcher.Close()
}
}()
if err = watcher.Add(plugin.pluginDir); err != nil {
logrus.Errorf("Failed to add watch on %q: %v", plugin.pluginDir, err)
return
if err = watcher.Add(confDir); err != nil {
return nil, fmt.Errorf("failed to add watch on %q: %v", confDir, err)
}
// Now that `watcher` is running and watching the `pluginDir`
// gather the initial configuration, before starting the
// goroutine which will actually process events. It has to be
// done in this order to avoid missing any updates which might
// otherwise occur between gathering the initial configuration
// and starting the watcher.
if err := plugin.syncNetworkConfig(); err != nil {
logrus.Infof("Initial CNI setting failed, continue monitoring: %v", err)
} else {
logrus.Infof("Initial CNI setting succeeded")
return watcher, nil
}
go func() {
func (plugin *cniNetworkPlugin) monitorConfDir(start *sync.WaitGroup) {
start.Done()
plugin.done.Add(1)
defer plugin.done.Done()
for {
select {
case event := <-watcher.Events:
logrus.Debugf("CNI monitoring event %v", event)
if event.Op&fsnotify.Create != fsnotify.Create &&
event.Op&fsnotify.Write != fsnotify.Write {
case event := <-plugin.watcher.Events:
logrus.Warningf("CNI monitoring event %v", event)
var defaultDeleted bool
createWrite := (event.Op&fsnotify.Create == fsnotify.Create ||
event.Op&fsnotify.Write == fsnotify.Write)
if event.Op&fsnotify.Remove == fsnotify.Remove {
// Care about the event if the default network
// was just deleted
defNet := plugin.getDefaultNetwork()
if defNet != nil && event.Name == defNet.filePath {
defaultDeleted = true
}
}
if !createWrite && !defaultDeleted {
continue
}
if err = plugin.syncNetworkConfig(); err == nil {
logrus.Infof("CNI asynchronous setting succeeded")
if err := plugin.syncNetworkConfig(); err != nil {
logrus.Errorf("CNI config loading failed, continue monitoring: %v", err)
continue
}
logrus.Errorf("CNI setting failed, continue monitoring: %v", err)
case err := <-watcher.Errors:
case err := <-plugin.watcher.Errors:
if err == nil {
continue
}
logrus.Errorf("CNI monitoring error %v", err)
close(plugin.monitorNetDirChan)
return
case <-plugin.shutdownChan:
return
}
}
}()
<-plugin.monitorNetDirChan
}
// InitCNI takes the plugin directory and CNI directories where the CNI config
// files should be searched for. If no valid CNI configs exist, network requests
// will fail until valid CNI config files are present in the config directory.
func InitCNI(pluginDir string, cniDirs ...string) (CNIPlugin, error) {
vendorCNIDirPrefix := ""
// InitCNI takes a binary directory in which to search for CNI plugins, and
// a configuration directory in which to search for CNI JSON config files.
// If no valid CNI configs exist, network requests will fail until valid CNI
// config files are present in the config directory.
// If defaultNetName is not empty, a CNI config with that network name will
// be used as the default CNI network, and container network operations will
// fail until that network config is present and valid.
func InitCNI(defaultNetName string, confDir string, binDirs ...string) (CNIPlugin, error) {
return initCNI(nil, "", defaultNetName, confDir, binDirs...)
}
// Internal function to allow faking out exec functions for testing
func initCNI(exec cniinvoke.Exec, cacheDir, defaultNetName string, confDir string, binDirs ...string) (CNIPlugin, error) {
if confDir == "" {
confDir = DefaultConfDir
}
if len(binDirs) == 0 {
binDirs = []string{DefaultBinDir}
}
plugin := &cniNetworkPlugin{
defaultNetwork: nil,
loNetwork: getLoNetwork(cniDirs, vendorCNIDirPrefix),
pluginDir: pluginDir,
cniDirs: cniDirs,
vendorCNIDirPrefix: vendorCNIDirPrefix,
monitorNetDirChan: make(chan struct{}),
defaultNetName: defaultNetName,
networks: make(map[string]*cniNetwork),
loNetwork: getLoNetwork(exec, binDirs),
confDir: confDir,
binDirs: binDirs,
shutdownChan: make(chan struct{}),
done: &sync.WaitGroup{},
pods: make(map[string]*podLock),
exec: exec,
cacheDir: cacheDir,
}
var err error
plugin.nsenterPath, err = exec.LookPath("nsenter")
if exec == nil {
exec = &cniinvoke.DefaultExec{
RawExec: &cniinvoke.RawExec{Stderr: os.Stderr},
PluginDecoder: cniversion.PluginDecoder{},
}
}
nsm, err := newNSManager()
if err != nil {
return nil, err
}
plugin.nsManager = nsm
plugin.syncNetworkConfig()
plugin.watcher, err = newWatcher(plugin.confDir)
if err != nil {
return nil, err
}
// Ensure plugin directory exists, because the following monitoring logic
// relies on that.
if err := os.MkdirAll(pluginDir, 0755); err != nil {
return nil, err
}
go plugin.monitorNetDir()
startWg := sync.WaitGroup{}
startWg.Add(1)
go plugin.monitorConfDir(&startWg)
startWg.Wait()
return plugin, nil
}
func getDefaultCNINetwork(pluginDir string, cniDirs []string, vendorCNIDirPrefix string) (*cniNetwork, error) {
if pluginDir == "" {
pluginDir = DefaultNetDir
}
if len(cniDirs) == 0 {
cniDirs = []string{DefaultCNIDir}
func (plugin *cniNetworkPlugin) Shutdown() error {
close(plugin.shutdownChan)
plugin.watcher.Close()
plugin.done.Wait()
return nil
}
files, err := libcni.ConfFiles(pluginDir, []string{".conf", ".conflist", ".json"})
switch {
case err != nil:
return nil, err
case len(files) == 0:
return nil, errMissingDefaultNetwork
func loadNetworks(exec cniinvoke.Exec, confDir string, binDirs []string) (map[string]*cniNetwork, string, error) {
files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"})
if err != nil {
return nil, "", err
}
networks := make(map[string]*cniNetwork)
defaultNetName := ""
sort.Strings(files)
for _, confFile := range files {
var confList *libcni.NetworkConfigList
@ -232,27 +282,28 @@ func getDefaultCNINetwork(pluginDir string, cniDirs []string, vendorCNIDirPrefix
logrus.Warningf("CNI config list %s has no networks, skipping", confFile)
continue
}
logrus.Infof("CNI network %s (type=%v) is used from %s", confList.Name, confList.Plugins[0].Network.Type, confFile)
// Search for vendor-specific plugins as well as default plugins in the CNI codebase.
vendorDir := vendorCNIDir(vendorCNIDirPrefix, confList.Plugins[0].Network.Type)
cninet := &libcni.CNIConfig{
Path: append(cniDirs, vendorDir),
}
network := &cniNetwork{name: confList.Name, NetworkConfig: confList, CNIConfig: cninet}
return network, nil
}
return nil, fmt.Errorf("No valid networks found in %s", pluginDir)
if confList.Name == "" {
confList.Name = path.Base(confFile)
}
func vendorCNIDir(prefix, pluginType string) string {
return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType)
logrus.Infof("Found CNI network %s (type=%v) at %s", confList.Name, confList.Plugins[0].Network.Type, confFile)
networks[confList.Name] = &cniNetwork{
name: confList.Name,
filePath: confFile,
NetworkConfig: confList,
CNIConfig: libcni.NewCNIConfig(binDirs, exec),
}
func getLoNetwork(cniDirs []string, vendorDirPrefix string) *cniNetwork {
if len(cniDirs) == 0 {
cniDirs = []string{DefaultCNIDir}
if defaultNetName == "" {
defaultNetName = confList.Name
}
}
return networks, defaultNetName, nil
}
func getLoNetwork(exec cniinvoke.Exec, binDirs []string) *cniNetwork {
loConfig, err := libcni.ConfListFromBytes([]byte(`{
"cniVersion": "0.2.0",
"name": "cni-loopback",
@ -265,45 +316,62 @@ func getLoNetwork(cniDirs []string, vendorDirPrefix string) *cniNetwork {
// catch this
panic(err)
}
vendorDir := vendorCNIDir(vendorDirPrefix, loConfig.Plugins[0].Network.Type)
cninet := &libcni.CNIConfig{
Path: append(cniDirs, vendorDir),
}
loNetwork := &cniNetwork{
name: "lo",
NetworkConfig: loConfig,
CNIConfig: cninet,
CNIConfig: libcni.NewCNIConfig(binDirs, exec),
}
return loNetwork
}
func (plugin *cniNetworkPlugin) syncNetworkConfig() error {
network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.cniDirs, plugin.vendorCNIDirPrefix)
networks, defaultNetName, err := loadNetworks(plugin.exec, plugin.confDir, plugin.binDirs)
if err != nil {
logrus.Errorf("error updating cni config: %s", err)
return err
}
plugin.setDefaultNetwork(network)
plugin.Lock()
defer plugin.Unlock()
if plugin.defaultNetName == "" {
plugin.defaultNetName = defaultNetName
}
plugin.networks = networks
return nil
}
func (plugin *cniNetworkPlugin) getDefaultNetwork() *cniNetwork {
func (plugin *cniNetworkPlugin) getNetwork(name string) (*cniNetwork, error) {
plugin.RLock()
defer plugin.RUnlock()
return plugin.defaultNetwork
net, ok := plugin.networks[name]
if !ok {
return nil, fmt.Errorf("CNI network %q not found", name)
}
return net, nil
}
func (plugin *cniNetworkPlugin) setDefaultNetwork(n *cniNetwork) {
plugin.Lock()
defer plugin.Unlock()
plugin.defaultNetwork = n
func (plugin *cniNetworkPlugin) getDefaultNetworkName() string {
plugin.RLock()
defer plugin.RUnlock()
return plugin.defaultNetName
}
func (plugin *cniNetworkPlugin) checkInitialized() error {
if plugin.getDefaultNetwork() == nil {
return errors.New("cni config uninitialized")
func (plugin *cniNetworkPlugin) getDefaultNetwork() *cniNetwork {
defaultNetName := plugin.getDefaultNetworkName()
if defaultNetName == "" {
return nil
}
network, _ := plugin.getNetwork(defaultNetName)
return network
}
// networksAvailable returns an error if the pod requests no networks and the
// plugin has no default network, and thus the plugin has no idea what network
// to attach the pod to.
func (plugin *cniNetworkPlugin) networksAvailable(podNetwork *PodNetwork) error {
if len(podNetwork.Networks) == 0 && plugin.getDefaultNetwork() == nil {
return errMissingDefaultNetwork
}
return nil
}
@ -312,59 +380,119 @@ func (plugin *cniNetworkPlugin) Name() string {
return CNIPluginName
}
func (plugin *cniNetworkPlugin) SetUpPod(podNetwork PodNetwork) (cnitypes.Result, error) {
if err := plugin.checkInitialized(); err != nil {
func (plugin *cniNetworkPlugin) forEachNetwork(podNetwork *PodNetwork, forEachFunc func(*cniNetwork, string, *PodNetwork) error) error {
networks := podNetwork.Networks
if len(networks) == 0 {
networks = append(networks, plugin.getDefaultNetworkName())
}
for i, netName := range networks {
// Interface names start at "eth0" and count up for each network
ifName := fmt.Sprintf("eth%d", i)
network, err := plugin.getNetwork(netName)
if err != nil {
logrus.Errorf(err.Error())
return err
}
if err := forEachFunc(network, ifName, podNetwork); err != nil {
return err
}
}
return nil
}
func (plugin *cniNetworkPlugin) SetUpPod(podNetwork PodNetwork) ([]cnitypes.Result, error) {
if err := plugin.networksAvailable(&podNetwork); err != nil {
return nil, err
}
plugin.podLock(podNetwork).Lock()
defer plugin.podUnlock(podNetwork)
_, err := plugin.loNetwork.addToNetwork(podNetwork)
_, err := plugin.loNetwork.addToNetwork(plugin.cacheDir, &podNetwork, "lo")
if err != nil {
logrus.Errorf("Error while adding to cni lo network: %s", err)
return nil, err
}
result, err := plugin.getDefaultNetwork().addToNetwork(podNetwork)
results := make([]cnitypes.Result, 0)
if err := plugin.forEachNetwork(&podNetwork, func(network *cniNetwork, ifName string, podNetwork *PodNetwork) error {
result, err := network.addToNetwork(plugin.cacheDir, podNetwork, ifName)
if err != nil {
logrus.Errorf("Error while adding to cni network: %s", err)
logrus.Errorf("Error while adding pod to CNI network %q: %s", network.name, err)
return err
}
results = append(results, result)
return nil
}); err != nil {
return nil, err
}
return result, err
return results, nil
}
func (plugin *cniNetworkPlugin) TearDownPod(podNetwork PodNetwork) error {
if err := plugin.checkInitialized(); err != nil {
if err := plugin.networksAvailable(&podNetwork); err != nil {
return err
}
plugin.podLock(podNetwork).Lock()
defer plugin.podUnlock(podNetwork)
return plugin.getDefaultNetwork().deleteFromNetwork(podNetwork)
return plugin.forEachNetwork(&podNetwork, func(network *cniNetwork, ifName string, podNetwork *PodNetwork) error {
if err := network.deleteFromNetwork(plugin.cacheDir, podNetwork, ifName); err != nil {
logrus.Errorf("Error while removing pod from CNI network %q: %s", network.name, err)
return err
}
return nil
})
}
// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin.
// Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls
func (plugin *cniNetworkPlugin) GetPodNetworkStatus(podNetwork PodNetwork) (string, error) {
// GetPodNetworkStatus returns IP addressing and interface details for all
// networks attached to the pod.
func (plugin *cniNetworkPlugin) GetPodNetworkStatus(podNetwork PodNetwork) ([]cnitypes.Result, error) {
plugin.podLock(podNetwork).Lock()
defer plugin.podUnlock(podNetwork)
ip, err := getContainerIP(plugin.nsenterPath, podNetwork.NetNS, DefaultInterfaceName, "-4")
results := make([]cnitypes.Result, 0)
if err := plugin.forEachNetwork(&podNetwork, func(network *cniNetwork, ifName string, podNetwork *PodNetwork) error {
version := "4"
ip, mac, err := getContainerDetails(plugin.nsManager, podNetwork.NetNS, ifName, "-4")
if err != nil {
ip, err = getContainerIP(plugin.nsenterPath, podNetwork.NetNS, DefaultInterfaceName, "-6")
ip, mac, err = getContainerDetails(plugin.nsManager, podNetwork.NetNS, ifName, "-6")
if err != nil {
return err
}
if err != nil {
return "", err
version = "6"
}
return ip.String(), nil
// Until CNI's GET request lands, construct the Result manually
results = append(results, &cnicurrent.Result{
CNIVersion: "0.3.1",
Interfaces: []*cnicurrent.Interface{
{
Name: ifName,
Mac: mac.String(),
Sandbox: podNetwork.NetNS,
},
},
IPs: []*cnicurrent.IPConfig{
{
Version: version,
Interface: cnicurrent.Int(0),
Address: *ip,
},
},
})
return nil
}); err != nil {
return nil, err
}
func (network *cniNetwork) addToNetwork(podNetwork PodNetwork) (cnitypes.Result, error) {
rt, err := buildCNIRuntimeConf(podNetwork)
return results, nil
}
func (network *cniNetwork) addToNetwork(cacheDir string, podNetwork *PodNetwork, ifName string) (cnitypes.Result, error) {
rt, err := buildCNIRuntimeConf(cacheDir, podNetwork, ifName)
if err != nil {
logrus.Errorf("Error adding network: %v", err)
return nil, err
@ -381,8 +509,8 @@ func (network *cniNetwork) addToNetwork(podNetwork PodNetwork) (cnitypes.Result,
return res, nil
}
func (network *cniNetwork) deleteFromNetwork(podNetwork PodNetwork) error {
rt, err := buildCNIRuntimeConf(podNetwork)
func (network *cniNetwork) deleteFromNetwork(cacheDir string, podNetwork *PodNetwork, ifName string) error {
rt, err := buildCNIRuntimeConf(cacheDir, podNetwork, ifName)
if err != nil {
logrus.Errorf("Error deleting network: %v", err)
return err
@ -398,13 +526,14 @@ func (network *cniNetwork) deleteFromNetwork(podNetwork PodNetwork) error {
return nil
}
func buildCNIRuntimeConf(podNetwork PodNetwork) (*libcni.RuntimeConf, error) {
func buildCNIRuntimeConf(cacheDir string, podNetwork *PodNetwork, ifName string) (*libcni.RuntimeConf, error) {
logrus.Infof("Got pod network %+v", podNetwork)
rt := &libcni.RuntimeConf{
ContainerID: podNetwork.ID,
NetNS: podNetwork.NetNS,
IfName: DefaultInterfaceName,
CacheDir: cacheDir,
IfName: ifName,
Args: [][2]string{
{"IgnoreUnknown", "1"},
{"K8S_POD_NAMESPACE", podNetwork.Namespace},
@ -424,5 +553,8 @@ func buildCNIRuntimeConf(podNetwork PodNetwork) (*libcni.RuntimeConf, error) {
}
func (plugin *cniNetworkPlugin) Status() error {
return plugin.checkInitialized()
if plugin.getDefaultNetwork() == nil {
return errMissingDefaultNetwork
}
return nil
}

View File

@ -36,6 +36,10 @@ type PodNetwork struct {
NetNS string
// PortMappings is the port mapping of the sandbox.
PortMappings []PortMapping
// Networks is a list of CNI network names to attach to the sandbox
// Leave this list empty to attach the default network to the sandbox
Networks []string
}
// CNIPlugin is the interface that needs to be implemented by a plugin
@ -47,14 +51,17 @@ type CNIPlugin interface {
// SetUpPod is the method called after the sandbox container of
// the pod has been created but before the other containers of the
// pod are launched.
SetUpPod(network PodNetwork) (types.Result, error)
SetUpPod(network PodNetwork) ([]types.Result, error)
// TearDownPod is the method called before a pod's sandbox container will be deleted
TearDownPod(network PodNetwork) error
// Status is the method called to obtain the ipv4 or ipv6 addresses of the pod sandbox
GetPodNetworkStatus(network PodNetwork) (string, error)
GetPodNetworkStatus(network PodNetwork) ([]types.Result, error)
// NetworkStatus returns error if the network plugin is in error state
Status() error
// Shutdown terminates all driver operations
Shutdown() error
}

View File

@ -3,10 +3,8 @@
package ocicni
const (
// DefaultNetDir is the place to look for CNI Network
DefaultNetDir = "/etc/cni/net.d"
// DefaultCNIDir is the place to look for cni config files
DefaultCNIDir = "/opt/cni/bin"
// VendorCNIDirTemplate is the template for looking up vendor specific cni config/executable files
VendorCNIDirTemplate = "%s/opt/%s/bin"
// DefaultConfDir is the default place to look for CNI Network
DefaultConfDir = "/etc/cni/net.d"
// DefaultBinDir is the default place to look for CNI config files
DefaultBinDir = "/opt/cni/bin"
)

View File

@ -3,10 +3,8 @@
package ocicni
const (
// DefaultNetDir is the place to look for CNI Network
DefaultNetDir = "C:\\cni\\etc\\net.d"
// DefaultCNIDir is the place to look for cni config files
DefaultCNIDir = "C:\\cni\\bin"
// VendorCNIDirTemplate is the template for looking up vendor specific cni config/executable files
VendorCNIDirTemplate = "C:\\cni\\%s\\opt\\%s\\bin" // XXX(vbatts) Not sure what to do here ...
// DefaultConfDir is the default place to look for CNI Network
DefaultConfDir = "C:\\cni\\etc\\net.d"
// DefaultBinDir is the default place to look for cni config files
DefaultBinDir = "C:\\cni\\bin"
)

View File

@ -1,32 +1,8 @@
package ocicni
import (
"fmt"
"net"
"os/exec"
"strings"
)
func getContainerIP(nsenterPath, netnsPath, interfaceName, addrType string) (net.IP, error) {
// Try to retrieve ip inside container network namespace
output, err := exec.Command(nsenterPath, fmt.Sprintf("--net=%s", netnsPath), "-F", "--",
"ip", "-o", addrType, "addr", "show", "dev", interfaceName, "scope", "global").CombinedOutput()
if err != nil {
return nil, fmt.Errorf("Unexpected command output %s with error: %v", output, err)
}
lines := strings.Split(string(output), "\n")
if len(lines) < 1 {
return nil, fmt.Errorf("Unexpected command output %s", output)
}
fields := strings.Fields(lines[0])
if len(fields) < 4 {
return nil, fmt.Errorf("Unexpected address output %s ", lines[0])
}
ip, _, err := net.ParseCIDR(fields[3])
if err != nil {
return nil, fmt.Errorf("CNI failed to parse ip from output %s due to %v", output, err)
}
return ip, nil
// newNSManager initializes a new namespace manager, which is a platform dependent struct.
func newNSManager() (*nsManager, error) {
nsm := &nsManager{}
err := nsm.init()
return nsm, err
}

View File

@ -0,0 +1,71 @@
// +build linux
package ocicni
import (
"fmt"
"net"
"os/exec"
"strings"
)
var defaultNamespaceEnterCommandName = "nsenter"
type nsManager struct {
nsenterPath string
}
func (nsm *nsManager) init() error {
var err error
nsm.nsenterPath, err = exec.LookPath(defaultNamespaceEnterCommandName)
return err
}
func getContainerDetails(nsm *nsManager, netnsPath, interfaceName, addrType string) (*net.IPNet, *net.HardwareAddr, error) {
// Try to retrieve ip inside container network namespace
output, err := exec.Command(nsm.nsenterPath, fmt.Sprintf("--net=%s", netnsPath), "-F", "--",
"ip", "-o", addrType, "addr", "show", "dev", interfaceName, "scope", "global").CombinedOutput()
if err != nil {
return nil, nil, fmt.Errorf("Unexpected command output %s with error: %v", output, err)
}
lines := strings.Split(string(output), "\n")
if len(lines) < 1 {
return nil, nil, fmt.Errorf("Unexpected command output %s", output)
}
fields := strings.Fields(lines[0])
if len(fields) < 4 {
return nil, nil, fmt.Errorf("Unexpected address output %s ", lines[0])
}
ip, ipNet, err := net.ParseCIDR(fields[3])
if err != nil {
return nil, nil, fmt.Errorf("CNI failed to parse ip from output %s due to %v", output, err)
}
if ip.To4() == nil {
ipNet.IP = ip
} else {
ipNet.IP = ip.To4()
}
// Try to retrieve MAC inside container network namespace
output, err = exec.Command(nsm.nsenterPath, fmt.Sprintf("--net=%s", netnsPath), "-F", "--",
"ip", "link", "show", "dev", interfaceName).CombinedOutput()
if err != nil {
return nil, nil, fmt.Errorf("unexpected 'ip link' command output %s with error: %v", output, err)
}
lines = strings.Split(string(output), "\n")
if len(lines) < 2 {
return nil, nil, fmt.Errorf("unexpected 'ip link' command output %s", output)
}
fields = strings.Fields(lines[1])
if len(fields) < 4 {
return nil, nil, fmt.Errorf("unexpected link output %s ", lines[0])
}
mac, err := net.ParseMAC(fields[1])
if err != nil {
return nil, nil, fmt.Errorf("failed to parse MAC from output %s due to %v", output, err)
}
return ipNet, &mac, nil
}

View File

@ -0,0 +1,19 @@
// +build !linux
package ocicni
import (
"fmt"
"net"
)
type nsManager struct {
}
func (nsm *nsManager) init() error {
return nil
}
func getContainerDetails(nsm *nsManager, netnsPath, interfaceName, addrType string) (*net.IPNet, *net.HardwareAddr, error) {
return nil, nil, fmt.Errorf("not supported yet")
}