Calico support

.circleci/config.yml

@@ -113,6 +113,43 @@ push_images: &push_images
           docker push "${img}"
         done
 
+e2e: &e2e
+  <<: *defaults
+  steps:
+  - run:
+      <<: *prereqs
+  - checkout
+  - setup_remote_docker
+  - run:
+      <<: *setup_env
+  - attach_workspace:
+      at: _output
+  - run:
+      name: Restore virtlet image
+      command: |
+        docker load -i _output/virtlet.tar
+  - run:
+      name: Start the demo
+      command: |
+        build/portforward.sh 8080&
+        if [[ ${CIRCLE_JOB} = e2e_calico ]]; then
+          export CNI_PLUGIN=calico
+          echo >&2 "*** Using Calico CNI"
+        fi
+        VIRTLET_DEMO_RELEASE=master \
+          SKIP_SNAPSHOT=1 \
+          NONINTERACTIVE=1 \
+          NO_VM_CONSOLE=1 \
+          INJECT_LOCAL_IMAGE=1 \
+          VIRTLET_DEMO_RELEASE=master \
+          BASE_LOCATION="$PWD" \
+          deploy/demo.sh
+  - run:
+      name: Run e2e tests
+      command: |
+        build/portforward.sh 8080&
+        _output/virtlet-e2e-tests -test.v
+
 version: 2
 jobs:
   prepare_build:
@@ -243,37 +280,10 @@ jobs:
           build/cmd.sh integration
 
   e2e:
-    <<: *defaults
-    steps:
-    - run:
-        <<: *prereqs
-    - checkout
-    - setup_remote_docker
-    - run:
-        <<: *setup_env
-    - attach_workspace:
-        at: _output
-    - run:
-        name: Restore virtlet image
-        command: |
-          docker load -i _output/virtlet.tar
-    - run:
-        name: Start the demo
-        command: |
-          build/portforward.sh 8080&
-          VIRTLET_DEMO_RELEASE=master \
-            SKIP_SNAPSHOT=1 \
-            NONINTERACTIVE=1 \
-            NO_VM_CONSOLE=1 \
-            INJECT_LOCAL_IMAGE=1 \
-            VIRTLET_DEMO_RELEASE=master \
-            BASE_LOCATION="$PWD" \
-            deploy/demo.sh
-    - run:
-        name: Run e2e tests
-        command: |
-          build/portforward.sh 8080&
-          _output/virtlet-e2e-tests -test.v
+    <<: *e2e
+
+  e2e_calico:
+    <<: *e2e
 
   push_branch:
     <<: *push_images
@@ -318,6 +328,13 @@ workflows:
           tags:
             only:
               - /^v[0-9].*/
+    - e2e_calico:
+        requires:
+        - build
+        filters:
+          tags:
+            only:
+              - /^v[0-9].*/
     - push_branch:
         requires:
         - build
@@ -329,6 +346,7 @@ workflows:
         requires:
         - test
         - e2e
+        - e2e_calico
         - integration
         filters:
           branches:

README.md

@@ -39,7 +39,7 @@ The demo will start a test cluster, deploy Virtlet on it and then boot a [CirrOS
 examples/vmssh.sh cirros@cirros-vm [command...]
 ```
 
-By default, CNI bridge plugin is used for cluster networking. It's also possible to override this with `flannel` or `weave` plugin, e.g.:
+By default, CNI bridge plugin is used for cluster networking. It's also possible to override this with `calico`, `flannel` or `weave` plugin, e.g.:
 ```
 CNI_PLUGIN=flannel ./demo.sh
 ```

deploy/virtlet-ds-dev.yaml

@@ -164,6 +164,12 @@ spec:
               name: virtlet-config
               key: loglevel
               optional: true
+        - name: VIRTLET_CALICO_SUBNET
+          valueFrom:
+            configMapKeyRef:
+              name: virtlet-config
+              key: calico-subnet
+              optional: true
         - name: IMAGE_REGEXP_TRANSLATION
           valueFrom:
             configMapKeyRef:

deploy/virtlet-ds.yaml

@@ -164,6 +164,12 @@ spec:
               name: virtlet-config
               key: loglevel
               optional: true
+        - name: VIRTLET_CALICO_SUBNET
+          valueFrom:
+            configMapKeyRef:
+              name: virtlet-config
+              key: calico-subnet
+              optional: true
         - name: IMAGE_REGEXP_TRANSLATION
           valueFrom:
             configMapKeyRef:

docs/networking.md

@@ -81,5 +81,14 @@ socket makes it possible to have all the network related code outside
 `vmwrapper` and have `vmwrapper` just `exec` the emulator instead of
 spawning it as a child process.
 
+[Calico](https://www.projectcalico.org/) CNI plugin needs special treatment
+as it tries to pass a routing configuration that cannot be passed
+over DHCP. For it to work Virtlet patches Calico-provided CNI result,
+replacing Calico's unreachable fake gateway with another fake gateway
+with an IP address acquired from Calico IPAM. A proper node subnet must
+be set for Calico-based virtlet installations. It's controlled by
+`calico-subnet` key Virtlet configmap (denoting the number of 1s in
+the netmask) and defaults to `24`.
+
 **NOTE:** Virtlet doesn't support `hostNetwork` pod setting because it
 cannot be impelemnted for VM in a meaningful way.

examples/ubuntu-vm.yaml

@@ -6,6 +6,14 @@ metadata:
     kubernetes.io/target-runtime: virtlet
     VirtletCloudInitUserData: |
       ssh_pwauth: True
+      users:
+      - name: testuser
+        gecos: User
+        primary-group: testuser
+        groups: users
+        lock_passwd: false
+        passwd: "$6$rounds=4096$wPs4Hz4tfs$a8ssMnlvH.3GX88yxXKF2cKMlVULsnydoOKgkuStTErTq2dzKZiIx9R/pPWWh5JLxzoZEx7lsSX5T2jW5WISi1"
+        sudo: ALL=(ALL) NOPASSWD:ALL
     VirtletSSHKeys: |
       ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
 spec:

pkg/cni/client.go

@@ -23,6 +23,8 @@ import (
 	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/golang/glog"
+
+	"github.com/Mirantis/virtlet/pkg/utils"
 )
 
 type Client struct {
@@ -32,6 +34,7 @@ type Client struct {
 
 func NewClient(pluginsDir, configsDir string) (*Client, error) {
 	configuration, err := ReadConfiguration(configsDir)
+	glog.V(3).Infof("CNI config: name: %q type: %q", configuration.Network.Name, configuration.Network.Type)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read CNI configuration: %v", err)
 	}
@@ -42,18 +45,37 @@ func NewClient(pluginsDir, configsDir string) (*Client, error) {
 	}, nil
 }
 
+func (c *Client) Type() string { return c.configuration.Network.Type }
+
 func (c *Client) cniRuntimeConf(podId, podName, podNs string) *libcni.RuntimeConf {
-	return &libcni.RuntimeConf{
+	r := &libcni.RuntimeConf{
 		ContainerID: podId,
 		NetNS:       PodNetNSPath(podId),
 		IfName:      "virtlet-eth0",
-		Args: [][2]string{
+	}
+	if podName != "" && podNs != "" {
+		r.Args = [][2]string{
 			{"IgnoreUnknown", "1"},
 			{"K8S_POD_NAMESPACE", podNs},
 			{"K8S_POD_NAME", podName},
 			{"K8S_POD_INFRA_CONTAINER_ID", podId},
-		},
+		}
+	}
+	return r
+}
+
+// GetDummyNetwork creates a dummy network using CNI plugin.
+// It's used for making a dummy gateway for Calico CNI plugin.
+func (c *Client) GetDummyNetwork() (*cnicurrent.Result, error) {
+	// TODO: virtlet pod restarts should not grab another address for
+	// the gateway. That's not a big problem usually though
+	// as the IPs are not returned to Calico so both old
+	// IPs on existing VMs and new ones should work.
+	podId := utils.NewUuid()
+	if err := CreateNetNS(podId); err != nil {
+		return nil, fmt.Errorf("couldn't create netns for fake pod %q: %v", podId, err)
 	}
+	return c.AddSandboxToNetwork(podId, "", "")
 }
 
 func (c *Client) AddSandboxToNetwork(podId, podName, podNs string) (*cnicurrent.Result, error) {

pkg/tapmanager/tapfdsource.go

@@ -18,7 +18,11 @@ package tapmanager
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
+	"net"
+	"os"
+	"strconv"
 	"sync"
 	"time"
 
@@ -33,6 +37,12 @@ import (
 	"github.com/Mirantis/virtlet/pkg/nettools"
 )
 
+const (
+	calicoNetType       = "calico"
+	calicoDefaultSubnet = 24
+	calicoSubnetVar     = "VIRTLET_CALICO_SUBNET"
+)
+
 // PodNetworkDesc contains the data that are required by TapFDSource
 // to set up a tap device for a VM
 type PodNetworkDesc struct {
@@ -70,8 +80,9 @@ type podNetwork struct {
 type TapFDSource struct {
 	sync.Mutex
 
-	cniClient *cni.Client
-	fdMap     map[string]*podNetwork
+	cniClient    *cni.Client
+	dummyGateway net.IP
+	fdMap        map[string]*podNetwork
 }
 
 var _ FDSource = &TapFDSource{}
@@ -84,10 +95,28 @@ func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) {
 		return nil, err
 	}
 
-	return &TapFDSource{
+	s := &TapFDSource{
 		cniClient: cniClient,
 		fdMap:     make(map[string]*podNetwork),
-	}, nil
+	}
+
+	// Calico needs special treatment here.
+	// We need to make network config DHCP-compatible by throwing away
+	// Calico's gateway and dev route and using a fake gateway instead.
+	// The fake gateway is just an IP address allocated by Calico IPAM,
+	// it's needed for proper ARP resppnses for VMs.
+	if cniClient.Type() == calicoNetType {
+		dummyResult, err := cniClient.GetDummyNetwork()
+		if err != nil {
+			return nil, err
+		}
+		if len(dummyResult.IPs) != 1 {
+			return nil, fmt.Errorf("expected 1 ip for the dummy network, but got %d", len(dummyResult.IPs))
+		}
+		s.dummyGateway = dummyResult.IPs[0].Address.IP
+	}
+
+	return s, nil
 }
 
 // GetFD implements GetFD method of FDSource interface
@@ -121,6 +150,27 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) {
 
 	netConfig := payload.CNIConfig
 
+	// Calico needs network config to be adjusted for DHCP compatibility
+	if s.dummyGateway != nil {
+		if len(netConfig.IPs) != 1 {
+			return 0, nil, errors.New("didn't expect more than one IP config")
+		}
+		if netConfig.IPs[0].Version != "4" {
+			return 0, nil, errors.New("IPv4 config was expected")
+		}
+		netConfig.IPs[0].Address.Mask = netmaskForCalico()
+		netConfig.IPs[0].Gateway = s.dummyGateway
+		netConfig.Routes = []*cnitypes.Route{
+			{
+				Dst: net.IPNet{
+					IP:   net.IP{0, 0, 0, 0},
+					Mask: net.IPMask{0, 0, 0, 0},
+				},
+				GW: s.dummyGateway,
+			},
+		}
+	}
+
 	netNSPath := cni.PodNetNSPath(pnd.PodId)
 	vmNS, err := ns.GetNS(netNSPath)
 	if err != nil {
@@ -143,8 +193,10 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) {
 
 		// NOTE: older CNI plugins don't include the hardware address
 		// in Result, but it's needed for Cloud-Init based
-		// network setup, so we add it here if it's missing
-		ensureCNIInterfaceHwAddress(netConfig, csn)
+		// network setup, so we add it here if it's missing.
+		// Also, some of the plugins may skip adding routes
+		// to the CNI result, so we must add them, too
+		fixCNIResult(netConfig, csn)
 
 		// TODO: now CNIConfig should always contain interface mac address, so there
 		// is no reason to pass it as separate field in dhcp.Config,
@@ -244,7 +296,7 @@ func (s *TapFDSource) GetInfo(key string) ([]byte, error) {
 	return pn.csn.HardwareAddr, nil
 }
 
-func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) {
+func fixCNIResult(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) {
 	// If there's no interface info in netConfig, we can assume that we're dealing
 	// with an old-style CNI plugin which only supports a single network interface
 	if len(netConfig.Interfaces) > 0 {
@@ -260,4 +312,21 @@ func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.Con
 	for _, IP := range netConfig.IPs {
 		IP.Interface = 0
 	}
+
+	if len(netConfig.Routes) == 0 {
+		netConfig.Routes = csn.Result.Routes
+	}
+}
+
+func netmaskForCalico() net.IPMask {
+	n := calicoDefaultSubnet
+	subnetStr := os.Getenv(calicoSubnetVar)
+	if subnetStr != "" {
+		n, err := strconv.Atoi(subnetStr)
+		if err != nil || n <= 0 || n > 30 {
+			glog.Warningf("bad calico subnet %q, using /%d", subnetStr, calicoDefaultSubnet)
+			n = calicoDefaultSubnet
+		}
+	}
+	return net.CIDRMask(n, 32)
 }