From e2850261e70197e28bb018e9fed058292b7cefe1 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Tue, 12 Sep 2017 16:04:59 +0300 Subject: [PATCH 1/9] Calico support PoC --- pkg/tapmanager/tapfdsource.go | 57 ++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/pkg/tapmanager/tapfdsource.go b/pkg/tapmanager/tapfdsource.go index d3563b165..2edfc78c3 100644 --- a/pkg/tapmanager/tapfdsource.go +++ b/pkg/tapmanager/tapfdsource.go @@ -18,7 +18,11 @@ package tapmanager import ( "encoding/json" + "errors" "fmt" + "io/ioutil" + "net" + "strings" "sync" "time" @@ -33,6 +37,10 @@ import ( "github.com/Mirantis/virtlet/pkg/nettools" ) +const ( + podIpFile = "/pod-ip" +) + // PodNetworkDesc contains the data that are required by TapFDSource // to set up a tap device for a VM type PodNetworkDesc struct { @@ -121,6 +129,50 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { netConfig := payload.CNIConfig + // serialize the original config before modifying it + // (in case if dummyGateway is set) + respData, err := json.Marshal(netConfig) + if err != nil { + return 0, nil, fmt.Errorf("error marshalling net config: %v", err) + } + + // TODO: do this upon startup + var dummyGateway net.IP + // TODO: detect Calico here + // TODO: get pod IP address from an env var set by cmd/virtlet/virtlet.go + // TODO: add descriptive comments why that's needed + // (incl. the fact that Virtlet runs in host network ns so it can't just grab + ipBytes, err := ioutil.ReadFile(podIpFile) + if err == nil { + ipStr := strings.TrimSpace(string(ipBytes)) + dummyGateway = net.ParseIP(ipStr) + if dummyGateway == nil { + return 0, nil, fmt.Errorf("error parsing pod IP: %q", ipStr) + } + } + + if dummyGateway != nil { + // TODO: better diagnostics + if len(netConfig.IPs) != 1 { + return 0, nil, errors.New("didn't expect more than one IP config") + } + if netConfig.IPs[0].Version != "4" { + return 0, nil, errors.New("IPv4 config was expected") + } + // TODO: calculate network mask based on the pod IP and the gateway IP + netConfig.IPs[0].Address.Mask = net.IPv4Mask(255, 255, 255, 0) + netConfig.IPs[0].Gateway = dummyGateway + netConfig.Routes = []*cnitypes.Route{ + { + Dst: net.IPNet{ + IP: net.IP{0, 0, 0, 0}, + Mask: net.IPMask{0, 0, 0, 0}, + }, + GW: dummyGateway, + }, + } + } + netNSPath := cni.PodNetNSPath(pnd.PodId) vmNS, err := ns.GetNS(netNSPath) if err != nil { @@ -176,11 +228,6 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { return 0, nil, err } - respData, err := json.Marshal(netConfig) - if err != nil { - return 0, nil, fmt.Errorf("error marshalling net config: %v", err) - } - s.Lock() defer s.Unlock() s.fdMap[key] = &podNetwork{ From 07e6cd052e74cc276df23e7bbaf8b362044b95f5 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 22 Sep 2017 19:56:46 +0300 Subject: [PATCH 2/9] Get dummy gateway IP from CNI --- pkg/cni/client.go | 25 +++++++++++++++--- pkg/tapmanager/tapfdsource.go | 48 +++++++++++++++-------------------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/pkg/cni/client.go b/pkg/cni/client.go index 6e7c9b794..5a701af8f 100644 --- a/pkg/cni/client.go +++ b/pkg/cni/client.go @@ -23,6 +23,8 @@ import ( cnicurrent "github.com/containernetworking/cni/pkg/types/current" "github.com/davecgh/go-spew/spew" "github.com/golang/glog" + + "github.com/Mirantis/virtlet/pkg/utils" ) type Client struct { @@ -43,17 +45,34 @@ func NewClient(pluginsDir, configsDir string) (*Client, error) { } func (c *Client) cniRuntimeConf(podId, podName, podNs string) *libcni.RuntimeConf { - return &libcni.RuntimeConf{ + r := &libcni.RuntimeConf{ ContainerID: podId, NetNS: PodNetNSPath(podId), IfName: "virtlet-eth0", - Args: [][2]string{ + } + if podName != "" && podNs != "" { + r.Args = [][2]string{ {"IgnoreUnknown", "1"}, {"K8S_POD_NAMESPACE", podNs}, {"K8S_POD_NAME", podName}, {"K8S_POD_INFRA_CONTAINER_ID", podId}, - }, + } + } + return r +} + +func (c *Client) GetDummyNetwork() (*cnicurrent.Result, error) { + // TODO: document the function + // TODO: the fake pod id should be generated in a way + // to make it unique for each node. Then a check should + // be made to see whether the corresponding netns is + // alive. If it is, it should be reused, otherwise + // a new one with the same id should be created. + podId := utils.NewUuid() + if err := CreateNetNS(podId); err != nil { + return nil, fmt.Errorf("couldn't create netns for fake pod %q: %v", podId, err) } + return c.AddSandboxToNetwork(podId, "", "") } func (c *Client) AddSandboxToNetwork(podId, podName, podNs string) (*cnicurrent.Result, error) { diff --git a/pkg/tapmanager/tapfdsource.go b/pkg/tapmanager/tapfdsource.go index 2edfc78c3..ce28f695c 100644 --- a/pkg/tapmanager/tapfdsource.go +++ b/pkg/tapmanager/tapfdsource.go @@ -20,9 +20,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "net" - "strings" "sync" "time" @@ -37,10 +35,6 @@ import ( "github.com/Mirantis/virtlet/pkg/nettools" ) -const ( - podIpFile = "/pod-ip" -) - // PodNetworkDesc contains the data that are required by TapFDSource // to set up a tap device for a VM type PodNetworkDesc struct { @@ -78,8 +72,9 @@ type podNetwork struct { type TapFDSource struct { sync.Mutex - cniClient *cni.Client - fdMap map[string]*podNetwork + cniClient *cni.Client + dummyGateway net.IP + fdMap map[string]*podNetwork } var _ FDSource = &TapFDSource{} @@ -92,10 +87,22 @@ func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) { return nil, err } - return &TapFDSource{ + s := &TapFDSource{ cniClient: cniClient, fdMap: make(map[string]*podNetwork), - }, nil + } + + // TODO: detect Calico here + dummyResult, err := cniClient.GetDummyNetwork() + if err != nil { + return nil, err + } + if len(dummyResult.IPs) != 1 { + return nil, fmt.Errorf("expected 1 ip for the dummy network, but got %d", len(dummyResult.IPs)) + } + s.dummyGateway = dummyResult.IPs[0].Address.IP + + return s, nil } // GetFD implements GetFD method of FDSource interface @@ -136,22 +143,7 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { return 0, nil, fmt.Errorf("error marshalling net config: %v", err) } - // TODO: do this upon startup - var dummyGateway net.IP - // TODO: detect Calico here - // TODO: get pod IP address from an env var set by cmd/virtlet/virtlet.go - // TODO: add descriptive comments why that's needed - // (incl. the fact that Virtlet runs in host network ns so it can't just grab - ipBytes, err := ioutil.ReadFile(podIpFile) - if err == nil { - ipStr := strings.TrimSpace(string(ipBytes)) - dummyGateway = net.ParseIP(ipStr) - if dummyGateway == nil { - return 0, nil, fmt.Errorf("error parsing pod IP: %q", ipStr) - } - } - - if dummyGateway != nil { + if s.dummyGateway != nil { // TODO: better diagnostics if len(netConfig.IPs) != 1 { return 0, nil, errors.New("didn't expect more than one IP config") @@ -161,14 +153,14 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { } // TODO: calculate network mask based on the pod IP and the gateway IP netConfig.IPs[0].Address.Mask = net.IPv4Mask(255, 255, 255, 0) - netConfig.IPs[0].Gateway = dummyGateway + netConfig.IPs[0].Gateway = s.dummyGateway netConfig.Routes = []*cnitypes.Route{ { Dst: net.IPNet{ IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{0, 0, 0, 0}, }, - GW: dummyGateway, + GW: s.dummyGateway, }, } } From 149d482c02c1158dd8ac7f6e69d03932982db9b4 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 22 Sep 2017 20:16:29 +0300 Subject: [PATCH 3/9] Add Calico detection --- pkg/cni/client.go | 3 +++ pkg/tapmanager/tapfdsource.go | 26 ++++++++++++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pkg/cni/client.go b/pkg/cni/client.go index 5a701af8f..8133099d8 100644 --- a/pkg/cni/client.go +++ b/pkg/cni/client.go @@ -34,6 +34,7 @@ type Client struct { func NewClient(pluginsDir, configsDir string) (*Client, error) { configuration, err := ReadConfiguration(configsDir) + glog.V(3).Infof("CNI config: name: %q type: %q", configuration.Network.Name, configuration.Network.Type) if err != nil { return nil, fmt.Errorf("failed to read CNI configuration: %v", err) } @@ -44,6 +45,8 @@ func NewClient(pluginsDir, configsDir string) (*Client, error) { }, nil } +func (c *Client) Type() string { return c.configuration.Network.Type } + func (c *Client) cniRuntimeConf(podId, podName, podNs string) *libcni.RuntimeConf { r := &libcni.RuntimeConf{ ContainerID: podId, diff --git a/pkg/tapmanager/tapfdsource.go b/pkg/tapmanager/tapfdsource.go index ce28f695c..39d01fbf0 100644 --- a/pkg/tapmanager/tapfdsource.go +++ b/pkg/tapmanager/tapfdsource.go @@ -35,6 +35,10 @@ import ( "github.com/Mirantis/virtlet/pkg/nettools" ) +const ( + calicoNetType = "calico" +) + // PodNetworkDesc contains the data that are required by TapFDSource // to set up a tap device for a VM type PodNetworkDesc struct { @@ -92,15 +96,20 @@ func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) { fdMap: make(map[string]*podNetwork), } - // TODO: detect Calico here - dummyResult, err := cniClient.GetDummyNetwork() - if err != nil { - return nil, err - } - if len(dummyResult.IPs) != 1 { - return nil, fmt.Errorf("expected 1 ip for the dummy network, but got %d", len(dummyResult.IPs)) + // Calico needs special treatment here. + // We need to make network config DHCP-compatible by throwing away + // Calico's gateway and dev route and using a fake gateway instead. + // TODO: add better explanation here + if cniClient.Type() == calicoNetType { + dummyResult, err := cniClient.GetDummyNetwork() + if err != nil { + return nil, err + } + if len(dummyResult.IPs) != 1 { + return nil, fmt.Errorf("expected 1 ip for the dummy network, but got %d", len(dummyResult.IPs)) + } + s.dummyGateway = dummyResult.IPs[0].Address.IP } - s.dummyGateway = dummyResult.IPs[0].Address.IP return s, nil } @@ -143,6 +152,7 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { return 0, nil, fmt.Errorf("error marshalling net config: %v", err) } + // Calico needs network config to be adjusted for DHCP compatibility if s.dummyGateway != nil { // TODO: better diagnostics if len(netConfig.IPs) != 1 { From 303bcab4a0352edc0f250dd01d67b0414c7c3b53 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 22 Sep 2017 21:28:49 +0300 Subject: [PATCH 4/9] Add Calico e2e job --- .circleci/config.yml | 80 +++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 74ddc0df3..461413230 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,6 +113,43 @@ push_images: &push_images docker push "${img}" done +e2e: &e2e + <<: *defaults + steps: + - run: + <<: *prereqs + - checkout + - setup_remote_docker + - run: + <<: *setup_env + - attach_workspace: + at: _output + - run: + name: Restore virtlet image + command: | + docker load -i _output/virtlet.tar + - run: + name: Start the demo + command: | + build/portforward.sh 8080& + if [[ ${CIRCLE_JOB} = e2e_calico ]]; then + export CNI_PLUGIN=calico + echo >&2 "*** Using Calico CNI" + fi + VIRTLET_DEMO_RELEASE=master \ + SKIP_SNAPSHOT=1 \ + NONINTERACTIVE=1 \ + NO_VM_CONSOLE=1 \ + INJECT_LOCAL_IMAGE=1 \ + VIRTLET_DEMO_RELEASE=master \ + BASE_LOCATION="$PWD" \ + deploy/demo.sh + - run: + name: Run e2e tests + command: | + build/portforward.sh 8080& + _output/virtlet-e2e-tests -test.v + version: 2 jobs: prepare_build: @@ -243,37 +280,10 @@ jobs: build/cmd.sh integration e2e: - <<: *defaults - steps: - - run: - <<: *prereqs - - checkout - - setup_remote_docker - - run: - <<: *setup_env - - attach_workspace: - at: _output - - run: - name: Restore virtlet image - command: | - docker load -i _output/virtlet.tar - - run: - name: Start the demo - command: | - build/portforward.sh 8080& - VIRTLET_DEMO_RELEASE=master \ - SKIP_SNAPSHOT=1 \ - NONINTERACTIVE=1 \ - NO_VM_CONSOLE=1 \ - INJECT_LOCAL_IMAGE=1 \ - VIRTLET_DEMO_RELEASE=master \ - BASE_LOCATION="$PWD" \ - deploy/demo.sh - - run: - name: Run e2e tests - command: | - build/portforward.sh 8080& - _output/virtlet-e2e-tests -test.v + <<: *e2e + + e2e_calico: + <<: *e2e push_branch: <<: *push_images @@ -318,6 +328,13 @@ workflows: tags: only: - /^v[0-9].*/ + - e2e_calico: + requires: + - build + filters: + tags: + only: + - /^v[0-9].*/ - push_branch: requires: - build @@ -329,6 +346,7 @@ workflows: requires: - test - e2e + - e2e_calico - integration filters: branches: From 1ecb42c7921665aa00f709d02cdc6bf27152b44b Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Wed, 27 Sep 2017 14:02:07 +0300 Subject: [PATCH 5/9] Fix logging e2e tests Make them less time-dependent. Pinging gateway during the boot of Cirros VM fails with Calico, causing these tests to fail. --- tests/e2e/basic_test.go | 48 ++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/tests/e2e/basic_test.go b/tests/e2e/basic_test.go index 9e5e14176..3330704a0 100644 --- a/tests/e2e/basic_test.go +++ b/tests/e2e/basic_test.go @@ -133,25 +133,43 @@ var _ = Describe("Basic cirros tests", func() { }) It("Should contain login string in VM log", func() { - out := do(framework.ExecSimple(nodeExecutor, "cat", - fmt.Sprintf("/var/log/virtlet/vms/%s/%s", sandboxID, filename))).(string) - Expect(strings.Count(out, - "login as 'cirros' user. default password: 'cubswin:)'. use 'sudo' for root.", - )).To(Equal(1)) + Eventually(func() error { + out, err := framework.ExecSimple(nodeExecutor, "cat", + fmt.Sprintf("/var/log/virtlet/vms/%s/%s", sandboxID, filename)) + if err != nil { + return err + } + fmt.Printf("OUT:\n%s\n---\n", out) + n := strings.Count(out, "login as 'cirros' user. default password: 'cubswin:)'. use 'sudo' for root.") + if n != 1 { + return fmt.Errorf("expected login prompt to appear exactly once in the log, but got %d occurences", n) + } + return nil + }, 60*5, 5) }) It("Should contain login string in pod log and each line of that log must be a valid JSON", func() { - out := do(framework.ExecSimple(nodeExecutor, "cat", - fmt.Sprintf("/var/log/pods/%s/%s", sandboxID, filename))).(string) - found := 0 - for _, line := range strings.Split(out, "\n") { - var entry map[string]string - Expect(json.Unmarshal([]byte(line), &entry)).To(Succeed()) - if strings.HasPrefix(entry["log"], "login as 'cirros' user. default password") { - found++ + Eventually(func() error { + out, err := framework.ExecSimple(nodeExecutor, "cat", + fmt.Sprintf("/var/log/pods/%s/%s", sandboxID, filename)) + if err != nil { + return err } - } - Expect(found).To(Equal(1)) + found := 0 + for _, line := range strings.Split(out, "\n") { + var entry map[string]string + if err := json.Unmarshal([]byte(line), &entry); err != nil { + return fmt.Errorf("error unmarshalling json: %v", err) + } + if strings.HasPrefix(entry["log"], "login as 'cirros' user. default password") { + found++ + } + } + if found != 1 { + return fmt.Errorf("expected login prompt to appear exactly once in the log, but got %d occurences", found) + } + return nil + }) }) }) From 2f677c8dfe5af7f76a39d979c3349a8ab86a453a Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Thu, 28 Sep 2017 03:23:13 +0300 Subject: [PATCH 6/9] Calculate proper netmask for Calico --- pkg/cni/client.go | 12 +++---- pkg/tapmanager/tapfdsource.go | 28 ++++++++++++--- pkg/tapmanager/tapfdsource_test.go | 55 ++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 pkg/tapmanager/tapfdsource_test.go diff --git a/pkg/cni/client.go b/pkg/cni/client.go index 8133099d8..bf863f5d5 100644 --- a/pkg/cni/client.go +++ b/pkg/cni/client.go @@ -64,13 +64,13 @@ func (c *Client) cniRuntimeConf(podId, podName, podNs string) *libcni.RuntimeCon return r } +// GetDummyNetwork creates a dummy network using CNI plugin. +// It's used for making a dummy gateway for Calico CNI plugin. func (c *Client) GetDummyNetwork() (*cnicurrent.Result, error) { - // TODO: document the function - // TODO: the fake pod id should be generated in a way - // to make it unique for each node. Then a check should - // be made to see whether the corresponding netns is - // alive. If it is, it should be reused, otherwise - // a new one with the same id should be created. + // TODO: virtlet pod restarts should not grab another address for + // the gateway. That's not a big problem usually though + // as the IPs are not returned to Calico so both old + // IPs on existing VMs and new ones should work. podId := utils.NewUuid() if err := CreateNetNS(podId); err != nil { return nil, fmt.Errorf("couldn't create netns for fake pod %q: %v", podId, err) diff --git a/pkg/tapmanager/tapfdsource.go b/pkg/tapmanager/tapfdsource.go index 39d01fbf0..78f41675e 100644 --- a/pkg/tapmanager/tapfdsource.go +++ b/pkg/tapmanager/tapfdsource.go @@ -99,7 +99,8 @@ func NewTapFDSource(cniPluginsDir, cniConfigsDir string) (*TapFDSource, error) { // Calico needs special treatment here. // We need to make network config DHCP-compatible by throwing away // Calico's gateway and dev route and using a fake gateway instead. - // TODO: add better explanation here + // The fake gateway is just an IP address allocated by Calico IPAM, + // it's needed for proper ARP resppnses for VMs. if cniClient.Type() == calicoNetType { dummyResult, err := cniClient.GetDummyNetwork() if err != nil { @@ -154,15 +155,16 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { // Calico needs network config to be adjusted for DHCP compatibility if s.dummyGateway != nil { - // TODO: better diagnostics if len(netConfig.IPs) != 1 { return 0, nil, errors.New("didn't expect more than one IP config") } if netConfig.IPs[0].Version != "4" { return 0, nil, errors.New("IPv4 config was expected") } - // TODO: calculate network mask based on the pod IP and the gateway IP - netConfig.IPs[0].Address.Mask = net.IPv4Mask(255, 255, 255, 0) + netConfig.IPs[0].Address.Mask, err = calcNetmaskForCalico(netConfig.IPs[0].Address.IP, s.dummyGateway) + if err != nil { + return 0, nil, err + } netConfig.IPs[0].Gateway = s.dummyGateway netConfig.Routes = []*cnitypes.Route{ { @@ -310,3 +312,21 @@ func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.Con IP.Interface = 0 } } + +func calcNetmaskForCalico(ipA, ipB net.IP) (net.IPMask, error) { + var a, b uint + for _, v := range ipA { + a = (a << 8) + uint(v) + } + for _, v := range ipB { + b = (b << 8) + uint(v) + } + for n := 30; n >= 0; n-- { + m := (uint(1) << uint(32-n)) - 1 + // avoid zero and broadcast addrs + if (a&^m) == (b&^m) && (a&m) != 0 && (b&m) != 0 && (a&m) != m && (b&m) != m { + return net.CIDRMask(n, 32), nil + } + } + return nil, errors.New("addresses too different") +} diff --git a/pkg/tapmanager/tapfdsource_test.go b/pkg/tapmanager/tapfdsource_test.go new file mode 100644 index 000000000..375389154 --- /dev/null +++ b/pkg/tapmanager/tapfdsource_test.go @@ -0,0 +1,55 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tapmanager + +import ( + "net" + "testing" +) + +// TODO: add test for TapFDSource itself + +func verifyNetmaskForCalico(t *testing.T, a, b net.IP, expectedOnes int) { + mask, err := calcNetmaskForCalico(a, b) + if err != nil { + t.Fatalf("%v - %v: can't calculate netmask: %v", err) + } + ones, bits := mask.Size() + if bits != 32 { + t.Errorf("%v - %v: bad mask bit count %d (expected 32)", a, b, bits) + } + if ones != expectedOnes { + t.Errorf("%v - %v: bad mask ones count %d (expected %d)", a, b, ones, expectedOnes) + } +} + +func TestCalcNetmaskForCalico(t *testing.T) { + for _, tc := range []struct { + a, b net.IP + ones int + }{ + // byte 3: 10000101, 10000110 + {net.IP{192, 168, 135, 133}, net.IP{192, 168, 135, 134}, 30}, + // byte 3: 10000100, 10000101 + {net.IP{192, 168, 135, 132}, net.IP{192, 168, 135, 133}, 29}, + // byte 2: 10101001, 10000111 + {net.IP{192, 168, 169, 129}, net.IP{192, 168, 135, 133}, 18}, + } { + verifyNetmaskForCalico(t, tc.a, tc.b, tc.ones) + verifyNetmaskForCalico(t, tc.b, tc.a, tc.ones) + } +} From 84c8c06ba0f576ced28aa23a1acef78e4b4e2e95 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Thu, 28 Sep 2017 03:29:09 +0300 Subject: [PATCH 7/9] Update docs concerning Calico --- README.md | 2 +- docs/networking.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 82945ccc8..f54f913f1 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The demo will start a test cluster, deploy Virtlet on it and then boot a [CirrOS examples/vmssh.sh cirros@cirros-vm [command...] ``` -By default, CNI bridge plugin is used for cluster networking. It's also possible to override this with `flannel` or `weave` plugin, e.g.: +By default, CNI bridge plugin is used for cluster networking. It's also possible to override this with `calico`, `flannel` or `weave` plugin, e.g.: ``` CNI_PLUGIN=flannel ./demo.sh ``` diff --git a/docs/networking.md b/docs/networking.md index ce0188ca5..5ce23bf14 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -81,5 +81,11 @@ socket makes it possible to have all the network related code outside `vmwrapper` and have `vmwrapper` just `exec` the emulator instead of spawning it as a child process. +[Calico](https://www.projectcalico.org/) CNI plugin needs special treatment +as it tries to pass a routing configuration that cannot be passed +over DHCP. For it to work Virtlet patches Calico-provided CNI result, +replacing Calico's unreachable fake gateway with another fake gateway +with an IP address acquired from Calico IPAM. + **NOTE:** Virtlet doesn't support `hostNetwork` pod setting because it cannot be impelemnted for VM in a meaningful way. From 71589983290be9d9dce231deb8bc3435abf1b73f Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 29 Sep 2017 12:13:38 +0300 Subject: [PATCH 8/9] Fix cloud-init netconfig generation for Calico --- pkg/tapmanager/tapfdsource.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/tapmanager/tapfdsource.go b/pkg/tapmanager/tapfdsource.go index 78f41675e..c05266000 100644 --- a/pkg/tapmanager/tapfdsource.go +++ b/pkg/tapmanager/tapfdsource.go @@ -146,13 +146,6 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { netConfig := payload.CNIConfig - // serialize the original config before modifying it - // (in case if dummyGateway is set) - respData, err := json.Marshal(netConfig) - if err != nil { - return 0, nil, fmt.Errorf("error marshalling net config: %v", err) - } - // Calico needs network config to be adjusted for DHCP compatibility if s.dummyGateway != nil { if len(netConfig.IPs) != 1 { @@ -161,10 +154,6 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { if netConfig.IPs[0].Version != "4" { return 0, nil, errors.New("IPv4 config was expected") } - netConfig.IPs[0].Address.Mask, err = calcNetmaskForCalico(netConfig.IPs[0].Address.IP, s.dummyGateway) - if err != nil { - return 0, nil, err - } netConfig.IPs[0].Gateway = s.dummyGateway netConfig.Routes = []*cnitypes.Route{ { @@ -199,8 +188,10 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { // NOTE: older CNI plugins don't include the hardware address // in Result, but it's needed for Cloud-Init based - // network setup, so we add it here if it's missing - ensureCNIInterfaceHwAddress(netConfig, csn) + // network setup, so we add it here if it's missing. + // Also, some of the plugins may skip adding routes + // to the CNI result, so we must add them, too + fixCNIResult(netConfig, csn) // TODO: now CNIConfig should always contain interface mac address, so there // is no reason to pass it as separate field in dhcp.Config, @@ -232,6 +223,11 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { return 0, nil, err } + respData, err := json.Marshal(netConfig) + if err != nil { + return 0, nil, fmt.Errorf("error marshalling net config: %v", err) + } + s.Lock() defer s.Unlock() s.fdMap[key] = &podNetwork{ @@ -295,7 +291,7 @@ func (s *TapFDSource) GetInfo(key string) ([]byte, error) { return pn.csn.HardwareAddr, nil } -func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) { +func fixCNIResult(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwork) { // If there's no interface info in netConfig, we can assume that we're dealing // with an old-style CNI plugin which only supports a single network interface if len(netConfig.Interfaces) > 0 { @@ -311,6 +307,10 @@ func ensureCNIInterfaceHwAddress(netConfig *cnicurrent.Result, csn *nettools.Con for _, IP := range netConfig.IPs { IP.Interface = 0 } + + if len(netConfig.Routes) == 0 { + netConfig.Routes = csn.Result.Routes + } } func calcNetmaskForCalico(ipA, ipB net.IP) (net.IPMask, error) { From d669ab841db4d9adc44345c16d961b4ec33727a2 Mon Sep 17 00:00:00 2001 From: Ivan Shvedunov Date: Fri, 29 Sep 2017 12:18:35 +0300 Subject: [PATCH 9/9] Use configurable netmask for Calico --- deploy/virtlet-ds-dev.yaml | 6 ++++ deploy/virtlet-ds.yaml | 6 ++++ docs/networking.md | 5 ++- examples/ubuntu-vm.yaml | 8 +++++ pkg/tapmanager/tapfdsource.go | 30 ++++++++-------- pkg/tapmanager/tapfdsource_test.go | 55 ------------------------------ 6 files changed, 39 insertions(+), 71 deletions(-) delete mode 100644 pkg/tapmanager/tapfdsource_test.go diff --git a/deploy/virtlet-ds-dev.yaml b/deploy/virtlet-ds-dev.yaml index 6c15f88f2..5f292780d 100644 --- a/deploy/virtlet-ds-dev.yaml +++ b/deploy/virtlet-ds-dev.yaml @@ -164,6 +164,12 @@ spec: name: virtlet-config key: loglevel optional: true + - name: VIRTLET_CALICO_SUBNET + valueFrom: + configMapKeyRef: + name: virtlet-config + key: calico-subnet + optional: true - name: IMAGE_REGEXP_TRANSLATION valueFrom: configMapKeyRef: diff --git a/deploy/virtlet-ds.yaml b/deploy/virtlet-ds.yaml index c4f5ef128..b62a231e9 100644 --- a/deploy/virtlet-ds.yaml +++ b/deploy/virtlet-ds.yaml @@ -164,6 +164,12 @@ spec: name: virtlet-config key: loglevel optional: true + - name: VIRTLET_CALICO_SUBNET + valueFrom: + configMapKeyRef: + name: virtlet-config + key: calico-subnet + optional: true - name: IMAGE_REGEXP_TRANSLATION valueFrom: configMapKeyRef: diff --git a/docs/networking.md b/docs/networking.md index 5ce23bf14..59d25a24c 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -85,7 +85,10 @@ spawning it as a child process. as it tries to pass a routing configuration that cannot be passed over DHCP. For it to work Virtlet patches Calico-provided CNI result, replacing Calico's unreachable fake gateway with another fake gateway -with an IP address acquired from Calico IPAM. +with an IP address acquired from Calico IPAM. A proper node subnet must +be set for Calico-based virtlet installations. It's controlled by +`calico-subnet` key Virtlet configmap (denoting the number of 1s in +the netmask) and defaults to `24`. **NOTE:** Virtlet doesn't support `hostNetwork` pod setting because it cannot be impelemnted for VM in a meaningful way. diff --git a/examples/ubuntu-vm.yaml b/examples/ubuntu-vm.yaml index 4495a673c..9acd1987d 100644 --- a/examples/ubuntu-vm.yaml +++ b/examples/ubuntu-vm.yaml @@ -6,6 +6,14 @@ metadata: kubernetes.io/target-runtime: virtlet VirtletCloudInitUserData: | ssh_pwauth: True + users: + - name: testuser + gecos: User + primary-group: testuser + groups: users + lock_passwd: false + passwd: "$6$rounds=4096$wPs4Hz4tfs$a8ssMnlvH.3GX88yxXKF2cKMlVULsnydoOKgkuStTErTq2dzKZiIx9R/pPWWh5JLxzoZEx7lsSX5T2jW5WISi1" + sudo: ALL=(ALL) NOPASSWD:ALL VirtletSSHKeys: | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost spec: diff --git a/pkg/tapmanager/tapfdsource.go b/pkg/tapmanager/tapfdsource.go index c05266000..34d592c9c 100644 --- a/pkg/tapmanager/tapfdsource.go +++ b/pkg/tapmanager/tapfdsource.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" "net" + "os" + "strconv" "sync" "time" @@ -36,7 +38,9 @@ import ( ) const ( - calicoNetType = "calico" + calicoNetType = "calico" + calicoDefaultSubnet = 24 + calicoSubnetVar = "VIRTLET_CALICO_SUBNET" ) // PodNetworkDesc contains the data that are required by TapFDSource @@ -154,6 +158,7 @@ func (s *TapFDSource) GetFD(key string, data []byte) (int, []byte, error) { if netConfig.IPs[0].Version != "4" { return 0, nil, errors.New("IPv4 config was expected") } + netConfig.IPs[0].Address.Mask = netmaskForCalico() netConfig.IPs[0].Gateway = s.dummyGateway netConfig.Routes = []*cnitypes.Route{ { @@ -313,20 +318,15 @@ func fixCNIResult(netConfig *cnicurrent.Result, csn *nettools.ContainerSideNetwo } } -func calcNetmaskForCalico(ipA, ipB net.IP) (net.IPMask, error) { - var a, b uint - for _, v := range ipA { - a = (a << 8) + uint(v) - } - for _, v := range ipB { - b = (b << 8) + uint(v) - } - for n := 30; n >= 0; n-- { - m := (uint(1) << uint(32-n)) - 1 - // avoid zero and broadcast addrs - if (a&^m) == (b&^m) && (a&m) != 0 && (b&m) != 0 && (a&m) != m && (b&m) != m { - return net.CIDRMask(n, 32), nil +func netmaskForCalico() net.IPMask { + n := calicoDefaultSubnet + subnetStr := os.Getenv(calicoSubnetVar) + if subnetStr != "" { + n, err := strconv.Atoi(subnetStr) + if err != nil || n <= 0 || n > 30 { + glog.Warningf("bad calico subnet %q, using /%d", subnetStr, calicoDefaultSubnet) + n = calicoDefaultSubnet } } - return nil, errors.New("addresses too different") + return net.CIDRMask(n, 32) } diff --git a/pkg/tapmanager/tapfdsource_test.go b/pkg/tapmanager/tapfdsource_test.go deleted file mode 100644 index 375389154..000000000 --- a/pkg/tapmanager/tapfdsource_test.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2017 Mirantis - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tapmanager - -import ( - "net" - "testing" -) - -// TODO: add test for TapFDSource itself - -func verifyNetmaskForCalico(t *testing.T, a, b net.IP, expectedOnes int) { - mask, err := calcNetmaskForCalico(a, b) - if err != nil { - t.Fatalf("%v - %v: can't calculate netmask: %v", err) - } - ones, bits := mask.Size() - if bits != 32 { - t.Errorf("%v - %v: bad mask bit count %d (expected 32)", a, b, bits) - } - if ones != expectedOnes { - t.Errorf("%v - %v: bad mask ones count %d (expected %d)", a, b, ones, expectedOnes) - } -} - -func TestCalcNetmaskForCalico(t *testing.T) { - for _, tc := range []struct { - a, b net.IP - ones int - }{ - // byte 3: 10000101, 10000110 - {net.IP{192, 168, 135, 133}, net.IP{192, 168, 135, 134}, 30}, - // byte 3: 10000100, 10000101 - {net.IP{192, 168, 135, 132}, net.IP{192, 168, 135, 133}, 29}, - // byte 2: 10101001, 10000111 - {net.IP{192, 168, 169, 129}, net.IP{192, 168, 135, 133}, 18}, - } { - verifyNetmaskForCalico(t, tc.a, tc.b, tc.ones) - verifyNetmaskForCalico(t, tc.b, tc.a, tc.ones) - } -}