Fix handling of node reboot while there were running VMs

.codeclimate.yml

@@ -2,7 +2,7 @@ version: "2"
 checks:
   argument-count:
     config:
-      threshold: 8
+      threshold: 9
   complex-logic:
     config:
       threshold: 8

pkg/libvirttools/gc.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 
 	"github.com/Mirantis/virtlet/pkg/blockdev"
+	"github.com/Mirantis/virtlet/pkg/metadata"
+	"github.com/Mirantis/virtlet/pkg/metadata/types"
 )
 
 const (
@@ -62,6 +64,11 @@ func (v *VirtualizationTool) retrieveListOfContainerIDs() ([]string, bool, []err
 
 	var allErrors []error
 	for _, sandbox := range sandboxes {
+		if err := v.checkSandboxNetNs(sandbox); err != nil {
+			allErrors = append(allErrors, err)
+			continue
+		}
+
 		containers, err := v.metadataStore.ListPodContainers(sandbox.GetID())
 		if err != nil {
 			allErrors = append(
@@ -82,6 +89,37 @@ func (v *VirtualizationTool) retrieveListOfContainerIDs() ([]string, bool, []err
 	return containerIDs, false, allErrors
 }
 
+func (v *VirtualizationTool) checkSandboxNetNs(sandbox metadata.PodSandboxMetadata) error {
+	sinfo, err := sandbox.Retrieve()
+	if err != nil {
+		return err
+	}
+
+	if !v.mountPointChecker.IsPathAnNs(sinfo.ContainerSideNetwork.NsPath) {
+		containers, err := v.metadataStore.ListPodContainers(sandbox.GetID())
+		if err != nil {
+			return err
+		}
+		for _, container := range containers {
+			// remove container from metadata store
+			if err := container.Save(func(*types.ContainerInfo) (*types.ContainerInfo, error) {
+				return nil, nil
+			}); err != nil {
+				return err
+			}
+		}
+
+		// remove sandbox from metadata store
+		if err := sandbox.Save(func(*types.PodSandboxInfo) (*types.PodSandboxInfo, error) {
+			return nil, nil
+		}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func inList(list []string, filter func(string) bool) bool {
 	for _, element := range list {
 		if filter(element) {

pkg/libvirttools/virtualization.go

@@ -217,15 +217,16 @@ type VirtualizationConfig struct {
 
 // VirtualizationTool provides methods to operate on libvirt.
 type VirtualizationTool struct {
-	domainConn    virt.DomainConnection
-	storageConn   virt.StorageConnection
-	imageManager  ImageManager
-	metadataStore metadata.Store
-	clock         clockwork.Clock
-	volumeSource  VMVolumeSource
-	config        VirtualizationConfig
-	mounter       utils.Mounter
-	commander     utils.Commander
+	domainConn        virt.DomainConnection
+	storageConn       virt.StorageConnection
+	imageManager      ImageManager
+	metadataStore     metadata.Store
+	clock             clockwork.Clock
+	volumeSource      VMVolumeSource
+	config            VirtualizationConfig
+	mounter           utils.Mounter
+	mountPointChecker utils.MountPointChecker
+	commander         utils.Commander
 }
 
 var _ volumeOwner = &VirtualizationTool{}
@@ -236,17 +237,19 @@ func NewVirtualizationTool(domainConn virt.DomainConnection,
 	storageConn virt.StorageConnection, imageManager ImageManager,
 	metadataStore metadata.Store, volumeSource VMVolumeSource,
 	config VirtualizationConfig, mounter utils.Mounter,
+	mountPointChecker utils.MountPointChecker,
 	commander utils.Commander) *VirtualizationTool {
 	return &VirtualizationTool{
-		domainConn:    domainConn,
-		storageConn:   storageConn,
-		imageManager:  imageManager,
-		metadataStore: metadataStore,
-		clock:         clockwork.NewRealClock(),
-		volumeSource:  volumeSource,
-		config:        config,
-		mounter:       mounter,
-		commander:     commander,
+		domainConn:        domainConn,
+		storageConn:       storageConn,
+		imageManager:      imageManager,
+		metadataStore:     metadataStore,
+		clock:             clockwork.NewRealClock(),
+		volumeSource:      volumeSource,
+		config:            config,
+		mounter:           mounter,
+		mountPointChecker: mountPointChecker,
+		commander:         commander,
 	}
 }
 

pkg/libvirttools/virtualization_test.go

@@ -96,7 +96,8 @@ func newContainerTester(t *testing.T, rec *testutils.TopLevelRecorder, cmds []fa
 	fakeCommander.ReplaceTempPath("__pods__", "/fakedev")
 	ct.virtTool = NewVirtualizationTool(
 		ct.domainConn, ct.storageConn, imageManager, ct.metadataStore,
-		GetDefaultVolumeSource(), virtConfig, utils.NullMounter, fakeCommander)
+		GetDefaultVolumeSource(), virtConfig, utils.NullMounter,
+		utils.FakeMountPointChecker, fakeCommander)
 	ct.virtTool.SetClock(ct.clock)
 
 	return ct

pkg/manager/manager.go

@@ -127,15 +127,20 @@ func (v *VirtletManager) Run() error {
 
 		err = s.Start()
 		if err != nil {
-			glog.Warningf("Could not start stream server: %s", err)
+			glog.Warningf("Could not start stream server: %v", err)
 
 		}
 		streamServer = s
 		virtConfig.StreamerSocketPath = streamerSocketPath
 	}
 
+	mpc, err := utils.NewMountPointChecker()
+	if err != nil {
+		return fmt.Errorf("couldn't create mountpoint checker: %v", err)
+	}
+
 	volSrc := libvirttools.GetDefaultVolumeSource()
-	v.virtTool = libvirttools.NewVirtualizationTool(conn, conn, v.imageStore, v.metadataStore, volSrc, virtConfig, utils.NewMounter(), utils.DefaultCommander)
+	v.virtTool = libvirttools.NewVirtualizationTool(conn, conn, v.imageStore, v.metadataStore, volSrc, virtConfig, utils.NewMounter(), mpc, utils.DefaultCommander)
 
 	runtimeService := NewVirtletRuntimeService(v.virtTool, v.metadataStore, v.fdManager, streamServer, v.imageStore, nil)
 	imageService := NewVirtletImageService(v.imageStore, translator, nil)

pkg/manager/runtime_test.go

@@ -238,7 +238,7 @@ func makeVirtletCRITester(t *testing.T) *virtletCRITester {
 		StreamerSocketPath: streamerSocketPath,
 	}
 	commander := fakeutils.NewCommander(rec, nil)
-	virtTool := libvirttools.NewVirtualizationTool(domainConn, storageConn, imageStore, metadataStore, libvirttools.GetDefaultVolumeSource(), virtConfig, utils.NullMounter, commander)
+	virtTool := libvirttools.NewVirtualizationTool(domainConn, storageConn, imageStore, metadataStore, libvirttools.GetDefaultVolumeSource(), virtConfig, utils.NullMounter, utils.FakeMountPointChecker, commander)
 	virtTool.SetClock(clock)
 	streamServer := newFakeStreamServer(rec.Child("streamServer"))
 	criHandler := &criHandler{

pkg/utils/mountinfo.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2018 Mirantis
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+type mountEntry struct {
+	Source string
+	Fs     string
+}
+
+// MountPointChecker is used to check if a directory entry is a mount point.
+// It returns its source info and filesystem type.
+type MountPointChecker interface {
+	// CheckMountPointInfo checks if entry is a mountpoint (second returned value will be true)
+	// and if so returns mountInfo for it. In other case it returns false as a second value.
+	CheckMountPointInfo(string) (mountEntry, bool)
+	// IsPathAnNs verifies if the path is a mountpoint with nsfs filesystem type
+	IsPathAnNs(string) bool
+}
+
+type mountPointChecker struct {
+	mountInfo map[string]mountEntry
+}
+
+var _ MountPointChecker = mountPointChecker{}
+
+// NewMountPointChecker returns a new instance of MountPointChecker
+func NewMountPointChecker() (MountPointChecker, error) {
+	file, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return mountPointChecker{}, err
+	}
+	defer file.Close()
+
+	mi := make(map[string]mountEntry)
+
+	reader := bufio.NewReader(file)
+LineReader:
+	for {
+		line, err := reader.ReadString('\n')
+		switch err {
+		case io.EOF:
+			break LineReader
+		case nil:
+			return mountPointChecker{}, err
+		}
+
+		// strip eol
+		line = strings.Trim(line, "\n")
+
+		// split and parse entries acording to section 3.5 in
+		// https://www.kernel.org/doc/Documentation/filesystems/proc.txt
+		// TODO: whitespaces and control chars in names are encoded as
+		// octal values (e.g. for "x x": "x\040x") what should be expanded
+		// in both mount point source and target
+		parts := strings.Split(line, " ")
+		mi[parts[4]] = mountEntry{Source: parts[9], Fs: parts[8]}
+	}
+	return mountPointChecker{mountInfo: mi}, nil
+}
+
+// CheckMountPointInfo implements CheckMountPointInfo method of MountPointChecker interface
+func (mpc mountPointChecker) CheckMountPointInfo(path string) (mountEntry, bool) {
+	entry, ok := mpc.mountInfo[path]
+	return entry, ok
+}
+
+// IsPathNs implements IsPathAnNs method of MountPointChecker interface
+func (mpc mountPointChecker) IsPathAnNs(path string) bool {
+	_, err := os.Stat(path)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			glog.Errorf("Cannot verify existence of %q: %v", path, err)
+		}
+		return false
+	}
+	realpath, err := filepath.EvalSymlinks(path)
+	if err != nil {
+		glog.Errorf("Cannot verify real path of %q: %v", path, err)
+		return false
+	}
+
+	entry, isMountPoint := mpc.CheckMountPointInfo(realpath)
+	if !isMountPoint {
+		return false
+	}
+	return entry.Fs == "nsfs"
+}
+
+type fakeMountPointChecker struct{}
+
+// FakeMountPointChecker is defined there for unittests
+var FakeMountPointChecker MountPointChecker = fakeMountPointChecker{}
+
+// CheckMountPointInfo is a fake implementation for MountPointChecker interface
+func (mpc fakeMountPointChecker) CheckMountPointInfo(path string) (mountEntry, bool) {
+	return mountEntry{}, false
+}
+
+// IsPathAnNs is a fake implementation for MountPointChecker interface
+func (mpc fakeMountPointChecker) IsPathAnNs(path string) bool {
+	return false
+}