Skip to content

Commit

Permalink
Remove sandboxes with missing netns during startup
Browse files Browse the repository at this point in the history
Closes #783
  • Loading branch information
jellonek committed Oct 31, 2018
1 parent fc6b062 commit 12c6922
Show file tree
Hide file tree
Showing 6 changed files with 193 additions and 22 deletions.
38 changes: 38 additions & 0 deletions pkg/libvirttools/gc.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"strings"

"github.com/Mirantis/virtlet/pkg/blockdev"
"github.com/Mirantis/virtlet/pkg/metadata"
"github.com/Mirantis/virtlet/pkg/metadata/types"
)

const (
Expand Down Expand Up @@ -62,6 +64,11 @@ func (v *VirtualizationTool) retrieveListOfContainerIDs() ([]string, bool, []err

var allErrors []error
for _, sandbox := range sandboxes {
if err := v.checkSandboxNetNs(sandbox); err != nil {
allErrors = append(allErrors, err)
continue
}

containers, err := v.metadataStore.ListPodContainers(sandbox.GetID())
if err != nil {
allErrors = append(
Expand All @@ -82,6 +89,37 @@ func (v *VirtualizationTool) retrieveListOfContainerIDs() ([]string, bool, []err
return containerIDs, false, allErrors
}

func (v *VirtualizationTool) checkSandboxNetNs(sandbox metadata.PodSandboxMetadata) error {
sinfo, err := sandbox.Retrieve()
if err != nil {
return err
}

if !v.mountPointChecker.IsPathAnNs(sinfo.ContainerSideNetwork.NsPath) {
containers, err := v.metadataStore.ListPodContainers(sandbox.GetID())
if err != nil {
return err
}
for _, container := range containers {
// remove container from metadata store
if err := container.Save(func(*types.ContainerInfo) (*types.ContainerInfo, error) {
return nil, nil
}); err != nil {
return err
}
}

// remove sandbox from metadata store
if err := sandbox.Save(func(*types.PodSandboxInfo) (*types.PodSandboxInfo, error) {
return nil, nil
}); err != nil {
return err
}
}

return nil
}

func inList(list []string, filter func(string) bool) bool {
for _, element := range list {
if filter(element) {
Expand Down
39 changes: 21 additions & 18 deletions pkg/libvirttools/virtualization.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,15 +217,16 @@ type VirtualizationConfig struct {

// VirtualizationTool provides methods to operate on libvirt.
type VirtualizationTool struct {
domainConn virt.DomainConnection
storageConn virt.StorageConnection
imageManager ImageManager
metadataStore metadata.Store
clock clockwork.Clock
volumeSource VMVolumeSource
config VirtualizationConfig
mounter utils.Mounter
commander utils.Commander
domainConn virt.DomainConnection
storageConn virt.StorageConnection
imageManager ImageManager
metadataStore metadata.Store
clock clockwork.Clock
volumeSource VMVolumeSource
config VirtualizationConfig
mounter utils.Mounter
mountPointChecker utils.MountPointChecker
commander utils.Commander
}

var _ volumeOwner = &VirtualizationTool{}
Expand All @@ -236,17 +237,19 @@ func NewVirtualizationTool(domainConn virt.DomainConnection,
storageConn virt.StorageConnection, imageManager ImageManager,
metadataStore metadata.Store, volumeSource VMVolumeSource,
config VirtualizationConfig, mounter utils.Mounter,
mountPointChecker utils.MountPointChecker,
commander utils.Commander) *VirtualizationTool {
return &VirtualizationTool{
domainConn: domainConn,
storageConn: storageConn,
imageManager: imageManager,
metadataStore: metadataStore,
clock: clockwork.NewRealClock(),
volumeSource: volumeSource,
config: config,
mounter: mounter,
commander: commander,
domainConn: domainConn,
storageConn: storageConn,
imageManager: imageManager,
metadataStore: metadataStore,
clock: clockwork.NewRealClock(),
volumeSource: volumeSource,
config: config,
mounter: mounter,
mountPointChecker: mountPointChecker,
commander: commander,
}
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/libvirttools/virtualization_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ func newContainerTester(t *testing.T, rec *testutils.TopLevelRecorder, cmds []fa
fakeCommander.ReplaceTempPath("__pods__", "/fakedev")
ct.virtTool = NewVirtualizationTool(
ct.domainConn, ct.storageConn, imageManager, ct.metadataStore,
GetDefaultVolumeSource(), virtConfig, utils.NullMounter, fakeCommander)
GetDefaultVolumeSource(), virtConfig, utils.NullMounter,
utils.FakeMountPointChecker, fakeCommander)
ct.virtTool.SetClock(ct.clock)

return ct
Expand Down
9 changes: 7 additions & 2 deletions pkg/manager/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,15 +127,20 @@ func (v *VirtletManager) Run() error {

err = s.Start()
if err != nil {
glog.Warningf("Could not start stream server: %s", err)
glog.Warningf("Could not start stream server: %v", err)

}
streamServer = s
virtConfig.StreamerSocketPath = streamerSocketPath
}

mpc, err := utils.NewMountPointChecker()
if err != nil {
return fmt.Errorf("couldn't create mountpoint checker: %v", err)
}

volSrc := libvirttools.GetDefaultVolumeSource()
v.virtTool = libvirttools.NewVirtualizationTool(conn, conn, v.imageStore, v.metadataStore, volSrc, virtConfig, utils.NewMounter(), utils.DefaultCommander)
v.virtTool = libvirttools.NewVirtualizationTool(conn, conn, v.imageStore, v.metadataStore, volSrc, virtConfig, utils.NewMounter(), mpc, utils.DefaultCommander)

runtimeService := NewVirtletRuntimeService(v.virtTool, v.metadataStore, v.fdManager, streamServer, v.imageStore, nil)
imageService := NewVirtletImageService(v.imageStore, translator, nil)
Expand Down
2 changes: 1 addition & 1 deletion pkg/manager/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ func makeVirtletCRITester(t *testing.T) *virtletCRITester {
StreamerSocketPath: streamerSocketPath,
}
commander := fakeutils.NewCommander(rec, nil)
virtTool := libvirttools.NewVirtualizationTool(domainConn, storageConn, imageStore, metadataStore, libvirttools.GetDefaultVolumeSource(), virtConfig, utils.NullMounter, commander)
virtTool := libvirttools.NewVirtualizationTool(domainConn, storageConn, imageStore, metadataStore, libvirttools.GetDefaultVolumeSource(), virtConfig, utils.NullMounter, utils.FakeMountPointChecker, commander)
virtTool.SetClock(clock)
streamServer := newFakeStreamServer(rec.Child("streamServer"))
criHandler := &criHandler{
Expand Down
124 changes: 124 additions & 0 deletions pkg/utils/mountinfo.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
Copyright 2018 Mirantis
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import (
"bufio"
"io"
"os"
"path/filepath"
"strings"

"github.com/golang/glog"
)

type mountEntry struct {
Source string
Fs string
}

// MountPointChecker provides methods to check if directory entry is a mount point,
// what is its source and its filesystem
type MountPointChecker interface {
CheckMountPointInfo(string) (mountEntry, bool)
IsPathAnNs(string) bool
}

type mountPointChecker struct {
mountInfo map[string]mountEntry
}

var _ MountPointChecker = mountPointChecker{}

// NewMountPointChecker returns new instance of MountPointChecker
func NewMountPointChecker() (MountPointChecker, error) {
file, err := os.Open("/proc/self/mountinfo")
if err != nil {
return mountPointChecker{}, err
}
defer file.Close()

mi := make(map[string]mountEntry)

reader := bufio.NewReader(file)
for {
line, err := reader.ReadString('\n')
if err == io.EOF {
break
}
if err != nil {
return mountPointChecker{}, err
}

// strip eol
line = strings.Trim(line, "\n")

// split and interpret entries acording to 3.5 paragraph in
// https://www.kernel.org/doc/Documentation/filesystems/proc.txt
// TODO: whitespaces and control chars in names are encoded as
// octal values (e.g. for "x x": "x\040x") what should be expanded
// in both mount point source and target
parts := strings.Split(line, " ")
mi[parts[4]] = mountEntry{Source: parts[9], Fs: parts[8]}
}
return mountPointChecker{mountInfo: mi}, nil
}

// CheckMountPointInfo chekcs if entry is a mountpoint and if so returns
// mountInfo for it
func (mpc mountPointChecker) CheckMountPointInfo(path string) (mountEntry, bool) {
entry, ok := mpc.mountInfo[path]
return entry, ok
}

// IsPathAnNs verifies if provided path is mountpoint with nsfs filesystem type
func (mpc mountPointChecker) IsPathAnNs(path string) bool {
_, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
glog.Errorf("Cannot verify existence of %q: %v", path, err)
}
return false
}
realpath, err := filepath.EvalSymlinks(path)
if err != nil {
glog.Errorf("Cannot verify real path of %q: %v", path, err)
return false
}

entry, isMountPoint := mpc.CheckMountPointInfo(realpath)
if !isMountPoint {
return false
}
return entry.Fs == "nsfs"
}

type fakeMountPointChecker struct {
}

// FakeMountPointChecker is defined there for static type checking and for unittest
var FakeMountPointChecker MountPointChecker = fakeMountPointChecker{}

// CheckMountPointInfo is fake implementation for MountPointChecker interface
func (mpc fakeMountPointChecker) CheckMountPointInfo(path string) (mountEntry, bool) {
return mountEntry{}, false
}

// IsPathAnNs is fake implementation for MountPointChecker interface
func (mpc fakeMountPointChecker) IsPathAnNs(path string) bool {
return false
}

0 comments on commit 12c6922

Please sign in to comment.