add 9pfs volume(one 9p mount per fs volume)

Signed-off-by: Yanqiang Miao <[email protected]>

cmd/virtlet/virtlet.go

@@ -86,7 +86,7 @@ func runTapManager(config *v1.VirtletConfig) {
 		glog.Errorf("FD server returned error: %v", err)
 		os.Exit(1)
 	}
-	if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath); err != nil {
+	if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath, false); err != nil {
 		glog.Warningf("Couldn't set tapmanager socket permissions: %v", err)
 	}
 	for {

deploy/data/virtlet-ds.yaml

@@ -185,6 +185,7 @@ spec:
           mountPath: /dev
         - mountPath: /var/lib/virtlet
           name: virtlet
+          mountPropagation: Bidirectional
         - mountPath: /var/lib/libvirt
           name: libvirt
         - mountPath: /var/run/libvirt
@@ -219,10 +220,14 @@ spec:
         volumeMounts:
         - mountPath: /var/lib/virtlet
           name: virtlet
+          mountPropagation: HostToContainer
         - mountPath: /var/lib/libvirt
           name: libvirt
         - name: vms-log
           mountPath: /var/log/vms
+        - mountPath: /var/lib/kubelet/pods
+          name: k8s-pods-dir
+          mountPropagation: HostToContainer
         - name: dev
           mountPath: /dev
         - name: modules

pkg/flexvolume/flexvolume.go

@@ -269,6 +269,10 @@ func formatResult(fields map[string]interface{}, err error) string {
 // means that no partition number was specified.
 func GetFlexvolumeInfo(dir string) (string, int, error) {
 	dataFile := filepath.Join(dir, flexvolumeDataFile)
+	if _, err := os.Stat(dataFile); os.IsNotExist(err) {
+		return "", 0, err
+	}
+
 	var opts map[string]interface{}
 	if err := utils.ReadJSON(dataFile, &opts); err != nil {
 		return "", 0, fmt.Errorf("can't read flexvolume data file %q: %v", dataFile, err)

pkg/libvirttools/ceph_flexvolume.go

@@ -93,24 +93,24 @@ func (v *cephVolume) UUID() string {
 	return v.opts.UUID
 }
 
-func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, error) {
+func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) {
 	ipPortPair := strings.Split(v.opts.Monitor, ":")
 	if len(ipPortPair) != 2 {
-		return nil, fmt.Errorf("invalid format of ceph monitor setting: %s. Expected ip:port", v.opts.Monitor)
+		return nil, nil, fmt.Errorf("invalid format of ceph monitor setting: %s. Expected ip:port", v.opts.Monitor)
 	}
 
 	secret, err := v.owner.DomainConnection().DefineSecret(v.secretDef())
 	if err != nil {
-		return nil, fmt.Errorf("error defining ceph secret: %v", err)
+		return nil, nil, fmt.Errorf("error defining ceph secret: %v", err)
 	}
 
 	key, err := base64.StdEncoding.DecodeString(v.opts.Secret)
 	if err != nil {
-		return nil, fmt.Errorf("error decoding ceph secret: %v", err)
+		return nil, nil, fmt.Errorf("error decoding ceph secret: %v", err)
 	}
 
 	if err := secret.SetValue([]byte(key)); err != nil {
-		return nil, fmt.Errorf("error setting value of secret %q: %v", v.secretUsageName(), err)
+		return nil, nil, fmt.Errorf("error setting value of secret %q: %v", v.secretUsageName(), err)
 	}
 
 	return &libvirtxml.DomainDisk{
@@ -135,7 +135,7 @@ func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, error) {
 				},
 			},
 		},
-	}, nil
+	}, nil, nil
 }
 
 func (v *cephVolume) Teardown() error {

pkg/libvirttools/cloudinit.go

@@ -467,43 +467,76 @@ func (g *CloudInitGenerator) generateMounts(volumeMap diskPathMap) ([]interface{
 	var mountScriptLines []string
 	for _, m := range g.config.Mounts {
 		// Skip file based mounts (including secrets and config maps).
-		if isRegularFile(m.HostPath) {
+		if isRegularFile(m.HostPath) ||
+			strings.Contains(m.HostPath, "kubernetes.io~secret") ||
+			strings.Contains(m.HostPath, "kubernetes.io~configmap") {
 			continue
 		}
 
-		uuid, part, err := flexvolume.GetFlexvolumeInfo(m.HostPath)
+		mountInfo, mountScriptLine, err := generateFlexvolumeMounts(volumeMap, m)
 		if err != nil {
-			glog.Errorf("Can't mount directory %q to %q inside the VM: can't get flexvolume uuid: %v", m.HostPath, m.ContainerPath, err)
-			continue
-		}
-		dpath, found := volumeMap[uuid]
-		if !found {
-			glog.Errorf("Can't mount directory %q to %q inside the VM: no device found for flexvolume uuid %q", m.HostPath, m.ContainerPath, uuid)
-			continue
-		}
-		if part < 0 {
-			part = 1
-		}
-		devPath := dpath.devPath
-		mountDevSuffix := ""
-		if part != 0 {
-			devPath += fmt.Sprintf("-part%d", part)
-			mountDevSuffix += strconv.Itoa(part)
+			if !os.IsNotExist(err) {
+				glog.Errorf("Can't mount directory %q to %q inside the VM: %v", m.HostPath, m.ContainerPath, err)
+				continue
+			}
+
+			// Fs based volume
+			mountInfo, mountScriptLine, err = generateFsBasedVolumeMounts(m)
+			if err != nil {
+				glog.Errorf("Can't mount directory %q to %q inside the VM: %v", m.HostPath, m.ContainerPath, err)
+				continue
+			}
 		}
-		r = append(r, []interface{}{devPath, m.ContainerPath})
-		mountScriptLines = append(
-			mountScriptLines,
-			// TODO: do better job at escaping m.ContainerPath
-			fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount /dev/`ls %s`%s '%s'; fi",
-				m.ContainerPath, m.ContainerPath, dpath.sysfsPath, mountDevSuffix, m.ContainerPath))
+
+		r = append(r, mountInfo)
+		mountScriptLines = append(mountScriptLines, mountScriptLine)
 	}
+
 	mountScript := ""
 	if len(mountScriptLines) != 0 {
 		mountScript = fmt.Sprintf("#!/bin/sh\n%s\n", strings.Join(mountScriptLines, "\n"))
 	}
 	return r, mountScript
 }
 
+func generateFlexvolumeMounts(volumeMap diskPathMap, mount types.VMMount) ([]interface{}, string, error) {
+	uuid, part, err := flexvolume.GetFlexvolumeInfo(mount.HostPath)
+	if err != nil {
+		// If the error is NotExist, return the original error
+		if os.IsNotExist(err) {
+			return nil, "", err
+		}
+		err = fmt.Errorf("can't get flexvolume uuid: %v", err)
+		return nil, "", err
+	}
+	dpath, found := volumeMap[uuid]
+	if !found {
+		err = fmt.Errorf("no device found for flexvolume uuid %q", uuid)
+		return nil, "", err
+	}
+	if part < 0 {
+		part = 1
+	}
+	devPath := dpath.devPath
+	mountDevSuffix := ""
+	if part != 0 {
+		devPath += fmt.Sprintf("-part%d", part)
+		mountDevSuffix += strconv.Itoa(part)
+	}
+	// TODO: do better job at escaping mount.ContainerPath
+	mountScriptLine := fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount /dev/`ls %s`%s '%s'; fi",
+		mount.ContainerPath, mount.ContainerPath, dpath.sysfsPath, mountDevSuffix, mount.ContainerPath)
+	return []interface{}{devPath, mount.ContainerPath}, mountScriptLine, nil
+}
+
+func generateFsBasedVolumeMounts(mount types.VMMount) ([]interface{}, string, error) {
+	mountTag := path.Base(mount.ContainerPath)
+	fsMountScript := fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount -t 9p -o trans=virtio %s '%s'; fi",
+		mount.ContainerPath, mount.ContainerPath, mountTag, mount.ContainerPath)
+	r := []interface{}{mountTag, mount.ContainerPath, "9p", "trans=virtio"}
+	return r, fsMountScript, nil
+}
+
 type writeFilesUpdater struct {
 	entries []interface{}
 	mounts  []types.VMMount

pkg/libvirttools/config_volumesource.go

@@ -51,8 +51,8 @@ func (v *configVolume) cloudInitGenerator() *CloudInitGenerator {
 	return NewCloudInitGenerator(v.config, configIsoDir)
 }
 
-func (v *configVolume) Setup() (*libvirtxml.DomainDisk, error) {
-	return v.cloudInitGenerator().DiskDef(), nil
+func (v *configVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) {
+	return v.cloudInitGenerator().DiskDef(),nil, nil
 }
 
 func (v *configVolume) WriteImage(volumeMap diskPathMap) error {

pkg/libvirttools/defaultvolumesrc.go

@@ -22,6 +22,7 @@ func GetDefaultVolumeSource() VMVolumeSource {
 	return CombineVMVolumeSources(
 		GetRootVolume,
 		ScanFlexVolumes,
+		GetFileSystemVolumes,
 		// XXX: GetConfigVolume must go last because it
 		// doesn't produce correct name for cdrom devices
 		GetConfigVolume)

pkg/libvirttools/disklist.go

@@ -32,14 +32,16 @@ type diskItem struct {
 	volume VMVolume
 }
 
-func (di *diskItem) setup(config *types.VMConfig) (*libvirtxml.DomainDisk, error) {
-	diskDef, err := di.volume.Setup()
+func (di *diskItem) setup(config *types.VMConfig) (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) {
+	diskDef, fsDef, err := di.volume.Setup()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
+	}
+	if diskDef != nil {
+		diskDef.Target = di.driver.target()
+		diskDef.Address = di.driver.address()
 	}
-	diskDef.Target = di.driver.target()
-	diskDef.Address = di.driver.address()
-	return diskDef, nil
+	return diskDef, fsDef, nil
 }
 
 type diskList struct {
@@ -72,23 +74,29 @@ func newDiskList(config *types.VMConfig, source VMVolumeSource, owner volumeOwne
 }
 
 // setup performs the setup procedure on each volume in the diskList
-// and returns a list of libvirtxml DomainDisk structs
-func (dl *diskList) setup() ([]libvirtxml.DomainDisk, error) {
+// and returns a list of libvirtxml DomainDisk and domainFileSystems structs
+func (dl *diskList) setup() ([]libvirtxml.DomainDisk, []libvirtxml.DomainFilesystem, error) {
 	var domainDisks []libvirtxml.DomainDisk
+	var domainFileSystems []libvirtxml.DomainFilesystem
 	for n, item := range dl.items {
-		diskDef, err := item.setup(dl.config)
+		diskDef, fsDef, err := item.setup(dl.config)
 		if err != nil {
 			// try to tear down volumes that were already set up
 			for _, item := range dl.items[:n] {
 				if err := item.volume.Teardown(); err != nil {
 					glog.Warningf("Failed to tear down a volume on error: %v", err)
 				}
 			}
-			return nil, err
+			return nil, nil, err
+		}
+		if diskDef != nil {
+			domainDisks = append(domainDisks, *diskDef)
+		}
+		if fsDef != nil {
+			domainFileSystems = append(domainFileSystems, *fsDef)
 		}
-		domainDisks = append(domainDisks, *diskDef)
 	}
-	return domainDisks, nil
+	return domainDisks, domainFileSystems, nil
 }
 
 // writeImages writes images for volumes that are based on generated

pkg/libvirttools/fileownership.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"os/user"
+	"path/filepath"
 	"strconv"
 	"sync"
 )
@@ -35,7 +36,7 @@ var emulatorUser struct {
 }
 
 // ChownForEmulator makes a file or directory owned by the emulator user.
-func ChownForEmulator(filePath string) error {
+func ChownForEmulator(filePath string, recursive bool) error {
 	emulatorUser.Lock()
 	defer emulatorUser.Unlock()
 	if !emulatorUser.initialized {
@@ -52,8 +53,23 @@ func ChownForEmulator(filePath string) error {
 			return fmt.Errorf("bad gid %q for user %q: %v", u.Gid, emulatorUserName, err)
 		}
 	}
-	if err := os.Chown(filePath, emulatorUser.uid, emulatorUser.gid); err != nil {
+
+	chown := os.Chown
+	if recursive {
+		chown = ChownR
+	}
+	if err := chown(filePath, emulatorUser.uid, emulatorUser.gid); err != nil {
 		return fmt.Errorf("can't set the owner of tapmanager socket: %v", err)
 	}
 	return nil
 }
+
+// ChownR makes a file or directory owned by the emulator user recursively.
+func ChownR(path string, uid, gid int) error {
+	return filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
+		if err == nil {
+			err = os.Chown(name, uid, gid)
+		}
+		return err
+	})
+}

pkg/libvirttools/filesystem_volumesource.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2018 ZTE corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package libvirttools
+
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/Mirantis/virtlet/pkg/metadata/types"
+)
+
+// GetFileSystemVolumes using prepared by kubelet volumes and contained in pod sandbox
+// annotations prepares volumes to be passed to libvirt as a DomainFileSystem definitions.
+func GetFileSystemVolumes(config *types.VMConfig, owner volumeOwner) ([]VMVolume, error) {
+	volumePoolPath := supportedStoragePools[owner.VolumePoolName()]
+	if _, err := os.Stat(volumePoolPath); err != nil {
+		return nil, err
+	}
+
+	var fsVolumes []VMVolume
+	for index, mount := range config.Mounts {
+		if isRegularFile(mount.HostPath) ||
+			strings.Contains(mount.HostPath, flexvolumeSubdir) ||
+			strings.Contains(mount.HostPath, "kubernetes.io~secret") ||
+			strings.Contains(mount.HostPath, "kubernetes.io~configmap") {
+			continue
+		}
+
+		// `Index` is used to avoid causing conflicts as multiple host paths can have the same `path.Base`
+		volumeDirName := fmt.Sprintf("virtlet_%s_%s_%d", config.DomainUUID, path.Base(mount.HostPath), index)
+		volumeMountPoint := path.Join(volumePoolPath, volumeDirName)
+		fsVolumes = append(fsVolumes, &filesystemVolume{mount: mount, volumeMountPoint: volumeMountPoint})
+	}
+
+	return fsVolumes, nil
+}