add 9pfs volume(one 9p mount per fs volume)

Signed-off-by: Yanqiang Miao <[email protected]>

cmd/virtlet/virtlet.go

@@ -86,7 +86,7 @@ func runTapManager(config *v1.VirtletConfig) {
 		glog.Errorf("FD server returned error: %v", err)
 		os.Exit(1)
 	}
-	if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath); err != nil {
+	if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath, false); err != nil {
 		glog.Warningf("Couldn't set tapmanager socket permissions: %v", err)
 	}
 	for {

deploy/data/virtlet-ds.yaml

@@ -185,6 +185,7 @@ spec:
           mountPath: /dev
         - mountPath: /var/lib/virtlet
           name: virtlet
+          mountPropagation: Bidirectional
         - mountPath: /var/lib/libvirt
           name: libvirt
         - mountPath: /var/run/libvirt
@@ -222,10 +223,14 @@ spec:
         volumeMounts:
         - mountPath: /var/lib/virtlet
           name: virtlet
+          mountPropagation: HostToContainer
         - mountPath: /var/lib/libvirt
           name: libvirt
         - name: vms-log
           mountPath: /var/log/vms
+        - mountPath: /var/lib/kubelet/pods
+          name: k8s-pods-dir
+          mountPropagation: HostToContainer
         - name: dev
           mountPath: /dev
         - name: modules

pkg/flexvolume/flexvolume.go

@@ -48,11 +48,11 @@ func init() {
 
 type nullMounter struct{}
 
-func (m *nullMounter) Mount(source string, target string, fstype string) error {
+func (m *nullMounter) Mount(source string, target string, fstype string, flags uintptr) error {
 	return nil
 }
 
-func (m *nullMounter) Unmount(target string) error {
+func (m *nullMounter) Unmount(target string, flags int) error {
 	return nil
 }
 
@@ -65,6 +65,7 @@ type FlexVolumeDriver struct {
 	mounter utils.Mounter
 }
 
+// NewFlexVolumeDriver creates a FlexVolumeDriver struct
 func NewFlexVolumeDriver(uuidGen UuidGen, mounter utils.Mounter) *FlexVolumeDriver {
 	return &FlexVolumeDriver{uuidGen: uuidGen, mounter: mounter}
 }
@@ -127,15 +128,15 @@ func (d *FlexVolumeDriver) mount(targetMountDir, jsonOptions string) (map[string
 		return nil, err
 	}
 
-	if err := d.mounter.Mount("tmpfs", targetMountDir, "tmpfs"); err != nil {
+	if err := d.mounter.Mount("tmpfs", targetMountDir, "tmpfs", 0); err != nil {
 		return nil, fmt.Errorf("error mounting tmpfs at %q: %v", targetMountDir, err)
 	}
 
 	done := false
 	defer func() {
 		// try to unmount upon error or panic
 		if !done {
-			d.mounter.Unmount(targetMountDir)
+			d.mounter.Unmount(targetMountDir, 0)
 		}
 	}()
 
@@ -149,7 +150,7 @@ func (d *FlexVolumeDriver) mount(targetMountDir, jsonOptions string) (map[string
 
 // Invocation: <driver executable> unmount <mount dir>
 func (d *FlexVolumeDriver) unmount(targetMountDir string) (map[string]interface{}, error) {
-	if err := d.mounter.Unmount(targetMountDir); err != nil {
+	if err := d.mounter.Unmount(targetMountDir, 0); err != nil {
 		return nil, fmt.Errorf("unmount %q: %v", targetMountDir, err.Error())
 	}
 
@@ -269,6 +270,10 @@ func formatResult(fields map[string]interface{}, err error) string {
 // means that no partition number was specified.
 func GetFlexvolumeInfo(dir string) (string, int, error) {
 	dataFile := filepath.Join(dir, flexvolumeDataFile)
+	if _, err := os.Stat(dataFile); os.IsNotExist(err) {
+		return "", 0, err
+	}
+
 	var opts map[string]interface{}
 	if err := utils.ReadJSON(dataFile, &opts); err != nil {
 		return "", 0, fmt.Errorf("can't read flexvolume data file %q: %v", dataFile, err)

pkg/flexvolume/flexvolume_test.go

@@ -53,7 +53,7 @@ func (mounter *fakeMounter) validatePath(target string) {
 	}
 }
 
-func (mounter *fakeMounter) Mount(source string, target string, fstype string) error {
+func (mounter *fakeMounter) Mount(source string, target string, fstype string, flags uintptr) error {
 	mounter.validatePath(target)
 	mounter.journal = append(mounter.journal, fmt.Sprintf("mount: %s %s %s", source, target, fstype))
 
@@ -81,7 +81,7 @@ func (mounter *fakeMounter) Mount(source string, target string, fstype string) e
 	return nil
 }
 
-func (mounter *fakeMounter) Unmount(target string) error {
+func (mounter *fakeMounter) Unmount(target string, flags int) error {
 	// we make sure that path is under our tmpdir before wiping it
 	mounter.validatePath(target)
 	mounter.journal = append(mounter.journal, fmt.Sprintf("unmount: %s", target))

pkg/libvirttools/TestContainerLifecycle.out.yaml

@@ -122,6 +122,7 @@
         UserDataOverwrite: false
         UserDataScript: ""
         VCPUCount: 1
+        VirtletChown9pfsMounts: false
       PodAnnotations:
         hello: world
         virt: let
@@ -176,6 +177,7 @@
         UserDataOverwrite: false
         UserDataScript: ""
         VCPUCount: 1
+        VirtletChown9pfsMounts: false
       PodAnnotations:
         hello: world
         virt: let
@@ -225,6 +227,7 @@
         UserDataOverwrite: false
         UserDataScript: ""
         VCPUCount: 1
+        VirtletChown9pfsMounts: false
       PodAnnotations:
         hello: world
         virt: let

pkg/libvirttools/TestDomainForcedShutdown.out.yaml

@@ -138,6 +138,7 @@
         UserDataOverwrite: false
         UserDataScript: ""
         VCPUCount: 1
+        VirtletChown9pfsMounts: false
       PodAnnotations:
         hello: world
         virt: let

pkg/libvirttools/ceph_flexvolume.go

@@ -93,24 +93,24 @@ func (v *cephVolume) UUID() string {
 	return v.opts.UUID
 }
 
-func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, error) {
+func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) {
 	ipPortPair := strings.Split(v.opts.Monitor, ":")
 	if len(ipPortPair) != 2 {
-		return nil, fmt.Errorf("invalid format of ceph monitor setting: %s. Expected ip:port", v.opts.Monitor)
+		return nil, nil, fmt.Errorf("invalid format of ceph monitor setting: %s. Expected ip:port", v.opts.Monitor)
 	}
 
 	secret, err := v.owner.DomainConnection().DefineSecret(v.secretDef())
 	if err != nil {
-		return nil, fmt.Errorf("error defining ceph secret: %v", err)
+		return nil, nil, fmt.Errorf("error defining ceph secret: %v", err)
 	}
 
 	key, err := base64.StdEncoding.DecodeString(v.opts.Secret)
 	if err != nil {
-		return nil, fmt.Errorf("error decoding ceph secret: %v", err)
+		return nil, nil, fmt.Errorf("error decoding ceph secret: %v", err)
 	}
 
 	if err := secret.SetValue([]byte(key)); err != nil {
-		return nil, fmt.Errorf("error setting value of secret %q: %v", v.secretUsageName(), err)
+		return nil, nil, fmt.Errorf("error setting value of secret %q: %v", v.secretUsageName(), err)
 	}
 
 	return &libvirtxml.DomainDisk{
@@ -135,7 +135,7 @@ func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, error) {
 				},
 			},
 		},
-	}, nil
+	}, nil, nil
 }
 
 func (v *cephVolume) Teardown() error {

pkg/libvirttools/cloudinit.go

@@ -467,43 +467,76 @@ func (g *CloudInitGenerator) generateMounts(volumeMap diskPathMap) ([]interface{
 	var mountScriptLines []string
 	for _, m := range g.config.Mounts {
 		// Skip file based mounts (including secrets and config maps).
-		if isRegularFile(m.HostPath) {
+		if isRegularFile(m.HostPath) ||
+			strings.Contains(m.HostPath, "kubernetes.io~secret") ||
+			strings.Contains(m.HostPath, "kubernetes.io~configmap") {
 			continue
 		}
 
-		uuid, part, err := flexvolume.GetFlexvolumeInfo(m.HostPath)
+		mountInfo, mountScriptLine, err := generateFlexvolumeMounts(volumeMap, m)
 		if err != nil {
-			glog.Errorf("Can't mount directory %q to %q inside the VM: can't get flexvolume uuid: %v", m.HostPath, m.ContainerPath, err)
-			continue
-		}
-		dpath, found := volumeMap[uuid]
-		if !found {
-			glog.Errorf("Can't mount directory %q to %q inside the VM: no device found for flexvolume uuid %q", m.HostPath, m.ContainerPath, uuid)
-			continue
-		}
-		if part < 0 {
-			part = 1
-		}
-		devPath := dpath.devPath
-		mountDevSuffix := ""
-		if part != 0 {
-			devPath += fmt.Sprintf("-part%d", part)
-			mountDevSuffix += strconv.Itoa(part)
+			if !os.IsNotExist(err) {
+				glog.Errorf("Can't mount directory %q to %q inside the VM: %v", m.HostPath, m.ContainerPath, err)
+				continue
+			}
+
+			// Fs based volume
+			mountInfo, mountScriptLine, err = generateFsBasedVolumeMounts(m)
+			if err != nil {
+				glog.Errorf("Can't mount directory %q to %q inside the VM: %v", m.HostPath, m.ContainerPath, err)
+				continue
+			}
 		}
-		r = append(r, []interface{}{devPath, m.ContainerPath})
-		mountScriptLines = append(
-			mountScriptLines,
-			// TODO: do better job at escaping m.ContainerPath
-			fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount /dev/`ls %s`%s '%s'; fi",
-				m.ContainerPath, m.ContainerPath, dpath.sysfsPath, mountDevSuffix, m.ContainerPath))
+
+		r = append(r, mountInfo)
+		mountScriptLines = append(mountScriptLines, mountScriptLine)
 	}
+
 	mountScript := ""
 	if len(mountScriptLines) != 0 {
 		mountScript = fmt.Sprintf("#!/bin/sh\n%s\n", strings.Join(mountScriptLines, "\n"))
 	}
 	return r, mountScript
 }
 
+func generateFlexvolumeMounts(volumeMap diskPathMap, mount types.VMMount) ([]interface{}, string, error) {
+	uuid, part, err := flexvolume.GetFlexvolumeInfo(mount.HostPath)
+	if err != nil {
+		// If the error is NotExist, return the original error
+		if os.IsNotExist(err) {
+			return nil, "", err
+		}
+		err = fmt.Errorf("can't get flexvolume uuid: %v", err)
+		return nil, "", err
+	}
+	dpath, found := volumeMap[uuid]
+	if !found {
+		err = fmt.Errorf("no device found for flexvolume uuid %q", uuid)
+		return nil, "", err
+	}
+	if part < 0 {
+		part = 1
+	}
+	devPath := dpath.devPath
+	mountDevSuffix := ""
+	if part != 0 {
+		devPath += fmt.Sprintf("-part%d", part)
+		mountDevSuffix += strconv.Itoa(part)
+	}
+	// TODO: do better job at escaping mount.ContainerPath
+	mountScriptLine := fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount /dev/`ls %s`%s '%s'; fi",
+		mount.ContainerPath, mount.ContainerPath, dpath.sysfsPath, mountDevSuffix, mount.ContainerPath)
+	return []interface{}{devPath, mount.ContainerPath}, mountScriptLine, nil
+}
+
+func generateFsBasedVolumeMounts(mount types.VMMount) ([]interface{}, string, error) {
+	mountTag := path.Base(mount.ContainerPath)
+	fsMountScript := fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount -t 9p -o trans=virtio %s '%s'; fi",
+		mount.ContainerPath, mount.ContainerPath, mountTag, mount.ContainerPath)
+	r := []interface{}{mountTag, mount.ContainerPath, "9p", "trans=virtio"}
+	return r, fsMountScript, nil
+}
+
 type writeFilesUpdater struct {
 	entries []interface{}
 	mounts  []types.VMMount

pkg/libvirttools/config_volumesource.go

@@ -51,8 +51,8 @@ func (v *configVolume) cloudInitGenerator() *CloudInitGenerator {
 	return NewCloudInitGenerator(v.config, configIsoDir)
 }
 
-func (v *configVolume) Setup() (*libvirtxml.DomainDisk, error) {
-	return v.cloudInitGenerator().DiskDef(), nil
+func (v *configVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) {
+	return v.cloudInitGenerator().DiskDef(), nil, nil
 }
 
 func (v *configVolume) WriteImage(volumeMap diskPathMap) error {

pkg/libvirttools/defaultvolumesrc.go

@@ -22,6 +22,7 @@ func GetDefaultVolumeSource() VMVolumeSource {
 	return CombineVMVolumeSources(
 		GetRootVolume,
 		ScanFlexVolumes,
+		GetFileSystemVolumes,
 		// XXX: GetConfigVolume must go last because it
 		// doesn't produce correct name for cdrom devices
 		GetConfigVolume)

pkg/libvirttools/disklist.go

@@ -32,14 +32,16 @@ type diskItem struct {
 	volume VMVolume
 }
 
-func (di *diskItem) setup(config *types.VMConfig) (*libvirtxml.DomainDisk, error) {
-	diskDef, err := di.volume.Setup()
+func (di *diskItem) setup(config *types.VMConfig) (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) {
+	diskDef, fsDef, err := di.volume.Setup()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
+	}
+	if diskDef != nil {
+		diskDef.Target = di.driver.target()
+		diskDef.Address = di.driver.address()
 	}
-	diskDef.Target = di.driver.target()
-	diskDef.Address = di.driver.address()
-	return diskDef, nil
+	return diskDef, fsDef, nil
 }
 
 type diskList struct {
@@ -72,23 +74,29 @@ func newDiskList(config *types.VMConfig, source VMVolumeSource, owner volumeOwne
 }
 
 // setup performs the setup procedure on each volume in the diskList
-// and returns a list of libvirtxml DomainDisk structs
-func (dl *diskList) setup() ([]libvirtxml.DomainDisk, error) {
+// and returns a list of libvirtxml DomainDisk and domainFileSystems structs
+func (dl *diskList) setup() ([]libvirtxml.DomainDisk, []libvirtxml.DomainFilesystem, error) {
 	var domainDisks []libvirtxml.DomainDisk
+	var domainFileSystems []libvirtxml.DomainFilesystem
 	for n, item := range dl.items {
-		diskDef, err := item.setup(dl.config)
+		diskDef, fsDef, err := item.setup(dl.config)
 		if err != nil {
 			// try to tear down volumes that were already set up
 			for _, item := range dl.items[:n] {
 				if err := item.volume.Teardown(); err != nil {
 					glog.Warningf("Failed to tear down a volume on error: %v", err)
 				}
 			}
-			return nil, err
+			return nil, nil, err
+		}
+		if diskDef != nil {
+			domainDisks = append(domainDisks, *diskDef)
+		}
+		if fsDef != nil {
+			domainFileSystems = append(domainFileSystems, *fsDef)
 		}
-		domainDisks = append(domainDisks, *diskDef)
 	}
-	return domainDisks, nil
+	return domainDisks, domainFileSystems, nil
 }
 
 // writeImages writes images for volumes that are based on generated