diff --git a/cmd/virtlet/virtlet.go b/cmd/virtlet/virtlet.go index e39cbb0d7..a5b6b1bbe 100644 --- a/cmd/virtlet/virtlet.go +++ b/cmd/virtlet/virtlet.go @@ -86,7 +86,7 @@ func runTapManager(config *v1.VirtletConfig) { glog.Errorf("FD server returned error: %v", err) os.Exit(1) } - if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath); err != nil { + if err := libvirttools.ChownForEmulator(*config.FDServerSocketPath, false); err != nil { glog.Warningf("Couldn't set tapmanager socket permissions: %v", err) } for { diff --git a/deploy/data/virtlet-ds.yaml b/deploy/data/virtlet-ds.yaml index 0d0f66229..171d68399 100644 --- a/deploy/data/virtlet-ds.yaml +++ b/deploy/data/virtlet-ds.yaml @@ -185,6 +185,7 @@ spec: mountPath: /dev - mountPath: /var/lib/virtlet name: virtlet + mountPropagation: Bidirectional - mountPath: /var/lib/libvirt name: libvirt - mountPath: /var/run/libvirt @@ -219,10 +220,14 @@ spec: volumeMounts: - mountPath: /var/lib/virtlet name: virtlet + mountPropagation: HostToContainer - mountPath: /var/lib/libvirt name: libvirt - name: vms-log mountPath: /var/log/vms + - mountPath: /var/lib/kubelet/pods + name: k8s-pods-dir + mountPropagation: HostToContainer - name: dev mountPath: /dev - name: modules diff --git a/pkg/flexvolume/flexvolume.go b/pkg/flexvolume/flexvolume.go index ff4387a21..3196b94a8 100644 --- a/pkg/flexvolume/flexvolume.go +++ b/pkg/flexvolume/flexvolume.go @@ -269,6 +269,10 @@ func formatResult(fields map[string]interface{}, err error) string { // means that no partition number was specified. func GetFlexvolumeInfo(dir string) (string, int, error) { dataFile := filepath.Join(dir, flexvolumeDataFile) + if _, err := os.Stat(dataFile); os.IsNotExist(err) { + return "", 0, err + } + var opts map[string]interface{} if err := utils.ReadJSON(dataFile, &opts); err != nil { return "", 0, fmt.Errorf("can't read flexvolume data file %q: %v", dataFile, err) diff --git a/pkg/libvirttools/ceph_flexvolume.go b/pkg/libvirttools/ceph_flexvolume.go index 4c6a06cb4..40f6a49e6 100644 --- a/pkg/libvirttools/ceph_flexvolume.go +++ b/pkg/libvirttools/ceph_flexvolume.go @@ -93,24 +93,24 @@ func (v *cephVolume) UUID() string { return v.opts.UUID } -func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, error) { +func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) { ipPortPair := strings.Split(v.opts.Monitor, ":") if len(ipPortPair) != 2 { - return nil, fmt.Errorf("invalid format of ceph monitor setting: %s. Expected ip:port", v.opts.Monitor) + return nil, nil, fmt.Errorf("invalid format of ceph monitor setting: %s. Expected ip:port", v.opts.Monitor) } secret, err := v.owner.DomainConnection().DefineSecret(v.secretDef()) if err != nil { - return nil, fmt.Errorf("error defining ceph secret: %v", err) + return nil, nil, fmt.Errorf("error defining ceph secret: %v", err) } key, err := base64.StdEncoding.DecodeString(v.opts.Secret) if err != nil { - return nil, fmt.Errorf("error decoding ceph secret: %v", err) + return nil, nil, fmt.Errorf("error decoding ceph secret: %v", err) } if err := secret.SetValue([]byte(key)); err != nil { - return nil, fmt.Errorf("error setting value of secret %q: %v", v.secretUsageName(), err) + return nil, nil, fmt.Errorf("error setting value of secret %q: %v", v.secretUsageName(), err) } return &libvirtxml.DomainDisk{ @@ -135,7 +135,7 @@ func (v *cephVolume) Setup() (*libvirtxml.DomainDisk, error) { }, }, }, - }, nil + }, nil, nil } func (v *cephVolume) Teardown() error { diff --git a/pkg/libvirttools/cloudinit.go b/pkg/libvirttools/cloudinit.go index f211d03a3..4613a167e 100644 --- a/pkg/libvirttools/cloudinit.go +++ b/pkg/libvirttools/cloudinit.go @@ -467,36 +467,31 @@ func (g *CloudInitGenerator) generateMounts(volumeMap diskPathMap) ([]interface{ var mountScriptLines []string for _, m := range g.config.Mounts { // Skip file based mounts (including secrets and config maps). - if isRegularFile(m.HostPath) { + if isRegularFile(m.HostPath) || + strings.Contains(m.HostPath, "kubernetes.io~secret") || + strings.Contains(m.HostPath, "kubernetes.io~configmap") { continue } - uuid, part, err := flexvolume.GetFlexvolumeInfo(m.HostPath) + mountInfo, mountScriptLine, err := generateFlexvolumeMounts(volumeMap, m) if err != nil { - glog.Errorf("Can't mount directory %q to %q inside the VM: can't get flexvolume uuid: %v", m.HostPath, m.ContainerPath, err) - continue - } - dpath, found := volumeMap[uuid] - if !found { - glog.Errorf("Can't mount directory %q to %q inside the VM: no device found for flexvolume uuid %q", m.HostPath, m.ContainerPath, uuid) - continue - } - if part < 0 { - part = 1 - } - devPath := dpath.devPath - mountDevSuffix := "" - if part != 0 { - devPath += fmt.Sprintf("-part%d", part) - mountDevSuffix += strconv.Itoa(part) + if !os.IsNotExist(err) { + glog.Errorf("Can't mount directory %q to %q inside the VM: %v", m.HostPath, m.ContainerPath, err) + continue + } + + // Fs based volume + mountInfo, mountScriptLine, err = generateFsBasedVolumeMounts(m) + if err != nil { + glog.Errorf("Can't mount directory %q to %q inside the VM: %v", m.HostPath, m.ContainerPath, err) + continue + } } - r = append(r, []interface{}{devPath, m.ContainerPath}) - mountScriptLines = append( - mountScriptLines, - // TODO: do better job at escaping m.ContainerPath - fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount /dev/`ls %s`%s '%s'; fi", - m.ContainerPath, m.ContainerPath, dpath.sysfsPath, mountDevSuffix, m.ContainerPath)) + + r = append(r, mountInfo) + mountScriptLines = append(mountScriptLines, mountScriptLine) } + mountScript := "" if len(mountScriptLines) != 0 { mountScript = fmt.Sprintf("#!/bin/sh\n%s\n", strings.Join(mountScriptLines, "\n")) @@ -504,6 +499,44 @@ func (g *CloudInitGenerator) generateMounts(volumeMap diskPathMap) ([]interface{ return r, mountScript } +func generateFlexvolumeMounts(volumeMap diskPathMap, mount types.VMMount) ([]interface{}, string, error) { + uuid, part, err := flexvolume.GetFlexvolumeInfo(mount.HostPath) + if err != nil { + // If the error is NotExist, return the original error + if os.IsNotExist(err) { + return nil, "", err + } + err = fmt.Errorf("can't get flexvolume uuid: %v", err) + return nil, "", err + } + dpath, found := volumeMap[uuid] + if !found { + err = fmt.Errorf("no device found for flexvolume uuid %q", uuid) + return nil, "", err + } + if part < 0 { + part = 1 + } + devPath := dpath.devPath + mountDevSuffix := "" + if part != 0 { + devPath += fmt.Sprintf("-part%d", part) + mountDevSuffix += strconv.Itoa(part) + } + // TODO: do better job at escaping mount.ContainerPath + mountScriptLine := fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount /dev/`ls %s`%s '%s'; fi", + mount.ContainerPath, mount.ContainerPath, dpath.sysfsPath, mountDevSuffix, mount.ContainerPath) + return []interface{}{devPath, mount.ContainerPath}, mountScriptLine, nil +} + +func generateFsBasedVolumeMounts(mount types.VMMount) ([]interface{}, string, error) { + mountTag := path.Base(mount.ContainerPath) + fsMountScript := fmt.Sprintf("if ! mountpoint '%s'; then mkdir -p '%s' && mount -t 9p -o trans=virtio %s '%s'; fi", + mount.ContainerPath, mount.ContainerPath, mountTag, mount.ContainerPath) + r := []interface{}{mountTag, mount.ContainerPath, "9p", "trans=virtio"} + return r, fsMountScript, nil +} + type writeFilesUpdater struct { entries []interface{} mounts []types.VMMount diff --git a/pkg/libvirttools/config_volumesource.go b/pkg/libvirttools/config_volumesource.go index 8523cbb27..b9bb01642 100644 --- a/pkg/libvirttools/config_volumesource.go +++ b/pkg/libvirttools/config_volumesource.go @@ -51,8 +51,8 @@ func (v *configVolume) cloudInitGenerator() *CloudInitGenerator { return NewCloudInitGenerator(v.config, configIsoDir) } -func (v *configVolume) Setup() (*libvirtxml.DomainDisk, error) { - return v.cloudInitGenerator().DiskDef(), nil +func (v *configVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) { + return v.cloudInitGenerator().DiskDef(),nil, nil } func (v *configVolume) WriteImage(volumeMap diskPathMap) error { diff --git a/pkg/libvirttools/defaultvolumesrc.go b/pkg/libvirttools/defaultvolumesrc.go index b56a99a1f..028771ff6 100644 --- a/pkg/libvirttools/defaultvolumesrc.go +++ b/pkg/libvirttools/defaultvolumesrc.go @@ -22,6 +22,7 @@ func GetDefaultVolumeSource() VMVolumeSource { return CombineVMVolumeSources( GetRootVolume, ScanFlexVolumes, + GetFileSystemVolumes, // XXX: GetConfigVolume must go last because it // doesn't produce correct name for cdrom devices GetConfigVolume) diff --git a/pkg/libvirttools/disklist.go b/pkg/libvirttools/disklist.go index f83b06902..a0733aa05 100644 --- a/pkg/libvirttools/disklist.go +++ b/pkg/libvirttools/disklist.go @@ -32,14 +32,16 @@ type diskItem struct { volume VMVolume } -func (di *diskItem) setup(config *types.VMConfig) (*libvirtxml.DomainDisk, error) { - diskDef, err := di.volume.Setup() +func (di *diskItem) setup(config *types.VMConfig) (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) { + diskDef, fsDef, err := di.volume.Setup() if err != nil { - return nil, err + return nil, nil, err + } + if diskDef != nil { + diskDef.Target = di.driver.target() + diskDef.Address = di.driver.address() } - diskDef.Target = di.driver.target() - diskDef.Address = di.driver.address() - return diskDef, nil + return diskDef, fsDef, nil } type diskList struct { @@ -72,11 +74,12 @@ func newDiskList(config *types.VMConfig, source VMVolumeSource, owner volumeOwne } // setup performs the setup procedure on each volume in the diskList -// and returns a list of libvirtxml DomainDisk structs -func (dl *diskList) setup() ([]libvirtxml.DomainDisk, error) { +// and returns a list of libvirtxml DomainDisk and domainFileSystems structs +func (dl *diskList) setup() ([]libvirtxml.DomainDisk, []libvirtxml.DomainFilesystem, error) { var domainDisks []libvirtxml.DomainDisk + var domainFileSystems []libvirtxml.DomainFilesystem for n, item := range dl.items { - diskDef, err := item.setup(dl.config) + diskDef, fsDef, err := item.setup(dl.config) if err != nil { // try to tear down volumes that were already set up for _, item := range dl.items[:n] { @@ -84,11 +87,16 @@ func (dl *diskList) setup() ([]libvirtxml.DomainDisk, error) { glog.Warningf("Failed to tear down a volume on error: %v", err) } } - return nil, err + return nil, nil, err + } + if diskDef != nil { + domainDisks = append(domainDisks, *diskDef) + } + if fsDef != nil { + domainFileSystems = append(domainFileSystems, *fsDef) } - domainDisks = append(domainDisks, *diskDef) } - return domainDisks, nil + return domainDisks, domainFileSystems, nil } // writeImages writes images for volumes that are based on generated diff --git a/pkg/libvirttools/fileownership.go b/pkg/libvirttools/fileownership.go index 267655f15..008abe299 100644 --- a/pkg/libvirttools/fileownership.go +++ b/pkg/libvirttools/fileownership.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "os/user" + "path/filepath" "strconv" "sync" ) @@ -35,7 +36,7 @@ var emulatorUser struct { } // ChownForEmulator makes a file or directory owned by the emulator user. -func ChownForEmulator(filePath string) error { +func ChownForEmulator(filePath string, recursive bool) error { emulatorUser.Lock() defer emulatorUser.Unlock() if !emulatorUser.initialized { @@ -52,8 +53,23 @@ func ChownForEmulator(filePath string) error { return fmt.Errorf("bad gid %q for user %q: %v", u.Gid, emulatorUserName, err) } } - if err := os.Chown(filePath, emulatorUser.uid, emulatorUser.gid); err != nil { + + chown := os.Chown + if recursive { + chown = ChownR + } + if err := chown(filePath, emulatorUser.uid, emulatorUser.gid); err != nil { return fmt.Errorf("can't set the owner of tapmanager socket: %v", err) } return nil } + +// ChownR makes a file or directory owned by the emulator user recursively. +func ChownR(path string, uid, gid int) error { + return filepath.Walk(path, func(name string, info os.FileInfo, err error) error { + if err == nil { + err = os.Chown(name, uid, gid) + } + return err + }) +} diff --git a/pkg/libvirttools/filesystem_volumesource.go b/pkg/libvirttools/filesystem_volumesource.go new file mode 100644 index 000000000..8a2462a32 --- /dev/null +++ b/pkg/libvirttools/filesystem_volumesource.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 ZTE corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libvirttools + + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/Mirantis/virtlet/pkg/metadata/types" +) + +// GetFileSystemVolumes using prepared by kubelet volumes and contained in pod sandbox +// annotations prepares volumes to be passed to libvirt as a DomainFileSystem definitions. +func GetFileSystemVolumes(config *types.VMConfig, owner volumeOwner) ([]VMVolume, error) { + volumePoolPath := supportedStoragePools[owner.VolumePoolName()] + if _, err := os.Stat(volumePoolPath); err != nil { + return nil, err + } + + var fsVolumes []VMVolume + for index, mount := range config.Mounts { + if isRegularFile(mount.HostPath) || + strings.Contains(mount.HostPath, flexvolumeSubdir) || + strings.Contains(mount.HostPath, "kubernetes.io~secret") || + strings.Contains(mount.HostPath, "kubernetes.io~configmap") { + continue + } + + // `Index` is used to avoid causing conflicts as multiple host paths can have the same `path.Base` + volumeDirName := fmt.Sprintf("virtlet_%s_%s_%d", config.DomainUUID, path.Base(mount.HostPath), index) + volumeMountPoint := path.Join(volumePoolPath, volumeDirName) + fsVolumes = append(fsVolumes, &filesystemVolume{mount: mount, volumeMountPoint: volumeMountPoint}) + } + + return fsVolumes, nil +} diff --git a/pkg/libvirttools/filesystemvolume.go b/pkg/libvirttools/filesystemvolume.go new file mode 100644 index 000000000..0a8281bf2 --- /dev/null +++ b/pkg/libvirttools/filesystemvolume.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 ZTE corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libvirttools + +import ( + "fmt" + "os" + "path" + + libvirtxml "github.com/libvirt/libvirt-go-xml" + + "github.com/Mirantis/virtlet/pkg/metadata/types" + "github.com/Mirantis/virtlet/pkg/utils" +) + +// rootVolume denotes the root disk of the VM +type filesystemVolume struct { + volumeBase + mount types.VMMount + volumeMountPoint string +} + +var _ VMVolume = &filesystemVolume{} + +var mounter = utils.NewMounter() + +func (v *filesystemVolume) UUID() string { return "" } + +func (v *filesystemVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) { + err := os.MkdirAll(v.volumeMountPoint, 0777) + if err == nil { + err = ChownForEmulator(v.volumeMountPoint, true) + } + if err == nil { + err = mounter.Mount(v.mount.HostPath, v.volumeMountPoint, "bind") + } + if err == nil { + err = ChownForEmulator(v.volumeMountPoint, true) + } + if err != nil { + return nil, nil, fmt.Errorf("failed to create vm pod path: %v", err) + } + + fsDef := &libvirtxml.DomainFilesystem{ + AccessMode: "squash", + Source: &libvirtxml.DomainFilesystemSource{Mount: &libvirtxml.DomainFilesystemSourceMount{Dir: v.volumeMountPoint}}, + Target: &libvirtxml.DomainFilesystemTarget{Dir: path.Base(v.mount.ContainerPath)}, + } + + return nil, fsDef, nil +} + +func (v *filesystemVolume) Teardown() error { + var err error + if _, err = os.Stat(v.volumeMountPoint); err == nil { + err = mounter.Unmount(v.volumeMountPoint) + } + if err == nil { + err = os.RemoveAll(v.volumeMountPoint) + } + if err != nil { + return fmt.Errorf("failed to tear down fs volume mountpoint '%s': %v", v.volumeMountPoint, err) + } + return nil +} diff --git a/pkg/libvirttools/flexvolume_volumesource.go b/pkg/libvirttools/flexvolume_volumesource.go index b23fae9ec..22fc07bb1 100644 --- a/pkg/libvirttools/flexvolume_volumesource.go +++ b/pkg/libvirttools/flexvolume_volumesource.go @@ -67,6 +67,9 @@ func ScanFlexVolumes(config *types.VMConfig, owner volumeOwner) ([]VMVolume, err dataFilePath := filepath.Join(dir, fi.Name(), flexvolumeDataFile) content, err := ioutil.ReadFile(dataFilePath) if err != nil { + if os.IsNotExist(err) { + continue + } return nil, fmt.Errorf("error reading flexvolume config %q: %v", dataFilePath, err) } var msi map[string]interface{} diff --git a/pkg/libvirttools/qcow2_flexvolume.go b/pkg/libvirttools/qcow2_flexvolume.go index 998ac890a..705d31b0b 100644 --- a/pkg/libvirttools/qcow2_flexvolume.go +++ b/pkg/libvirttools/qcow2_flexvolume.go @@ -97,27 +97,27 @@ func (v *qcow2Volume) UUID() string { return v.uuid } -func (v *qcow2Volume) Setup() (*libvirtxml.DomainDisk, error) { +func (v *qcow2Volume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) { vol, err := v.createQCOW2Volume(uint64(v.capacity), v.capacityUnit) if err != nil { - return nil, fmt.Errorf("error during creation of volume '%s' with virtlet description %s: %v", v.volumeName(), v.name, err) + return nil, nil, fmt.Errorf("error during creation of volume '%s' with virtlet description %s: %v", v.volumeName(), v.name, err) } path, err := vol.Path() if err != nil { - return nil, err + return nil, nil, err } err = vol.Format() if err != nil { - return nil, err + return nil, nil, err } return &libvirtxml.DomainDisk{ Device: "disk", Source: &libvirtxml.DomainDiskSource{File: &libvirtxml.DomainDiskSourceFile{File: path}}, Driver: &libvirtxml.DomainDiskDriver{Name: "qemu", Type: "qcow2"}, - }, nil + }, nil, nil } func (v *qcow2Volume) Teardown() error { diff --git a/pkg/libvirttools/qcow2_flexvolume_test.go b/pkg/libvirttools/qcow2_flexvolume_test.go index 02f67bbff..413411d8d 100644 --- a/pkg/libvirttools/qcow2_flexvolume_test.go +++ b/pkg/libvirttools/qcow2_flexvolume_test.go @@ -91,7 +91,7 @@ func TestQCOW2VolumeLifeCycle(t *testing.T) { t.Fatalf("newQCOW2Volume returned an error: %v", err) } - vol, err := volume.Setup() + vol, _, err := volume.Setup() if err != nil { t.Errorf("Setup returned an error: %v", err) } diff --git a/pkg/libvirttools/raw_flexvolume.go b/pkg/libvirttools/raw_flexvolume.go index 60681fb05..6c418e866 100644 --- a/pkg/libvirttools/raw_flexvolume.go +++ b/pkg/libvirttools/raw_flexvolume.go @@ -78,19 +78,19 @@ func (v *rawDeviceVolume) UUID() string { return v.opts.UUID } -func (v *rawDeviceVolume) Setup() (*libvirtxml.DomainDisk, error) { +func (v *rawDeviceVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) { if err := v.verifyRawDeviceWhitelisted(v.opts.Path); err != nil { - return nil, err + return nil, nil, err } if err := verifyRawDeviceAccess(v.opts.Path); err != nil { - return nil, err + return nil, nil, err } return &libvirtxml.DomainDisk{ Device: "disk", Source: &libvirtxml.DomainDiskSource{Block: &libvirtxml.DomainDiskSourceBlock{Dev: v.opts.Path}}, Driver: &libvirtxml.DomainDiskDriver{Name: "qemu", Type: "raw"}, - }, nil + }, nil, nil } func init() { diff --git a/pkg/libvirttools/root_volumesource.go b/pkg/libvirttools/root_volumesource.go index 66a7f6173..99a55b370 100644 --- a/pkg/libvirttools/root_volumesource.go +++ b/pkg/libvirttools/root_volumesource.go @@ -83,21 +83,21 @@ func (v *rootVolume) createVolume() (virt.StorageVolume, error) { func (v *rootVolume) UUID() string { return "" } -func (v *rootVolume) Setup() (*libvirtxml.DomainDisk, error) { +func (v *rootVolume) Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) { vol, err := v.createVolume() if err != nil { - return nil, err + return nil, nil, err } volPath, err := vol.Path() if err != nil { - return nil, fmt.Errorf("error getting root volume path: %v", err) + return nil, nil, fmt.Errorf("error getting root volume path: %v", err) } return &libvirtxml.DomainDisk{ Device: "disk", Driver: &libvirtxml.DomainDiskDriver{Name: "qemu", Type: "qcow2"}, Source: &libvirtxml.DomainDiskSource{File: &libvirtxml.DomainDiskSourceFile{File: volPath}}, - }, nil + }, nil, nil } func (v *rootVolume) Teardown() error { diff --git a/pkg/libvirttools/root_volumesource_test.go b/pkg/libvirttools/root_volumesource_test.go index 99d477287..2951e7d9b 100644 --- a/pkg/libvirttools/root_volumesource_test.go +++ b/pkg/libvirttools/root_volumesource_test.go @@ -159,7 +159,7 @@ func TestRootVolumeLifeCycle(t *testing.T) { Image: "rootfs image name", }) - vol, err := rootVol.Setup() + vol, _, err := rootVol.Setup() if err != nil { t.Fatalf("Setup returned an error: %v", err) } @@ -218,3 +218,5 @@ func (vo fakeVolumeOwner) ImageManager() ImageManager { func (vo fakeVolumeOwner) RawDevices() []string { return nil } func (vo fakeVolumeOwner) KubeletRootDir() string { return "" } + +func (vo fakeVolumeOwner) VolumePoolName() string { return "" } diff --git a/pkg/libvirttools/virtualization.go b/pkg/libvirttools/virtualization.go index 71e702d8b..d26b1a198 100644 --- a/pkg/libvirttools/virtualization.go +++ b/pkg/libvirttools/virtualization.go @@ -330,12 +330,11 @@ func (v *VirtualizationTool) CreateContainer(config *types.VMConfig, netFdKey st } domainDef := settings.createDomain(config) - diskList, err := newDiskList(config, v.volumeSource, v) if err != nil { return "", err } - domainDef.Devices.Disks, err = diskList.setup() + domainDef.Devices.Disks, domainDef.Devices.Filesystems, err = diskList.setup() if err != nil { return "", err } @@ -564,10 +563,10 @@ func (v *VirtualizationTool) cleanupVolumes(containerID string) error { err = diskList.teardown() } + var errs []string if err != nil { glog.Errorf("Volume teardown failed for domain %q: %v", containerID, err) - return err - + errs = append(errs, err.Error()) } return nil @@ -833,6 +832,9 @@ func (v *VirtualizationTool) RawDevices() []string { return v.config.RawDevices // KubeletRootDir implements volumeOwner KubeletRootDir method func (v *VirtualizationTool) KubeletRootDir() string { return v.config.KubeletRootDir } +// KubeletRootDir implements volumeOwner VolumePoolName method +func (v *VirtualizationTool) VolumePoolName() string { return v.config.VolumePoolName } + func filterContainer(containerInfo *types.ContainerInfo, filter types.ContainerFilter) bool { if filter.Id != "" && containerInfo.Id != filter.Id { return false diff --git a/pkg/libvirttools/volumes.go b/pkg/libvirttools/volumes.go index 799624a64..764096c4c 100644 --- a/pkg/libvirttools/volumes.go +++ b/pkg/libvirttools/volumes.go @@ -34,6 +34,7 @@ type volumeOwner interface { ImageManager() ImageManager RawDevices() []string KubeletRootDir() string + VolumePoolName() string } // VMVolumeSource is a function that provides `VMVolume`s for VMs @@ -42,7 +43,7 @@ type VMVolumeSource func(config *types.VMConfig, owner volumeOwner) ([]VMVolume, // VMVolume describes a volume provider. type VMVolume interface { UUID() string - Setup() (*libvirtxml.DomainDisk, error) + Setup() (*libvirtxml.DomainDisk, *libvirtxml.DomainFilesystem, error) WriteImage(diskPathMap) error Teardown() error } diff --git a/pkg/tools/TestGenCommand__compat.out.yaml b/pkg/tools/TestGenCommand__compat.out.yaml index e606436ab..982baf87a 100755 --- a/pkg/tools/TestGenCommand__compat.out.yaml +++ b/pkg/tools/TestGenCommand__compat.out.yaml @@ -87,6 +87,7 @@ spec: - mountPath: /dev name: dev - mountPath: /var/lib/virtlet + mountPropagation: Bidirectional name: virtlet - mountPath: /var/lib/libvirt name: libvirt @@ -112,11 +113,14 @@ spec: resources: {} volumeMounts: - mountPath: /var/lib/virtlet + mountPropagation: HostToContainer name: virtlet - mountPath: /var/lib/libvirt name: libvirt - mountPath: /var/log/vms name: vms-log + - mountPath: /var/lib/kubelet/pods:shared + name: k8s-pods-dir - mountPath: /dev name: dev - mountPath: /lib/modules diff --git a/pkg/tools/TestGenCommand__compat_dev.out.yaml b/pkg/tools/TestGenCommand__compat_dev.out.yaml index 0381afe1c..6a53b9a47 100755 --- a/pkg/tools/TestGenCommand__compat_dev.out.yaml +++ b/pkg/tools/TestGenCommand__compat_dev.out.yaml @@ -89,6 +89,7 @@ spec: - mountPath: /dev name: dev - mountPath: /var/lib/virtlet + mountPropagation: Bidirectional name: virtlet - mountPath: /var/lib/libvirt name: libvirt @@ -116,11 +117,14 @@ spec: resources: {} volumeMounts: - mountPath: /var/lib/virtlet + mountPropagation: HostToContainer name: virtlet - mountPath: /var/lib/libvirt name: libvirt - mountPath: /var/log/vms name: vms-log + - mountPath: /var/lib/kubelet/pods:shared + name: k8s-pods-dir - mountPath: /dev name: dev - mountPath: /lib/modules diff --git a/pkg/tools/TestGenCommand__dev.out.yaml b/pkg/tools/TestGenCommand__dev.out.yaml index 613ca0708..414d2e07b 100755 --- a/pkg/tools/TestGenCommand__dev.out.yaml +++ b/pkg/tools/TestGenCommand__dev.out.yaml @@ -91,6 +91,7 @@ spec: - mountPath: /dev name: dev - mountPath: /var/lib/virtlet + mountPropagation: Bidirectional name: virtlet - mountPath: /var/lib/libvirt name: libvirt @@ -119,11 +120,15 @@ spec: resources: {} volumeMounts: - mountPath: /var/lib/virtlet + mountPropagation: HostToContainer name: virtlet - mountPath: /var/lib/libvirt name: libvirt - mountPath: /var/log/vms name: vms-log + - mountPath: /var/lib/kubelet/pods + mountPropagation: HostToContainer + name: k8s-pods-dir - mountPath: /dev name: dev - mountPath: /lib/modules diff --git a/pkg/tools/TestGenCommand__plain.out.yaml b/pkg/tools/TestGenCommand__plain.out.yaml index f20c78776..fefe1831c 100755 --- a/pkg/tools/TestGenCommand__plain.out.yaml +++ b/pkg/tools/TestGenCommand__plain.out.yaml @@ -89,6 +89,7 @@ spec: - mountPath: /dev name: dev - mountPath: /var/lib/virtlet + mountPropagation: Bidirectional name: virtlet - mountPath: /var/lib/libvirt name: libvirt @@ -115,11 +116,15 @@ spec: resources: {} volumeMounts: - mountPath: /var/lib/virtlet + mountPropagation: HostToContainer name: virtlet - mountPath: /var/lib/libvirt name: libvirt - mountPath: /var/log/vms name: vms-log + - mountPath: /var/lib/kubelet/pods + mountPropagation: HostToContainer + name: k8s-pods-dir - mountPath: /dev name: dev - mountPath: /lib/modules diff --git a/pkg/tools/TestGenCommand__tag.out.yaml b/pkg/tools/TestGenCommand__tag.out.yaml index 581992883..adc859d8a 100755 --- a/pkg/tools/TestGenCommand__tag.out.yaml +++ b/pkg/tools/TestGenCommand__tag.out.yaml @@ -89,6 +89,7 @@ spec: - mountPath: /dev name: dev - mountPath: /var/lib/virtlet + mountPropagation: Bidirectional name: virtlet - mountPath: /var/lib/libvirt name: libvirt @@ -115,11 +116,15 @@ spec: resources: {} volumeMounts: - mountPath: /var/lib/virtlet + mountPropagation: HostToContainer name: virtlet - mountPath: /var/lib/libvirt name: libvirt - mountPath: /var/log/vms name: vms-log + - mountPath: /var/lib/kubelet/pods + mountPropagation: HostToContainer + name: k8s-pods-dir - mountPath: /dev name: dev - mountPath: /lib/modules diff --git a/pkg/tools/bindata.go b/pkg/tools/bindata.go index 6f9d1f941..f4338156b 100644 --- a/pkg/tools/bindata.go +++ b/pkg/tools/bindata.go @@ -66,7 +66,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _deployDataVirtletDsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5b\x53\xe3\x38\x16\x7e\xe7\x57\x9c\x1a\xaa\xb6\xa7\x1f\x8c\xa1\x6b\x67\xbb\x27\xb5\xfb\x40\x93\x0c\x9b\x6a\x48\x52\xe1\xd2\xf3\x96\x92\xe5\x13\x47\x1b\x59\xf2\x4a\xb2\x21\xfb\xeb\xb7\x24\x5f\xe2\x5b\x42\xa0\x81\x9a\xce\x0b\x41\xd2\xf9\x74\xee\xe7\x48\x91\xe7\x79\x47\x24\x61\xf7\xa8\x34\x93\x62\x00\x24\x49\xb4\x9f\x9d\x1d\xad\x99\x08\x07\x30\x24\x18\x4b\x71\x83\xe6\x28\x46\x43\x42\x62\xc8\xe0\x08\x40\x90\x18\x07\x90\x31\x65\x38\x9a\xe2\x7f\x9d\x10\x8a\x03\x58\xa7\x01\x7a\x7a\xa3\x0d\xc6\x47\x3a\x41\x6a\x97\x6b\xe4\x48\x8d\x54\xf6\x3b\x40\x4c\x0c\x5d\x5d\x91\x00\xb9\xce\x07\x00\x54\x2a\x0c\x6b\x42\x1a\x8c\x13\x4e\x0c\x16\x34\xb5\xcd\xed\xa7\xcd\x80\xfd\xf0\x06\x64\x2f\x28\x40\xc9\x92\xfd\xac\xa4\x36\x13\x34\x0f\x52\xad\x07\x60\x54\x8a\xc5\x78\x28\xf4\x4c\x72\x46\x37\x03\xb8\xe0\xa9\x36\xa8\xfe\x60\x4a\x9b\xef\xcc\xac\xfe\x9d\x93\x14\x0b\x8f\x1d\xc4\x6c\x3c\x04\xa6\x1d\x00\x18\x09\xbf\x9e\x7d\x04\x14\x24\xe0\x08\xf7\xd7\xda\x8e\xe8\x54\x65\x2c\xc3\x92\x0f\xa0\x52\x18\xc2\x04\x2a\x50\xa8\x0d\x51\x5b\xb8\x5f\x8d\x84\x00\x81\xae\x90\xae\x31\xfc\x08\x44\x84\xf0\xeb\xa7\x8f\x16\xa4\x80\x34\x2b\x84\x54\x23\xc8\x25\x08\x8d\xc2\xa0\x02\x26\x80\x09\x56\x83\xad\x89\x37\x1b\x0f\x1b\xa2\x1d\x43\x20\xa5\xd1\x46\x91\x04\x12\x25\x29\x86\xa9\x42\x10\x88\xa1\xe3\x94\x2a\x24\x06\x81\x58\xac\x25\x8b\x62\x92\x58\xf4\x9a\x49\xb7\x96\x2e\x00\x35\xaa\x8c\x51\x3c\xa7\x54\xa6\xc2\x4c\x1a\x66\xa9\xf6\x94\x82\x6f\xac\x39\xe0\xbe\xd0\x40\x22\x43\x0d\x52\x38\x69\x84\x0c\x51\xc3\x03\x33\x2b\xc0\x47\xa3\xc8\x3c\x37\xdb\xbf\x4a\x6d\x39\xb3\x16\x50\x64\xb9\xb4\xa2\x6e\xb6\x46\xb6\xd4\xe7\x9d\x51\x00\x85\xff\x4d\x99\xc2\x70\x98\x2a\x26\xa2\x1b\xba\xc2\x30\xe5\x4c\x44\xe3\x48\xc8\x6a\x78\xf4\x88\x34\x35\xd6\xeb\x6b\x94\x39\xe6\x4d\xe1\xb2\xb7\xa8\x62\xdd\x9c\xf6\x72\x0f\x1e\x3d\x26\x0a\xb5\x8d\x99\xd6\xbc\x5d\xb1\xc6\xcd\xa0\x21\x4e\x6b\x05\x80\x4c\x50\x11\x1b\x13\x30\x16\x9d\xc9\x8c\xf0\x14\x3b\xb0\x16\xb8\xa5\x5b\x2b\xf7\x45\x69\xf7\x8a\xe0\x18\x6e\x57\xd8\x72\x0a\xa0\x32\x61\xa8\x4b\x80\x0f\x1a\x96\x1c\x1f\x33\xc9\xd3\x18\x21\x54\x2c\xab\xfc\xe6\xd8\x7a\x82\xb5\x4c\x88\x4b\x92\x72\xe3\xec\xef\xac\xc6\xd3\x88\x09\x08\x99\x72\x8e\x89\x42\xa7\x0a\x35\x98\x15\xd9\x7a\xb0\xa3\x63\xca\xe9\xce\x6e\x67\x5d\x0b\x43\x08\x36\xc0\x59\x60\xf7\x86\xbf\x55\x71\x80\x8f\x4c\x9b\xd2\x0d\xac\xb7\x1e\x95\x52\xe6\xe1\x9d\x28\x4c\x88\x42\xcf\xda\xa3\x52\x05\x8b\x49\x84\x03\x88\x99\x22\xc2\x30\xed\x37\x73\x40\x31\x3f\x4b\x39\x2f\x43\x78\xbc\x9c\x48\x33\x53\x68\xa3\xa5\x5a\x45\x65\x1c\x13\x11\x6e\x35\xec\x81\x5f\xdf\xee\x44\xaf\xaa\xa9\x5c\x47\xd7\xd6\xbf\x75\x9d\x20\x67\x72\xfd\x45\x7b\x5b\x4d\x7a\xb9\x8e\xb4\x17\x32\x55\xb3\x5e\x6c\x89\x67\xc4\xac\x06\xe0\x17\xda\xf4\x9a\x04\x1d\x5c\x95\x8a\x0e\x80\x92\x09\x89\x88\x73\x58\xf8\xca\x72\x35\x33\x29\x08\xdf\xb1\x55\x1d\xa3\xc4\x0d\x25\x5d\xa3\xd2\x92\xae\x77\x10\x65\x44\x59\x42\x3f\x5f\x78\xd2\x58\x59\x82\x70\x19\xed\xa0\xb6\x66\xac\xcf\x1e\xc3\x52\xaa\xdc\x55\x98\x88\x9c\xaf\xe4\x5b\x70\x16\xf8\x85\x4b\xf8\xce\x66\x3a\xf7\x07\x97\x17\x1a\x16\x2f\x37\xcd\x88\xf2\x38\x0b\xf6\x6c\xec\xb5\x97\x54\x42\x63\xb6\x83\xac\x3e\xe3\x75\xf4\x60\x99\x6c\x3b\x58\x7f\xf1\xb1\x99\x90\xa6\x8a\x99\x8d\x0d\x47\x7c\x34\xf5\xe0\x4d\x14\xcb\x18\xc7\x08\xc3\x46\x32\x06\x40\x91\x75\x3d\xea\xdb\xdd\xd7\xd1\x62\x32\x1d\x8e\x16\x93\xf3\xeb\x51\x0d\xc6\x65\x85\x3f\x94\x8c\x9b\x89\x61\xc9\x90\x87\x73\x5c\xb6\xd3\x45\xbd\xa8\x67\x67\xad\x49\x47\x94\x4b\x6a\x4b\xe2\x89\xd5\xb8\xcd\xde\x1d\x6e\xee\xc7\xf3\xdb\xab\xd1\xed\x62\x38\xbe\x39\xff\x7a\x35\x5a\x7c\xbb\xbf\x7e\x9a\xa5\xbc\x7c\x5c\x93\xe4\x1b\x6e\x7a\x38\x6b\x28\xd0\xcb\x17\xb7\x96\xb8\x04\x1a\x32\x6d\x8b\xde\x62\x9d\xc5\xad\x69\x99\xe4\x8e\xdf\xd2\x67\x9b\xe9\x9b\xf9\x78\x7a\xbf\xb8\xb9\x9b\xcd\xa6\xf3\xdb\x77\x63\x5b\x2b\x26\xb3\x85\x4e\x93\x44\x2a\xf3\x32\xc6\x87\xd3\xef\x93\xab\xe9\xf9\x70\x31\x9b\x4f\x6f\xa7\x17\xd3\xab\xf7\xd3\xb9\x7c\x10\x5c\x92\x70\x91\x28\x69\x24\x95\xfc\x65\x02\x5c\x4d\x2f\xaf\x46\xf7\xa3\xf7\xe3\x9b\xcb\x88\x63\x86\x2f\x64\xf7\xe2\xfc\x6a\x7c\x31\x5d\xdc\xdc\x7d\x9d\x8c\xde\xcf\x51\x28\xe1\x8c\x4a\x4f\xa7\x81\xc0\x67\x3a\xca\xf8\xfa\xfc\x72\xb4\x98\x8f\x2e\x47\x7f\xce\x16\xb7\xf3\xf3\xc9\xcd\xd5\xf9\xed\x78\x3a\x79\x37\xde\x5d\xce\x5e\x28\x8c\xf0\x31\x59\x18\x45\x84\xe6\xae\x32\xbd\x4c\xff\xf3\xf3\xef\x8b\xe1\xe8\x7e\x7c\x31\xba\x79\x37\x09\x14\x79\x58\x84\x68\xbb\x57\xfd\xc2\x20\x2d\x52\xe2\xd5\xf4\xf2\x72\x3c\xb9\x7c\xf7\xb4\xc8\x65\x14\x31\xd1\x5e\x72\xa8\xc7\xcf\xee\x16\xd7\xd3\xe1\x3b\x46\x28\x4d\x52\x2f\x96\xe1\x4b\x43\x34\xf7\xf8\x9a\xab\xdf\x2c\x86\xe3\x79\x9b\xfb\x01\xf8\x68\x68\x59\xb4\x8b\xce\xa2\xec\x96\x69\xa7\x53\xae\x1a\x9a\xbc\x13\x39\xb8\xcb\x3c\x86\xb1\x00\x4a\x34\xc2\x83\x6d\xb4\xff\x83\xd4\x00\x97\x94\xf0\xaa\xb9\x75\x08\x76\xf6\x81\x08\x63\x3b\x6a\x7b\x6a\x63\x06\x84\x34\x20\x97\x4b\x46\x19\xe1\x7c\x03\x24\x23\x8c\xbb\x93\x9d\x14\xf8\x0a\x4d\x6c\x21\xc8\x21\xfd\x6b\xbd\xd9\xd1\x1b\xed\x2f\xb5\x4f\x23\x25\xd3\xa4\xd3\xea\xb4\x86\x9b\xa4\xb6\x47\x8a\x65\x98\xf2\x46\x18\xe5\x84\xdd\x71\x85\x24\x9c\x0a\xbe\xe9\x18\xbb\x0e\x69\xcf\xa8\x1d\xac\xd6\xe0\x41\x40\xcd\x26\xfa\x47\x1a\xeb\x1f\xeb\x0d\xfb\xa9\xdb\x4e\x07\x3b\x9c\xb1\x4b\x6d\xfb\xf3\x27\xa8\x3d\xdb\xb8\xa3\xd1\x35\x97\xb5\xc7\x2c\x2e\x23\x77\x80\x63\xd5\xd1\x6c\x85\x0a\x21\x40\x4a\xdc\xb5\x82\x59\xa1\x7a\x60\x1a\xab\xe3\xda\x03\xe3\x1c\x12\x25\xc3\x94\x22\xa0\x52\x52\xd5\x21\x39\x5b\x23\x98\x15\xab\x39\xd6\x31\xdc\x15\x57\x15\xd2\x9e\xe0\xbc\xe2\x4e\x81\xae\x88\x0a\x31\x83\x25\xe3\x08\x1f\x72\x1d\xc8\xc8\xcf\x62\xed\x93\x65\xf8\xf9\xb7\x20\x08\xbc\x2f\xf8\xfb\x67\xef\xec\x0c\x3f\x7b\xbf\xff\xf6\x8f\x33\xef\xf4\xd3\xdf\x3f\x9d\x12\x7a\x7a\x7a\x7a\xfa\xc9\xa7\x4c\x29\xa9\xbd\x2c\x5e\x9c\x9e\x70\x19\x7d\x18\xc0\x44\x82\x4e\xe9\x2a\x47\x94\xaa\x3a\x76\x6e\xba\x27\x87\x58\x7b\xbb\x8f\x2c\x35\x56\xba\x07\x9d\x42\x99\x4f\x53\x77\x8d\xf6\x9c\xa3\xc7\x4b\x0e\x0f\x36\x02\x98\x40\xad\x67\x4a\x06\x58\x27\xc1\xc7\xed\x25\x57\xfe\xe9\xa4\x8a\x9c\x45\x3f\x60\xc2\xaf\xa5\x8a\x7c\xd4\xa3\xad\x01\x2d\x29\x31\xe0\xc1\xdd\x64\xfc\xe7\xa0\xed\x80\x7e\xdd\xe1\x3c\x25\xe1\x9f\x56\x32\x5f\xa4\x9c\xb7\x92\x6c\xef\x51\xfd\xaf\x9e\x64\x0f\xc9\x9e\xaf\x97\x66\x8e\xf3\xe4\xe7\xee\x57\xea\x99\x15\x88\xc2\xea\x4e\x0b\x82\x0d\xe8\x34\x41\x15\x33\xf1\x93\x24\xe5\xf7\x3b\x86\xff\x15\x52\x6d\x13\x25\xd5\x8e\x07\x1b\x94\xee\xf2\x47\x09\x34\xa8\xab\x7b\xa0\xe2\x02\xc8\xcf\x1d\xcd\xb7\xcb\x3a\x1b\x1d\x70\xc9\xd4\x2f\x77\xb1\x89\x9f\xc8\xb0\xeb\x08\x16\xd5\x4e\xf4\x5e\x56\x1d\x52\x12\x5f\x9e\x5d\xeb\x2b\x7a\xfa\xb5\x36\xa7\x6e\xd8\xb3\xdf\xbd\xda\x51\xa3\x9b\xae\x9d\x34\x4f\xf3\xd2\xd0\xc6\x71\x59\x08\x97\xae\x86\x90\x48\x48\x6d\x18\x85\x24\x55\x89\xd4\xf8\x9a\x35\xe1\x67\xca\xf2\xd6\xef\x0b\xab\xb8\x3b\xc0\x3d\x69\xbd\x66\xd9\xb7\xbb\x9d\xcd\x62\xfd\xdc\xa6\xf6\xbd\x53\xc7\x8f\x36\x1b\x87\xa6\xc7\x5d\x69\x7c\x7f\x01\xc8\x35\x56\xfb\x8d\xc0\xa2\xd6\xfa\x40\xeb\xfe\x2b\xa9\x8d\x3d\x17\x43\x7e\x2e\x06\x42\x29\x6a\x5d\xd9\xdb\xfd\xa2\x64\xf1\xeb\x8e\xdb\xe5\xb0\x2d\xcd\x5e\xc2\xfe\x93\x47\xcf\xb9\x63\x2f\x4a\x5f\xbd\xeb\x53\xd3\x5e\x90\x46\x31\xeb\xd4\xb7\xbd\xa4\xf5\x6a\xdf\xae\xff\xc7\x70\x3b\x1d\x4e\x07\x10\x4a\xf1\xc1\x80\x6d\x83\xa9\x0c\xb1\xb8\x88\x87\xbc\x76\xb8\xbe\xc6\xe6\x34\xd7\x8e\x6f\x09\x57\x4c\xe7\x1d\x78\x51\xfb\xe1\x62\x3e\xb6\xdd\xf8\xe3\x06\x98\xd0\x86\xf0\x3c\x13\xda\xd6\xa7\xbe\x21\x13\xb9\x29\x9d\x47\x6c\x7f\xb3\x3b\x39\x44\x94\x7d\xf7\xff\x3b\x7e\x42\x78\x12\xaf\x2f\x0a\xfb\x62\xf0\x20\xa0\x76\xe0\xf5\x85\xe3\xd3\x40\xb5\x08\x6d\xff\xa6\xb1\x97\xf8\x07\xaa\xf9\x81\xb5\xfc\x20\x25\xf4\x16\xf6\x9d\x65\xfd\x10\xc8\xb6\x61\x1a\x3f\xa5\x1c\xa2\xcf\xaa\x88\xd7\x73\x5b\x5f\x4e\x3c\x08\x6c\xaf\x95\x9f\x03\xd6\xd7\xc0\xed\x6b\xdf\x0e\xe2\xae\x47\xed\xad\xde\xc3\xdb\x5e\x9b\x0d\x76\x55\x1e\x2f\xef\x6b\x7a\x5b\x9a\xfd\x8d\x4f\xfb\xd1\x86\x0a\x08\x3d\x21\xa9\x59\x49\xc5\xfe\xe7\xd6\x9c\xac\xbf\xe8\x13\x26\xfd\xec\x2c\x40\x43\xca\xe7\x1c\xc5\x7b\x86\xb9\xe4\xf8\x95\x89\x90\x89\x68\xcf\xbb\x0e\x25\x39\x16\xf7\x7d\x24\x61\x97\x36\x17\xef\xd9\xe9\x08\xa0\xb3\x47\x07\x52\xa7\x81\x3d\xc6\xe9\xc1\x91\x57\xac\xbe\x69\x3c\x20\x38\xfc\x6d\x89\xd5\x40\x77\xbf\xe7\xe9\xe4\x05\x4f\x5a\x94\x2d\x26\x76\xbd\x57\xe9\xa4\x28\xa9\x1e\xfc\xf2\x8b\xfb\xa2\x50\xcb\x54\x51\xac\xc6\xab\xc7\x14\xba\x18\x70\x4f\x1e\xdc\xf7\x0c\x55\xb0\x5d\xe7\x6e\x4a\x8a\x7f\x22\x34\xaf\x61\xe5\x1e\x19\x2b\x76\x3c\xdb\x5b\xa2\x2a\x65\x6a\x49\x54\xc8\xd3\x90\xa6\x25\x4b\xc5\x7d\xce\xae\xfd\xcb\x99\xce\xbf\x3c\x10\x43\x57\x6f\x24\x41\x19\x3e\xa9\x46\x65\x67\x7e\x58\x10\xcf\xb6\xe6\x2a\xcf\x01\x2d\xa1\xde\x34\xd2\xca\xaa\x61\x1d\xc2\x0b\x8a\x65\xaf\x18\x76\x1d\x53\xd7\xe3\xef\x39\xe0\x97\x45\x23\x96\xc3\xe6\xb1\x30\xc8\xdd\xf8\x6d\x53\x51\xbc\x35\xf2\x1b\xe8\x67\x97\x23\xfd\x24\x69\xca\xa3\x2a\xdc\xed\xf4\x24\x61\xf8\x68\x50\xb8\x27\x4a\x05\x66\x5f\x20\xa4\xda\xc8\xb8\x1c\x0c\xd1\xbd\xa5\x2a\x4a\x51\x2d\x16\x8a\xe4\xd4\xdd\xa6\x3c\x24\xae\xbf\xe8\x1e\xf4\x62\xd6\xd5\xb1\x98\x24\x09\x13\x91\xae\x4f\x54\x1e\x5a\xce\xd4\xb6\xac\x72\xc9\x9b\xc7\x61\x43\x9f\xaf\xef\x5e\x16\xf6\x75\x5d\xaa\xf5\xb6\xa3\x17\xf0\x05\xd5\xed\xff\x01\x00\x00\xff\xff\xf1\x5a\x5a\xd7\x0c\x2a\x00\x00") +var _deployDataVirtletDsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x49\x73\xe3\xb8\x15\xbe\xfb\x57\xbc\x1a\x57\xa5\xa7\x0f\xb4\xec\xae\x4c\xba\x47\x95\x1c\x6c\x4b\xe3\x51\xb5\x2d\xa9\x24\xdb\x3d\x37\x15\x08\x3e\x51\x88\x40\x80\x01\x40\xda\xca\xaf\x4f\x01\x5c\xc4\x4d\xb2\xbc\x56\xda\x97\x56\x63\xf9\xf0\xf6\x05\x84\xe7\x79\x47\x24\x66\xf7\xa8\x34\x93\xa2\x0f\x24\x8e\x75\x2f\x3d\x3b\x5a\x33\x11\xf4\x61\x40\x30\x92\x62\x8e\xe6\x28\x42\x43\x02\x62\x48\xff\x08\x40\x90\x08\xfb\x90\x32\x65\x38\x9a\xfc\xff\x3a\x26\x14\xfb\xb0\x4e\x7c\xf4\xf4\x46\x1b\x8c\x8e\x74\x8c\xd4\x2e\xd7\xc8\x91\x1a\xa9\xec\x6f\x80\x88\x18\xba\xba\x26\x3e\x72\x9d\x0d\x00\xa8\x44\x18\x56\x87\x34\x18\xc5\x9c\x18\xcc\xf7\x54\x0e\xb7\x7f\x4d\x02\xec\x1f\xaf\x41\x76\x82\x02\x14\x24\xd9\xbf\x95\xd4\x66\x8c\xe6\x41\xaa\x75\x1f\x8c\x4a\x30\x1f\x0f\x84\x9e\x4a\xce\xe8\xa6\x0f\x97\x3c\xd1\x06\xd5\x1f\x4c\x69\xf3\x83\x99\xd5\x9f\xd9\x96\x7c\xe1\xb1\x83\x98\x8e\x06\xc0\xb4\x03\x00\x23\xe1\xd7\xb3\xcf\x80\x82\xf8\x1c\xe1\xfe\x46\xdb\x11\x9d\xa8\x94\xa5\x58\xd0\x01\x54\x0a\x43\x98\x40\x05\x0a\xb5\x21\x6a\x0b\xf7\xab\x91\xe0\x23\xd0\x15\xd2\x35\x06\x9f\x81\x88\x00\x7e\xfd\xf2\xd9\x82\xe4\x90\x66\x85\x90\x68\x04\xb9\x04\xa1\x51\x18\x54\xc0\x04\x30\xc1\x2a\xb0\x15\xf6\xa6\xa3\x41\x8d\xb5\x63\xf0\xa5\x34\xda\x28\x12\x43\xac\x24\xc5\x20\x51\x08\x02\x31\x70\x94\x52\x85\xc4\x20\x10\x8b\xb5\x64\x61\x44\x62\x8b\x5e\x51\xe9\x56\xd3\x39\xa0\x46\x95\x32\x8a\xe7\x94\xca\x44\x98\x71\x4d\x2d\xe5\x99\x52\xf0\x8d\x55\x07\xdc\xe7\x12\x88\x65\xa0\x41\x0a\xc7\x8d\x90\x01\x6a\x78\x60\x66\x05\xf8\x68\x14\x99\x65\x6a\xfb\x57\x21\x2d\xa7\xd6\x1c\x8a\x2c\x97\x96\xd5\xcd\x56\xc9\x76\xf7\x79\x6b\x14\x40\xe1\x7f\x12\xa6\x30\x18\x24\x8a\x89\x70\x4e\x57\x18\x24\x9c\x89\x70\x14\x0a\x59\x0e\x0f\x1f\x91\x26\xc6\x5a\x7d\x65\x67\x86\x39\xcf\x4d\xf6\x16\x55\xa4\xeb\xd3\x5e\x66\xc1\xc3\xc7\x58\xa1\xb6\x3e\xd3\x98\xb7\x2b\xd6\xb8\xe9\xd7\xd8\x69\xac\x00\x90\x31\x2a\x62\x7d\x02\x46\xa2\x35\x99\x12\x9e\x60\x0b\xd6\x02\x37\x64\x6b\xf9\xbe\x2c\xf4\x5e\x6e\x38\x86\xdb\x15\x36\x8c\x02\xa8\x8c\x19\xea\x02\xe0\x93\x86\x25\xc7\xc7\x54\xf2\x24\x42\x08\x14\x4b\x4b\xbb\x39\xb6\x96\x60\x35\x13\xe0\x92\x24\xdc\x38\xfd\x3b\xad\xf1\x24\x64\x02\x02\xa6\x9c\x61\xa2\xd0\x89\x42\x0d\x66\x45\xb6\x16\xec\xf6\x31\xe5\x64\x67\x8f\xb3\xa6\x85\x01\xf8\x1b\xe0\xcc\xb7\x67\xc3\xdf\x4a\x3f\xc0\x47\xa6\x4d\x61\x06\xd6\x5a\x8f\x0a\x2e\x33\xf7\x8e\x15\xc6\x44\xa1\x67\xf5\x51\x8a\x82\x45\x24\xc4\x3e\x44\x4c\x11\x61\x98\xee\xd5\x63\x40\x3e\x3f\x4d\x38\x2f\x5c\x78\xb4\x1c\x4b\x33\x55\x68\xbd\xa5\x5c\x45\x65\x14\x11\x11\x6c\x25\xec\x41\xaf\x7a\xdc\x89\x5e\x95\x53\x99\x8c\x6e\xac\x7d\xeb\xea\x86\x8c\xc8\xf5\x37\xed\x6d\x25\xe9\x65\x32\xd2\x5e\xc0\x54\x45\x7b\x91\xdd\x3c\x25\x66\xd5\x87\x5e\x2e\x4d\xaf\xbe\xa1\x85\xab\x12\xd1\x02\x50\x32\x26\x21\x71\x06\x0b\x17\x2c\x13\x33\x93\x82\xf0\x1d\x47\x55\x31\x0a\xdc\x40\xd2\x35\x2a\x2d\xe9\x7a\xc7\xa6\x94\x28\xbb\xb1\x97\x2d\x3c\xa9\xad\x2c\x40\xb8\x0c\x77\xec\xb6\x6a\xac\xce\x1e\xc3\x52\xaa\xcc\x54\x98\x08\x9d\xad\x64\x47\x70\xe6\xf7\x72\x93\xe8\x39\x9d\xe9\xcc\x1e\x5c\x5c\xa8\x69\xbc\x38\x34\x25\xca\xe3\xcc\xdf\x73\xb0\xd7\x5c\x52\x32\x8d\xe9\x8e\x6d\xd5\x19\xaf\x25\x07\x4b\x64\xd3\xc0\xba\x93\x8f\x8d\x84\x34\x51\xcc\x6c\xac\x3b\xe2\xa3\xa9\x3a\x6f\xac\x58\xca\x38\x86\x18\xd4\x82\x31\x00\x8a\xb4\x6d\x51\xdf\xef\x2e\x86\x8b\xf1\x64\x30\x5c\x8c\xcf\x6f\x86\x15\x18\x17\x15\xfe\x50\x32\xaa\x07\x86\x25\x43\x1e\xcc\x70\xd9\x0c\x17\xd5\xa4\x9e\x9e\x35\x26\xdd\xa6\x8c\x53\x9b\x12\x4f\xac\xc4\x6d\xf4\x6e\x51\x73\x3f\x9a\xdd\x5e\x0f\x6f\x17\x83\xd1\xfc\xfc\xe2\x7a\xb8\xf8\x7e\x7f\xf3\x34\x49\x59\xfa\xb8\x21\xf1\x77\xdc\x74\x50\x56\x13\xa0\x97\x2d\x6e\x2c\x71\x01\x34\x60\xda\x26\xbd\xc5\x3a\x8d\x1a\xd3\x32\xce\x0c\xbf\x21\xcf\x26\xd1\xf3\xd9\x68\x72\xbf\x98\xdf\x4d\xa7\x93\xd9\xed\x87\x91\xad\x15\x93\xe9\x42\x27\x71\x2c\x95\x79\x19\xe1\x83\xc9\x8f\xf1\xf5\xe4\x7c\xb0\x98\xce\x26\xb7\x93\xcb\xc9\xf5\xc7\xc9\x5c\x3e\x08\x2e\x49\xb0\x88\x95\x34\x92\x4a\xfe\x32\x06\xae\x27\x57\xd7\xc3\xfb\xe1\xc7\xd1\xcd\x65\xc8\x31\xc5\x17\x92\x7b\x79\x7e\x3d\xba\x9c\x2c\xe6\x77\x17\xe3\xe1\xc7\x19\x0a\x25\x9c\x51\xe9\xe9\xc4\x17\xf8\x4c\x43\x19\xdd\x9c\x5f\x0d\x17\xb3\xe1\xd5\xf0\xaf\xe9\xe2\x76\x76\x3e\x9e\x5f\x9f\xdf\x8e\x26\xe3\x0f\xa3\xdd\xc5\xec\x85\xc2\x10\x1f\xe3\x85\x51\x44\x68\xee\x32\xd3\xcb\xe4\x3f\x3b\xff\xb1\x18\x0c\xef\x47\x97\xc3\xf9\x87\x71\xa0\xc8\xc3\x22\x40\x5b\xbd\xea\x17\x3a\x69\x1e\x12\xaf\x27\x57\x57\xa3\xf1\xd5\x87\x87\x45\x2e\xc3\x90\x89\xe6\x92\x43\x2d\x7e\x7a\xb7\xb8\x99\x0c\x3e\xd0\x43\x69\x9c\x78\x91\x0c\x5e\xea\xa2\x99\xc5\x57\x4c\x7d\xbe\x18\x8c\x66\x4d\xea\xfb\xd0\x43\x43\x8b\xa4\x9d\x57\x16\x45\xb5\x4c\x5b\x95\x72\x59\xd0\x64\x95\xc8\xc1\x55\xe6\x31\x8c\x04\x50\xa2\x11\x1e\x6c\xa1\xfd\x6f\xa4\x06\xb8\xa4\x84\x97\xc5\xad\x43\xb0\xb3\x0f\x44\x18\x5b\x51\xdb\xae\x8d\x19\x10\xd2\x80\x5c\x2e\x19\x65\x84\xf3\x0d\x90\x94\x30\xee\x3a\x3b\x29\xf0\x0d\x8a\xd8\x9c\x91\x43\xea\xd7\x6a\xb1\xa3\x37\xba\xb7\xd4\x3d\x1a\x2a\x99\xc4\xad\x52\xa7\x31\x5c\xdf\x6a\x6b\xa4\x48\x06\x09\xaf\xb9\x51\xb6\xb1\x3d\xae\x90\x04\x13\xc1\x37\x2d\x65\x57\x21\x6d\x8f\xda\xc2\x6a\x0c\x1e\x04\x54\x2f\xa2\x5f\x53\x58\xbf\xae\x36\xec\xde\xdd\x34\x3a\xd8\x61\x8c\xed\xdd\xb6\x3e\x7f\x62\xb7\x67\x0b\x77\x34\xba\x62\xb2\xb6\xcd\xe2\x32\x74\x0d\x1c\x2b\x5b\xb3\x15\x2a\x04\x1f\x29\x71\xd7\x0a\x66\x85\xea\x81\x69\x2c\xdb\xb5\x07\xc6\x39\xc4\x4a\x06\x09\x45\x40\xa5\xa4\xaa\x42\x72\xb6\x46\x30\x2b\x56\x31\xac\x63\xb8\xcb\xaf\x2a\xa4\xed\xe0\xbc\xfc\x4e\x81\xae\x88\x0a\x30\x85\x25\xe3\x08\x9f\x32\x19\xc8\xb0\x97\x46\xba\x47\x96\xc1\xd7\xdf\x7c\xdf\xf7\xbe\xe1\xef\x5f\xbd\xb3\x33\xfc\xea\xfd\xfe\xdb\x3f\xce\xbc\xd3\x2f\x7f\xff\x72\x4a\xe8\xe9\xe9\xe9\xe9\x97\x1e\x65\x4a\x49\xed\xa5\xd1\xe2\xf4\x84\xcb\xf0\x53\x1f\xc6\x12\x74\x42\x57\x19\xa2\x54\x65\xdb\xb9\x69\x77\x0e\x91\xf6\x76\xb7\x2c\x15\x52\xda\x8d\x4e\x2e\xcc\xa7\x77\xb7\x95\xf6\x9c\xd6\xe3\x25\xcd\x83\xf5\x00\x26\x50\xeb\xa9\x92\x3e\x56\xb7\xe0\xe3\xf6\x92\x2b\xfb\x6b\x85\x8a\x8c\xc4\x9e\xcf\x44\xaf\x12\x2a\xb2\x51\x8f\x36\x06\xb4\xa4\xc4\x80\x07\x77\xe3\xd1\x5f\xfd\xa6\x01\xf6\xaa\x06\xe7\x29\x09\xff\xb4\x9c\xf5\x44\xc2\x79\x23\xc8\x76\xb6\xea\xff\xef\x41\xf6\x90\xe8\xf9\x76\x61\xe6\x38\x0b\x7e\xee\x7e\xa5\x1a\x59\x81\x28\x2c\xef\xb4\xc0\xdf\x80\x4e\x62\x54\x11\x13\x3f\x49\x50\xfe\xb8\x36\xfc\xb5\x21\xfd\x63\x83\x72\x1d\x25\xd1\x8e\x06\xeb\xbe\xee\x9a\x48\x09\x34\xa8\xcb\x1b\xa3\xfc\xaa\xa8\x97\x99\x64\xcf\x2e\x6b\x1d\x74\xc0\x75\x54\x37\xdf\xf9\x21\xbd\x58\x06\x6d\x93\xb1\xa8\x76\xa2\xf3\x5a\xeb\x10\x49\xbf\x3c\x0e\x57\x57\x74\x54\x76\x4d\x4a\xdd\xb0\x67\x7f\x7b\x95\xa6\xa4\x1d\xd8\x1d\x37\x4f\xd3\x52\x93\xc6\x71\x91\x32\x97\x2e\xdb\x90\x50\x48\x6d\x18\x85\x38\x51\xb1\xd4\xf8\x96\xd9\xe3\x67\xca\x07\xd6\xee\x73\xad\xb8\xdb\xc2\x3d\x09\xa0\xa2\xd9\xf7\xbb\xc7\x4d\x23\xfd\xdc\xf2\xf7\x2d\x82\xcc\x9f\x52\x9b\x5b\x79\xd9\xf8\x08\xf3\xfa\x30\xf3\x36\xae\xf3\xb6\x6e\xbe\x9b\xd7\xe7\x05\xfa\x5d\x09\x69\x7f\x2a\xcb\x34\x5a\xf9\xda\x61\x51\x2b\x15\xad\x75\xcf\x95\xd4\xc6\x76\xf8\x90\x75\xf8\x40\x28\x45\xad\x4b\x7b\x74\xdf\xc6\x2c\x7e\xd5\xb1\xda\x14\x36\xb9\xd9\xbb\xb1\xbb\x87\xea\xe8\xa0\xf6\xa2\x74\x65\xee\x2e\x31\xed\x05\xa9\xa5\xe5\x56\xa6\xde\xbb\xb5\x5a\xb7\x34\x2b\x99\x63\xb8\x9d\x0c\x26\x7d\x08\xa4\xf8\x64\xc0\x16\xf4\x54\x06\x98\x7f\x52\x80\x2c\xb7\xb9\x0a\xcd\x5a\x89\x6b\x2c\xb6\x1b\x57\x4c\x67\xbd\x44\x5e\xc5\xc0\xe5\x6c\x64\xfb\x8a\xc7\x0d\x30\xa1\x0d\xe1\x59\xa4\xb6\x45\x5c\xf5\x40\x26\x32\x55\x3a\x8b\xd8\x7e\x7d\x3c\x39\x84\x95\x7d\x5f\x32\x76\x7c\x0c\x79\x12\xaf\x2b\x4a\x74\xc5\x88\x83\x80\x9a\xce\xde\x15\x02\x9e\x06\xaa\x44\x85\xe6\xd7\x99\xbd\x9b\x5f\x51\x6d\x1c\x58\x6b\x1c\x24\x84\xce\x88\xb4\x33\x1e\x1d\x02\xd9\x54\x4c\xed\xa3\xd0\x21\xf2\x2c\x8b\x8c\x6a\x3c\xed\x8a\xc3\x07\x81\xed\xd5\xf2\x73\xc0\xba\x0a\xcc\x7d\xe5\xe5\x41\xd4\x75\x88\xbd\x51\x1b\x79\xdb\x0b\xc0\xfe\xae\xcc\xe8\x65\x75\x57\x67\xc9\xb5\xbf\x30\x6b\x3e\x3f\x51\x3e\xa1\x27\x24\x31\x2b\xa9\xd8\x7f\xdd\x9a\x93\xf5\x37\x7d\xc2\x64\x2f\x3d\xf3\xd1\x90\xe2\x61\x4a\xfe\x32\x63\x26\x39\x5e\x30\x11\x30\x11\xee\x79\xa1\xa2\x24\xc7\xfc\xe6\x92\xc4\xec\xca\xc6\xe2\x3d\x27\x1d\x01\xb4\xce\x68\x41\xea\xc4\xb7\x0d\xa9\xee\x1f\x79\xf9\xea\x79\xed\x29\xc4\xe1\xaf\x64\xac\x04\xda\xe7\x3d\x4f\x26\x2f\x78\x9c\xa3\x6c\x32\xb1\xeb\xbd\x52\x26\x79\x4a\xf5\xe0\x97\x5f\xdc\x0f\x85\x5a\x26\x8a\x62\x39\x5e\x3e\x0b\xd1\xf9\x80\x7b\xbc\xe1\x7e\xa7\xa8\xfc\xed\x3a\x77\xe7\x93\xff\x27\x44\xf3\x16\x5a\xee\xe0\xb1\x24\xc7\xb3\xb5\x2f\xaa\x82\xa7\x06\x47\x39\x3f\x35\x6e\x1a\xbc\x94\xd4\x67\xe4\xda\x7f\x39\xd3\xd9\x8f\x07\x62\xe8\xea\x9d\x38\x28\xdc\x27\xd1\xa8\xec\xcc\xab\x19\xf1\x6c\xeb\xa0\xb2\x18\xd0\x60\xea\x5d\x3d\xad\xc8\x1a\xd6\x20\x3c\x3f\x5f\xf6\x86\x6e\xd7\x52\x75\xd5\xff\x9e\x03\x7e\x95\x17\x62\x19\x6c\xe6\x0b\xfd\xcc\x8c\xdf\x37\x14\x45\x5b\x25\xbf\x83\x7c\x76\x19\xd2\x4f\x12\xa6\x3c\xaa\x82\xdd\x46\x4f\x62\x86\x8f\x06\x85\x7b\x6c\x95\x63\x76\x39\x42\xa2\x8d\x8c\x8a\xc1\x00\xdd\xab\xb0\x3c\x15\x55\x7c\x21\x0f\x4e\xed\x63\x8a\x26\x76\xfd\x4d\x77\xa0\xe7\xb3\x2e\x8f\x45\x24\x8e\x99\x08\x75\x75\xa2\xb4\xd0\x62\xa6\x72\x64\x19\x4b\xde\xdd\x0f\x6b\xf2\x7c\x7b\xf3\xb2\xb0\x6f\x6b\x52\x8d\x57\x2a\x9d\x80\x2f\xc8\x6e\xff\x0b\x00\x00\xff\xff\xec\xc3\x8e\x14\xd6\x2a\x00\x00") func deployDataVirtletDsYamlBytes() ([]byte, error) { return bindataRead( @@ -81,7 +81,7 @@ func deployDataVirtletDsYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "deploy/data/virtlet-ds.yaml", size: 10764, mode: os.FileMode(420), modTime: time.Unix(1522279343, 0)} + info := bindataFileInfo{name: "deploy/data/virtlet-ds.yaml", size: 10966, mode: os.FileMode(420), modTime: time.Unix(1522279343, 0)} a := &asset{bytes: bytes, info: info} return a, nil }