Skip to content

Commit

Permalink
Merge pull request kubernetes#10471 from phantooom/master
Browse files Browse the repository at this point in the history
kvm2 driver: Add flag --kvm-numa-count" support topology-manager simulate numa
  • Loading branch information
medyagh authored Mar 3, 2021
2 parents cb15985 + 529881e commit 5da07f1
Show file tree
Hide file tree
Showing 9 changed files with 190 additions and 4 deletions.
25 changes: 25 additions & 0 deletions cmd/minikube/cmd/start_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ const (
kvmQemuURI = "kvm-qemu-uri"
kvmGPU = "kvm-gpu"
kvmHidden = "kvm-hidden"
kvmNUMACount = "kvm-numa-count"
minikubeEnvPrefix = "MINIKUBE"
installAddons = "install-addons"
defaultDiskSize = "20000mb"
Expand Down Expand Up @@ -193,6 +194,7 @@ func initDriverFlags() {
startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (kvm2 driver only)")
startCmd.Flags().Bool(kvmGPU, false, "Enable experimental NVIDIA GPU support in minikube")
startCmd.Flags().Bool(kvmHidden, false, "Hide the hypervisor signature from the guest in minikube (kvm2 driver only)")
startCmd.Flags().Int(kvmNUMACount, 1, "Simulate numa node count in minikube, supported numa node count range is 1-8 (kvm2 driver only)")

// virtualbox
startCmd.Flags().String(hostOnlyCIDR, "192.168.99.1/24", "The CIDR to be used for the minikube VM (virtualbox driver only)")
Expand Down Expand Up @@ -311,6 +313,8 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
out.WarningT("--network flag is only valid with the docker/podman drivers, it will be ignored")
}

checkNumaCount(k8sVersion)

cc = config.ClusterConfig{
Name: ClusterFlagValue(),
KeepContext: viper.GetBool(keepContext),
Expand Down Expand Up @@ -338,6 +342,7 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
KVMQemuURI: viper.GetString(kvmQemuURI),
KVMGPU: viper.GetBool(kvmGPU),
KVMHidden: viper.GetBool(kvmHidden),
KVMNUMACount: viper.GetInt(kvmNUMACount),
DisableDriverMounts: viper.GetBool(disableDriverMounts),
UUID: viper.GetString(uuid),
NoVTXCheck: viper.GetBool(noVTXCheck),
Expand Down Expand Up @@ -407,6 +412,22 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
return createNode(cc, kubeNodeName, existing)
}

func checkNumaCount(k8sVersion string) {
if viper.GetInt(kvmNUMACount) < 1 || viper.GetInt(kvmNUMACount) > 8 {
exit.Message(reason.Usage, "--kvm-numa-count range is 1-8")
}

if viper.GetInt(kvmNUMACount) > 1 {
v, err := pkgutil.ParseKubernetesVersion(k8sVersion)
if err != nil {
exit.Message(reason.Usage, "invalid kubernetes version")
}
if v.LT(semver.Version{Major: 1, Minor: 18}) {
exit.Message(reason.Usage, "numa node is only supported on k8s v1.18 and later")
}
}
}

// upgradeExistingConfig upgrades legacy configuration files
func upgradeExistingConfig(cc *config.ClusterConfig) {
if cc == nil {
Expand Down Expand Up @@ -545,6 +566,10 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
cc.KVMHidden = viper.GetBool(kvmHidden)
}

if cmd.Flags().Changed(kvmNUMACount) {
cc.KVMNUMACount = viper.GetInt(kvmNUMACount)
}

if cmd.Flags().Changed(disableDriverMounts) {
cc.DisableDriverMounts = viper.GetBool(disableDriverMounts)
}
Expand Down
1 change: 1 addition & 0 deletions cmd/minikube/cmd/start_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ func TestMirrorCountry(t *testing.T) {
cmd := &cobra.Command{}
viper.SetDefault(imageRepository, test.imageRepository)
viper.SetDefault(imageMirrorCountry, test.mirrorCountry)
viper.SetDefault(kvmNUMACount, 1)
config, _, err := generateClusterConfig(cmd, nil, k8sVersion, "none")
if err != nil {
t.Fatalf("Got unexpected error %v during config generation", err)
Expand Down
8 changes: 5 additions & 3 deletions pkg/drivers/kvm/domain.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ const domainTmpl = `
</kvm>
{{end}}
</features>
<cpu mode='host-passthrough'/>
<cpu mode='host-passthrough'>
{{if gt .NUMANodeCount 1}}
{{.NUMANodeXML}}
{{end}}
</cpu>
<os>
<type>hvm</type>
<boot dev='cdrom'/>
Expand Down Expand Up @@ -158,14 +162,12 @@ func (d *Driver) createDomain() (*libvirt.Domain, error) {
}
d.PrivateMAC = mac.String()
}

// create the XML for the domain using our domainTmpl template
tmpl := template.Must(template.New("domain").Parse(domainTmpl))
var domainXML bytes.Buffer
if err := tmpl.Execute(&domainXML, d); err != nil {
return nil, errors.Wrap(err, "executing domain xml")
}

conn, err := getConnection(d.ConnectionURI)
if err != nil {
return nil, errors.Wrap(err, "error getting libvirt connection")
Expand Down
15 changes: 14 additions & 1 deletion pkg/drivers/kvm/kvm.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,12 @@ type Driver struct {

// QEMU Connection URI
ConnectionURI string

// NUMA node count default value is 1
NUMANodeCount int

// NUMA XML
NUMANodeXML string
}

const (
Expand Down Expand Up @@ -301,7 +307,6 @@ func (d *Driver) Start() (err error) {
func (d *Driver) Create() (err error) {
log.Info("Creating KVM machine...")
defer log.Infof("KVM machine creation complete!")

err = d.createNetwork()
if err != nil {
return errors.Wrap(err, "creating network")
Expand All @@ -314,6 +319,14 @@ func (d *Driver) Create() (err error) {
}
}

if d.NUMANodeCount > 1 {
numaXML, err := numaXML(d.CPU, d.Memory, d.NUMANodeCount)
if err != nil {
return errors.Wrap(err, "creating NUMA XML")
}
d.NUMANodeXML = numaXML
}

store := d.ResolveStorePath(".")
log.Infof("Setting up store path in %s ...", store)
// 0755 because it must be accessible by libvirt/qemu across a variety of configs
Expand Down
93 changes: 93 additions & 0 deletions pkg/drivers/kvm/numa.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
Copyright 2021 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kvm

import (
"bytes"
"fmt"
"strconv"
"strings"
"text/template"
)

// numaTmpl NUMA XML Template
const numaTmpl = `
<numa>
{{- range $idx,$val :=. }}
<cell id='{{$idx}}' cpus='{{$val.CPUTopology}}' memory='{{$val.Memory}}' unit='MiB'/>
{{- end }}
</numa>
`

// NUMA this struct use for numaTmpl
type NUMA struct {
// cpu count on numa node
CPUCount int
// memory on numa node
Memory int
// cpu sequence on numa node eg: 0,1,2,3
CPUTopology string
}

// numaXML generate numa xml
// evenly distributed cpu core & memory to each numa node
func numaXML(cpu, memory, numaCount int) (string, error) {
if numaCount < 1 {
return "", fmt.Errorf("numa node count must >= 1")
}
if cpu < numaCount {
return "", fmt.Errorf("cpu count must >= numa node count")
}
numaNodes := make([]*NUMA, numaCount)
CPUSeq := 0
cpuBaseCount := cpu / numaCount
cpuExtraCount := cpu % numaCount

for i := range numaNodes {
numaNodes[i] = &NUMA{CPUCount: cpuBaseCount}
}

for i := 0; i < cpuExtraCount; i++ {
numaNodes[i].CPUCount++
}
for i := range numaNodes {
CPUTopologySlice := make([]string, 0)
for seq := CPUSeq; seq < CPUSeq+numaNodes[i].CPUCount; seq++ {
CPUTopologySlice = append(CPUTopologySlice, strconv.Itoa(seq))
}
numaNodes[i].CPUTopology = strings.Join(CPUTopologySlice, ",")
CPUSeq += numaNodes[i].CPUCount
}

memoryBaseCount := memory / numaCount
memoryExtraCount := memory % numaCount

for i := range numaNodes {
numaNodes[i].Memory = memoryBaseCount
}

for i := 0; i < memoryExtraCount; i++ {
numaNodes[i].Memory++
}

tmpl := template.Must(template.New("numa").Parse(numaTmpl))
var numaXML bytes.Buffer
if err := tmpl.Execute(&numaXML, numaNodes); err != nil {
return "", fmt.Errorf("couldn't generate numa XML: %v", err)
}
return numaXML.String(), nil
}
48 changes: 48 additions & 0 deletions pkg/drivers/kvm/numa_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
Copyright 2021 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kvm

import (
"strings"
"testing"
)

func TestNumaXml(t *testing.T) {
_, err := numaXML(1, 1024, 0)
if err == nil {
t.Errorf("check invalid numa count failed: %s", err)
}

xml, err := numaXML(10, 10240, 8)
expXML := `<numa>
<cell id='0' cpus='0,1' memory='1280' unit='MiB'/>
<cell id='1' cpus='2,3' memory='1280' unit='MiB'/>
<cell id='2' cpus='4' memory='1280' unit='MiB'/>
<cell id='3' cpus='5' memory='1280' unit='MiB'/>
<cell id='4' cpus='6' memory='1280' unit='MiB'/>
<cell id='5' cpus='7' memory='1280' unit='MiB'/>
<cell id='6' cpus='8' memory='1280' unit='MiB'/>
<cell id='7' cpus='9' memory='1280' unit='MiB'/>
</numa>`
if err != nil {
t.Errorf("gen xml failed: %s", err)
}
if strings.TrimSpace(xml) != expXML {
t.Errorf("gen xml: %s not match expect xml: %s", xml, expXML)
}

}
1 change: 1 addition & 0 deletions pkg/minikube/config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type ClusterConfig struct {
KVMQemuURI string // Only used by kvm2
KVMGPU bool // Only used by kvm2
KVMHidden bool // Only used by kvm2
KVMNUMACount int // Only used by kvm2
DockerOpt []string // Each entry is formatted as KEY=VALUE.
DisableDriverMounts bool // Only used by virtualbox
NFSShare []string
Expand Down
2 changes: 2 additions & 0 deletions pkg/minikube/registry/drvs/kvm2/kvm2.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ type kvmDriver struct {
GPU bool
Hidden bool
ConnectionURI string
NUMANodeCount int
}

func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
Expand All @@ -89,6 +90,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
GPU: cc.KVMGPU,
Hidden: cc.KVMHidden,
ConnectionURI: cc.KVMQemuURI,
NUMANodeCount: cc.KVMNUMACount,
}, nil
}

Expand Down
1 change: 1 addition & 0 deletions site/content/en/docs/commands/start.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ minikube start [flags]
--kvm-gpu Enable experimental NVIDIA GPU support in minikube
--kvm-hidden Hide the hypervisor signature from the guest in minikube (kvm2 driver only)
--kvm-network string The KVM network name. (kvm2 driver only) (default "default")
--kvm-numa-count int Simulate numa node count in minikube, supported numa node count range is 1-8 (kvm2 driver only) (default 1)
--kvm-qemu-uri string The KVM QEMU connection URI. (kvm2 driver only) (default "qemu:///system")
--memory string Amount of RAM to allocate to Kubernetes (format: <number>[<unit>], where unit = b, k, m or g).
--mount This will start the mount daemon and automatically mount files into minikube.
Expand Down

0 comments on commit 5da07f1

Please sign in to comment.