From 337db1707daa3c110b079f6d1baa64a3aa1f7207 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 22 Nov 2019 15:11:51 -0800 Subject: [PATCH 01/14] First pass at multi-runtime pause/unpause --- cmd/minikube/cmd/mount.go | 1 - cmd/minikube/cmd/pause.go | 67 +++++ cmd/minikube/cmd/root.go | 2 + cmd/minikube/cmd/unpause.go | 64 +++++ go.sum | 3 + pkg/drivers/none/none.go | 58 +--- pkg/minikube/config/profile.go.orig | 201 +++++++++++++ pkg/minikube/cruntime/containerd.go | 15 +- pkg/minikube/cruntime/cri.go | 424 ++++++---------------------- pkg/minikube/cruntime/crio.go | 340 +++++++++++++++++++++- pkg/minikube/cruntime/cruntime.go | 25 +- pkg/minikube/cruntime/docker.go | 45 ++- pkg/minikube/kubelet/kubelet.go | 105 +++++++ pkg/minikube/logs/logs.go | 2 +- pkg/minikube/out/style.go | 2 + pkg/minikube/out/style_enum.go | 2 + pkg/minikube/pause/pause.go | 69 +++++ 17 files changed, 1025 insertions(+), 400 deletions(-) create mode 100644 cmd/minikube/cmd/pause.go create mode 100644 cmd/minikube/cmd/unpause.go create mode 100644 pkg/minikube/config/profile.go.orig create mode 100644 pkg/minikube/kubelet/kubelet.go create mode 100644 pkg/minikube/pause/pause.go diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go index d5cdaf418a10..e5f6abf45301 100644 --- a/cmd/minikube/cmd/mount.go +++ b/cmd/minikube/cmd/mount.go @@ -107,7 +107,6 @@ var mountCmd = &cobra.Command{ exit.WithError("Error getting config", err) } host, err := api.Load(cc.Name) - if err != nil { exit.WithError("Error loading api", err) } diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go new file mode 100644 index 000000000000..731c5f87d0bc --- /dev/null +++ b/cmd/minikube/cmd/pause.go @@ -0,0 +1,67 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/spf13/cobra" + + "k8s.io/minikube/pkg/minikube/cluster" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/pause" +) + +// pauseCmd represents the docker-pause command +var pauseCmd = &cobra.Command{ + Use: "pause", + Short: "pause containers", + Run: func(cmd *cobra.Command, args []string) { + api, err := machine.NewAPIClient() + if err != nil { + exit.WithError("Error getting client", err) + } + defer api.Close() + cc, err := config.Load() + if err != nil { + exit.WithError("Error getting config", err) + } + host, err := cluster.CheckIfHostExistsAndLoad(api, cc.Name) + if err != nil { + exit.WithError("Error getting host", err) + } + + r, err := machine.CommandRunner(host) + if err != nil { + exit.WithError("Failed to get command runner", err) + } + + config := cruntime.Config{Type: cc.ContainerRuntime} + cr, err := cruntime.New(config) + if err != nil { + exit.WithError("Failed runtime", err) + } + + err = pause.Pause(cr, r) + if err != nil { + exit.WithError("Pause", err) + } + out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name}) + }, +} diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go index 38f84246d9ed..6c09694ebfbf 100644 --- a/cmd/minikube/cmd/root.go +++ b/cmd/minikube/cmd/root.go @@ -172,6 +172,8 @@ func init() { stopCmd, deleteCmd, dashboardCmd, + pauseCmd, + unpauseCmd, }, }, { diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go new file mode 100644 index 000000000000..3e45eaf71195 --- /dev/null +++ b/cmd/minikube/cmd/unpause.go @@ -0,0 +1,64 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/spf13/cobra" + + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/pause" +) + +// unpauseCmd represents the docker-pause command +var unpauseCmd = &cobra.Command{ + Use: "unpause", + Short: "unpause Kubernetes", + Run: func(cmd *cobra.Command, args []string) { + api, err := machine.NewAPIClient() + if err != nil { + exit.WithError("Error getting client", err) + } + defer api.Close() + + cc, err := config.Load() + if err != nil { + exit.WithError("Error getting config", err) + } + host, err := api.Load(cc.Name) + + config := cruntime.Config{Type: cc.ContainerRuntime} + cr, err := cruntime.New(config) + if err != nil { + exit.WithError("Failed runtime", err) + } + + r, err := machine.CommandRunner(host) + if err != nil { + exit.WithError("Failed to get command runner", err) + } + + err = pause.Pause(cr, r) + if err != nil { + exit.WithError("Pause", err) + } + out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name}) + }, +} diff --git a/go.sum b/go.sum index 1971629b40ec..f8c771f02605 100644 --- a/go.sum +++ b/go.sum @@ -720,6 +720,7 @@ k8s.io/kubernetes v1.15.2 h1:RO9EuRw5vlN3oa/lnmPxmywOoJRtg9o40KcklHXNIAQ= k8s.io/kubernetes v1.15.2/go.mod h1:3RE5ikMc73WK+dSxk4pQuQ6ZaJcPXiZX2dj98RcdCuM= k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc h1:vZ5+77WP1yImZo23wc75vV5b5zCGq9gv484q8Yw5sBw= k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:pU9hbGZc8Z6+6HlNLEFY1GiNGzcCykU1Glsd4vEea2U= +k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc h1:8L3YgoEmmOxIGWNv9Hj6WhJuUspT+sw4gJs2nEc0qI0= k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:Q49J/iUBV6A9nn8loyV72DK2EXhN8sqCR8FyfxIFDA4= k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc h1:SHxaBZWgNouwsZCVg2+iffu0Um1ExSLPKgvO1drWShs= k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:rRBYbORqofLsn4/tsQWkeXkdKUoGrTfUwbI9s7NhU0Q= @@ -738,10 +739,12 @@ k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20190623232353-8c3b7d7679cc/ k8s.io/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:5RWpGgZKzUcW9gCtmSVRq8maZkOetGv87HrohpTrnLI= k8s.io/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:ogOX4l9UCMfFGIF+FZqmsln4NtCGPqf9zTMCIlm2YX4= k8s.io/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:o6aAFW1lCnr5CJm1riWnhQskrAHhyt8btyv5UHhgZ6c= +k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc h1:j30roBbl6b5Mom66efcNOHyjdYXU2RD8UWYnL0Adb8I= k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:y0hpsQGN8h3HcNqYbpSZEH4yC1ohi45N35c8ma9yg6M= k8s.io/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:/hbCTKdfutEO2iTQv8NuYcnAxd8Tuu4mMEymYv/EZHk= k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc h1:Bsljf/3UDy91qqLkevAiq6y+wl0qJrkLjWfBCQs9rss= k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:Vg6Q3IDU3hfYMICKyb43lClOXWtCtOBh2o1FfuQw8mQ= +k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc h1:ZUouIndlzPLGsRpeNAswxcs//fyODrNZOYybP6JZ9mM= k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:9UInPlSttlDwZBFNMAsqhTtl7zH00dE2M88B9Z0Ennc= k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:xlTRb77uaXbuT6evILwFescWPMENFKYGYj3a/kOjYQE= k8s.io/kubernetes/staging/src/k8s.io/metrics v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:6Cs3k9ccbWbJo3CQOrGDu2QEVLwsWbBlu9HitjPhuSk= diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index 4d99fd8b8ed1..9546cee2d625 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -19,19 +19,17 @@ package none import ( "fmt" "os/exec" - "strings" - "time" "github.com/docker/machine/libmachine/drivers" "github.com/docker/machine/libmachine/state" "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/net" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" pkgdrivers "k8s.io/minikube/pkg/drivers" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/vmpath" - "k8s.io/minikube/pkg/util/retry" ) // cleanupPaths are paths to be removed by cleanup, and are used by both kubeadm and minikube. @@ -123,7 +121,7 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { - if err := checkKubelet(d.exec); err != nil { + if err := kubelet.Check(d.exec); err != nil { glog.Infof("kubelet not running: %v", err) return state.Stopped, nil } @@ -132,7 +130,7 @@ func (d *Driver) GetState() (state.State, error) { // Kill stops a host forcefully, including any containers that we are managing. func (d *Driver) Kill() error { - if err := stopKubelet(d.exec); err != nil { + if err := kubelet.TryStopKubelet(d.exec); err != nil { return errors.Wrap(err, "kubelet") } @@ -177,7 +175,7 @@ func (d *Driver) Remove() error { // Restart a host func (d *Driver) Restart() error { - return restartKubelet(d.exec) + return kubelet.Restart(d.exec) } // Start a host @@ -196,7 +194,7 @@ func (d *Driver) Start() error { // Stop a host gracefully, including any containers that we are managing. func (d *Driver) Stop() error { - if err := stopKubelet(d.exec); err != nil { + if err := kubelet.Stop(d.exec); err != nil { return err } containers, err := d.runtime.ListContainers("") @@ -215,49 +213,3 @@ func (d *Driver) Stop() error { func (d *Driver) RunSSHCommandFromDriver() error { return fmt.Errorf("driver does not support ssh commands") } - -// stopKubelet idempotently stops the kubelet -func stopKubelet(cr command.Runner) error { - glog.Infof("stopping kubelet.service ...") - stop := func() error { - cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service") - if rr, err := cr.RunCmd(cmd); err != nil { - glog.Errorf("temporary error for %q : %v", rr.Command(), err) - } - cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet") - rr, err := cr.RunCmd(cmd) - if err != nil { - glog.Errorf("temporary error: for %q : %v", rr.Command(), err) - } - if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") { - return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String()) - } - return nil - } - - if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil { - return errors.Wrapf(err, "error stopping kubelet") - } - - return nil -} - -// restartKubelet restarts the kubelet -func restartKubelet(cr command.Runner) error { - glog.Infof("restarting kubelet.service ...") - c := exec.Command("sudo", "systemctl", "restart", "kubelet.service") - if _, err := cr.RunCmd(c); err != nil { - return err - } - return nil -} - -// checkKubelet returns an error if the kubelet is not running. -func checkKubelet(cr command.Runner) error { - glog.Infof("checking for running kubelet ...") - c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "check kubelet") - } - return nil -} diff --git a/pkg/minikube/config/profile.go.orig b/pkg/minikube/config/profile.go.orig new file mode 100644 index 000000000000..3546ec2e9341 --- /dev/null +++ b/pkg/minikube/config/profile.go.orig @@ -0,0 +1,201 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/golang/glog" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/util/lock" +) + +var keywords = []string{"start", "stop", "status", "delete", "config", "open", "profile", "addons", "cache", "logs"} + +// IsValid checks if the profile has the essential info needed for a profile +func (p *Profile) IsValid() bool { + if p.Config == nil { + return false + } + if len(p.Config) == 0 { + return false + } + // This will become a loop for multinode + if p.Config[0] == nil { + return false + } + if p.Config[0].VMDriver == "" { + return false + } + if p.Config[0].KubernetesConfig.KubernetesVersion == "" { + return false + } + return true +} + +// ProfileNameInReservedKeywords checks if the profile is an internal keywords +func ProfileNameInReservedKeywords(name string) bool { + for _, v := range keywords { + if strings.EqualFold(v, name) { + return true + } + } + return false +} + +// ProfileExists returns true if there is a profile config (regardless of being valid) +func ProfileExists(name string, miniHome ...string) bool { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + + p := profileFilePath(name, miniPath) + _, err := os.Stat(p) + return err == nil +} + +// CreateEmptyProfile creates an empty profile stores in $MINIKUBE_HOME/profiles//config.json +func CreateEmptyProfile(name string, miniHome ...string) error { + cfg := &MachineConfig{} + return CreateProfile(name, cfg, miniHome...) +} + +// CreateProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles//config.json +func CreateProfile(name string, cfg *MachineConfig, miniHome ...string) error { + data, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + path := profileFilePath(name, miniHome...) + glog.Infof("Saving config to %s ...", path) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return err + } + + // If no config file exists, don't worry about swapping paths + if _, err := os.Stat(path); os.IsNotExist(err) { + if err := lock.WriteFile(path, data, 0600); err != nil { + return err + } + return nil + } + + tf, err := ioutil.TempFile(filepath.Dir(path), "config.json.tmp") + if err != nil { + return err + } + defer os.Remove(tf.Name()) + + if err = lock.WriteFile(tf.Name(), data, 0600); err != nil { + return err + } + + if err = tf.Close(); err != nil { + return err + } + + if err = os.Remove(path); err != nil { + return err + } + + if err = os.Rename(tf.Name(), path); err != nil { + return err + } + return nil +} + +// DeleteProfile deletes a profile and removes the profile dir +func DeleteProfile(profile string, miniHome ...string) error { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + return os.RemoveAll(ProfileFolderPath(profile, miniPath)) +} + +// ListProfiles returns all valid and invalid (if any) minikube profiles +// invalidPs are the profiles that have a directory or config file but not usable +// invalidPs would be suggested to be deleted +func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, err error) { + pDirs, err := profileDirs(miniHome...) + if err != nil { + return nil, nil, err + } + for _, n := range pDirs { + p, err := LoadProfile(n, miniHome...) + if err != nil { + inValidPs = append(inValidPs, p) + continue + } + if !p.IsValid() { + inValidPs = append(inValidPs, p) + continue + } + validPs = append(validPs, p) + } + return validPs, inValidPs, nil +} + +// LoadProfile loads type Profile based on its name +func LoadProfile(name string, miniHome ...string) (*Profile, error) { + cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...) + p := &Profile{ + Name: name, + Config: []*MachineConfig{cfg}, + } + return p, err +} + +// profileDirs gets all the folders in the user's profiles folder regardless of valid or invalid config +func profileDirs(miniHome ...string) (dirs []string, err error) { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + pRootDir := filepath.Join(miniPath, "profiles") + items, err := ioutil.ReadDir(pRootDir) + for _, f := range items { + if f.IsDir() { + dirs = append(dirs, f.Name()) + } + } + return dirs, err +} + +// profileFilePath returns the Minikube profile config file +func profileFilePath(profile string, miniHome ...string) string { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + + return filepath.Join(miniPath, "profiles", profile, "config.json") +} + +// ProfileFolderPath returns path of profile folder +func ProfileFolderPath(profile string, miniHome ...string) string { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + return filepath.Join(miniPath, "profiles", profile) +} diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 934f65c416ba..e11d9f8ec512 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -32,6 +32,7 @@ import ( ) const ( + containerdNamespaceRoot = "/run/containerd/runc/k8s.io" // ContainerdConfFile is the path to the containerd configuration containerdConfigFile = "/etc/containerd/config.toml" containerdConfigTemplate = `root = "/var/lib/containerd" @@ -242,8 +243,18 @@ func (r *Containerd) KubeletOptions() map[string]string { } // ListContainers returns a list of managed by this container runtime -func (r *Containerd) ListContainers(filter string) ([]string, error) { - return listCRIContainers(r.Runner, filter) +func (r *Containerd) ListContainers(o ListOptions) ([]string, error) { + return listCRIContainers(r.Runner, containerdNamespaceRoot, o) +} + +// PauseContainers pauses a running container based on ID +func (r *Containerd) PauseContainers(ids []string) error { + return pauseCRIContainers(r.Runner, containerdNamespaceRoot, ids) +} + +// PauseContainers pauses a running container based on ID +func (r *Containerd) UnpauseContainers(ids []string) error { + return unpauseCRIContainers(r.Runner, containerdNamespaceRoot, ids) } // KillContainers removes containers based on ID diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 173e6523011f..ca77a58dabc9 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -18,7 +18,7 @@ package cruntime import ( "bytes" - "encoding/base64" + "encoding/json" "fmt" "html/template" "os/exec" @@ -27,331 +27,107 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/bootstrapper/images" - "k8s.io/minikube/pkg/minikube/command" ) -const ( - // CRIOConfFile is the path to the CRI-O configuration - crioConfigFile = "/etc/crio/crio.conf" - crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration -# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime -# daemon, but in a TOML format that can be more easily modified and versioned. -# -# Please refer to crio.conf(5) for details of all configuration options. - -# CRI-O supports partial configuration reload during runtime, which can be -# done by sending SIGHUP to the running process. Currently supported options -# are explicitly mentioned with: 'This option supports live configuration -# reload'. - -# CRI-O reads its storage defaults from the containers-storage.conf(5) file -# located at /etc/containers/storage.conf. Modify this storage configuration if -# you want to change the system's defaults. If you want to modify storage just -# for CRI-O, you can change the storage configuration options here. -[crio] - -# Path to the "root directory". CRI-O stores all of its data, including -# containers images, in this directory. -root = "/var/lib/containers/storage" - -# Path to the "run directory". CRI-O stores all of its state in this directory. -runroot = "/var/run/containers/storage" - -# Storage driver used to manage the storage of images and containers. Please -# refer to containers-storage.conf(5) to see all available storage drivers. -storage_driver = "overlay" - -# List to pass options to the storage driver. Please refer to -# containers-storage.conf(5) to see all available storage options. -#storage_option = [ -#] - -# If set to false, in-memory locking will be used instead of file-based locking. -# **Deprecated** this option will be removed in the future. -file_locking = false - -# Path to the lock file. -# **Deprecated** this option will be removed in the future. -file_locking_path = "/run/crio.lock" - - -# The crio.api table contains settings for the kubelet/gRPC interface. -[crio.api] - -# Path to AF_LOCAL socket on which CRI-O will listen. -listen = "/var/run/crio/crio.sock" - -# IP address on which the stream server will listen. -stream_address = "127.0.0.1" - -# The port on which the stream server will listen. -stream_port = "0" - -# Enable encrypted TLS transport of the stream server. -stream_enable_tls = false - -# Path to the x509 certificate file used to serve the encrypted stream. This -# file can change, and CRI-O will automatically pick up the changes within 5 -# minutes. -stream_tls_cert = "" - -# Path to the key file used to serve the encrypted stream. This file can -# change, and CRI-O will automatically pick up the changes within 5 minutes. -stream_tls_key = "" - -# Path to the x509 CA(s) file used to verify and authenticate client -# communication with the encrypted stream. This file can change, and CRI-O will -# automatically pick up the changes within 5 minutes. -stream_tls_ca = "" - -# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. -grpc_max_send_msg_size = 16777216 - -# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. -grpc_max_recv_msg_size = 16777216 - -# The crio.runtime table contains settings pertaining to the OCI runtime used -# and options for how to set up and manage the OCI runtime. -[crio.runtime] - -# A list of ulimits to be set in containers by default, specified as -# "=:", for example: -# "nofile=1024:2048" -# If nothing is set here, settings will be inherited from the CRI-O daemon -#default_ulimits = [ -#] - -# default_runtime is the _name_ of the OCI runtime to be used as the default. -# The name is matched against the runtimes map below. -default_runtime = "runc" - -# If true, the runtime will not use pivot_root, but instead use MS_MOVE. -no_pivot = true - -# Path to the conmon binary, used for monitoring the OCI runtime. -conmon = "/usr/libexec/crio/conmon" - -# Cgroup setting for conmon -conmon_cgroup = "pod" - -# Environment variable list for the conmon process, used for passing necessary -# environment variables to conmon or the runtime. -conmon_env = [ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", -] - -# If true, SELinux will be used for pod separation on the host. -selinux = false - -# Path to the seccomp.json profile which is used as the default seccomp profile -# for the runtime. If not specified, then the internal default seccomp profile -# will be used. -seccomp_profile = "" - -# Used to change the name of the default AppArmor profile of CRI-O. The default -# profile name is "crio-default-" followed by the version string of CRI-O. -apparmor_profile = "crio-default" - -# Cgroup management implementation used for the runtime. -cgroup_manager = "cgroupfs" - -# List of default capabilities for containers. If it is empty or commented out, -# only the capabilities defined in the containers json file by the user/kube -# will be added. -default_capabilities = [ - "CHOWN", - "DAC_OVERRIDE", - "FSETID", - "FOWNER", - "NET_RAW", - "SETGID", - "SETUID", - "SETPCAP", - "NET_BIND_SERVICE", - "SYS_CHROOT", - "KILL", -] - -# List of default sysctls. If it is empty or commented out, only the sysctls -# defined in the container json file by the user/kube will be added. -default_sysctls = [ -] - -# List of additional devices. specified as -# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". -#If it is empty or commented out, only the devices -# defined in the container json file by the user/kube will be added. -additional_devices = [ -] - -# Path to OCI hooks directories for automatically executed hooks. -hooks_dir = [ -] - -# List of default mounts for each container. **Deprecated:** this option will -# be removed in future versions in favor of default_mounts_file. -default_mounts = [ -] - -# Path to the file specifying the defaults mounts for each container. The -# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads -# its default mounts from the following two files: -# -# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the -# override file, where users can either add in their own default mounts, or -# override the default mounts shipped with the package. -# -# 2) /usr/share/containers/mounts.conf: This is the default file read for -# mounts. If you want CRI-O to read from a different, specific mounts file, -# you can change the default_mounts_file. Note, if this is done, CRI-O will -# only add mounts it finds in this file. -# -#default_mounts_file = "" - -# Maximum number of processes allowed in a container. -pids_limit = 1024 - -# Maximum sized allowed for the container log file. Negative numbers indicate -# that no size limit is imposed. If it is positive, it must be >= 8192 to -# match/exceed conmon's read buffer. The file is truncated and re-opened so the -# limit is never exceeded. -log_size_max = -1 - -# Whether container output should be logged to journald in addition to the kuberentes log file -log_to_journald = false - -# Path to directory in which container exit files are written to by conmon. -container_exits_dir = "/var/run/crio/exits" - -# Path to directory for container attach sockets. -container_attach_socket_dir = "/var/run/crio" - -# If set to true, all containers will run in read-only mode. -read_only = false - -# Changes the verbosity of the logs based on the level it is set to. Options -# are fatal, panic, error, warn, info, and debug. This option supports live -# configuration reload. -log_level = "error" - -# The default log directory where all logs will go unless directly specified by the kubelet -log_dir = "/var/log/crio/pods" - -# The UID mappings for the user namespace of each container. A range is -# specified in the form containerUID:HostUID:Size. Multiple ranges must be -# separated by comma. -uid_mappings = "" - -# The GID mappings for the user namespace of each container. A range is -# specified in the form containerGID:HostGID:Size. Multiple ranges must be -# separated by comma. -gid_mappings = "" - -# The minimal amount of time in seconds to wait before issuing a timeout -# regarding the proper termination of the container. -ctr_stop_timeout = 0 - -# ManageNetworkNSLifecycle determines whether we pin and remove network namespace -# and manage its lifecycle. -manage_network_ns_lifecycle = false - -# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. -# The runtime to use is picked based on the runtime_handler provided by the CRI. -# If no runtime_handler is provided, the runtime will be picked based on the level -# of trust of the workload. - -[crio.runtime.runtimes.runc] -runtime_path = "/usr/bin/runc" -runtime_type = "oci" -runtime_root = "/run/runc" - - -# The crio.image table contains settings pertaining to the management of OCI images. -# -# CRI-O reads its configured registries defaults from the system wide -# containers-registries.conf(5) located in /etc/containers/registries.conf. If -# you want to modify just CRI-O, you can change the registries configuration in -# this file. Otherwise, leave insecure_registries and registries commented out to -# use the system's defaults from /etc/containers/registries.conf. -[crio.image] - -# Default transport for pulling images from a remote container storage. -default_transport = "docker://" - -# The path to a file containing credentials necessary for pulling images from -# secure registries. The file is similar to that of /var/lib/kubelet/config.json -global_auth_file = "" - -# The image used to instantiate infra containers. -# This option supports live configuration reload. -pause_image = "{{ .PodInfraContainerImage }}" - -# The path to a file containing credentials specific for pulling the pause_image from -# above. The file is similar to that of /var/lib/kubelet/config.json -# This option supports live configuration reload. -pause_image_auth_file = "" +// container maps to 'runc list -f json' +type container struct { + pid int + id string + status string + annotations map[string]string +} -# The command to run to have a container stay in the paused state. -# This option supports live configuration reload. -pause_command = "/pause" +// listCRIContainers returns a list of containers +func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, error) { + // First use crictl, because it reliably matches names + args := []string{"crictl", "ps", "--quiet"} + if o.State == All { + args = append(args, "-a") + } + if o.Name != "" { + args = append(args, fmt.Sprintf("--name=%s", o.Name)) + } + rr, err := cr.RunCmd(exec.Command("sudo", args...)) + if err != nil { + return nil, err + } -# Path to the file which decides what sort of policy we use when deciding -# whether or not to trust an image that we've pulled. It is not recommended that -# this option be used, as the default behavior of using the system-wide default -# policy (i.e., /etc/containers/policy.json) is most often preferred. Please -# refer to containers-policy.json(5) for more details. -signature_policy = "" + // Avoid an id named "" + var ids []string + seen := map[string]bool{} + for _, id := range strings.Split(rr.Stderr.String(), "\n") { + if id != "" && !seen[id] { + ids = append(ids, id) + seen[id] = true + } + } -# Controls how image volumes are handled. The valid values are mkdir, bind and -# ignore; the latter will ignore volumes entirely. -image_volumes = "mkdir" + if len(ids) == 0 { + return nil, nil + } + if o.State == All { + return ids, nil + } -# List of registries to be used when pulling an unqualified image (e.g., -# "alpine:latest"). By default, registries is set to "docker.io" for -# compatibility reasons. Depending on your workload and usecase you may add more -# registries (e.g., "quay.io", "registry.fedoraproject.org", -# "registry.opensuse.org", etc.). -registries = [ - "docker.io" -] + // crictl does not understand paused pods + cs := []container{} + args = []string{"runc", "list", "-f", "json"} + if root != "" { + args = append(args, "--root", root) + } + if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil { + return nil, errors.Wrap(err, "runc") + } -# The crio.network table containers settings pertaining to the management of -# CNI plugins. -[crio.network] + d := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())) + if err := d.Decode(&cs); err != nil { + return nil, err + } -# Path to the directory where CNI configuration files are located. -network_dir = "/etc/cni/net.d/" + var fids []string + for _, c := range cs { + if !seen[c.id] { + continue + } + if o.State.String() != c.status { + continue + } + fids = append(fids, c.id) + } + return fids, nil +} -# Paths to directories where CNI plugin binaries are located. -plugin_dirs = [ - "/opt/cni/bin/", -] -` -) +// pauseCRIContainers pauses a list of containers +func pauseCRIContainers(cr CommandRunner, root string, ids []string) error { + args := []string{"runc", "pause"} + if root != "" { + args = append(args, "--root", root) + } -// listCRIContainers returns a list of containers using crictl -func listCRIContainers(cr CommandRunner, filter string) ([]string, error) { - var err error - var rr *command.RunResult - state := "Running" - if filter != "" { - c := exec.Command("sudo", "crictl", "ps", "-a", fmt.Sprintf("--name=%s", filter), fmt.Sprintf("--state=%s", state), "--quiet") - rr, err = cr.RunCmd(c) - } else { - rr, err = cr.RunCmd(exec.Command("sudo", "crictl", "ps", "-a", fmt.Sprintf("--state=%s", state), "--quiet")) + for _, id := range ids { + cargs := append(args, id) + if _, err := cr.RunCmd(exec.Command("sudo", cargs...)); err != nil { + return errors.Wrap(err, "runc") + } } - if err != nil { - return nil, err + return nil +} + +// unpauseCRIContainers pauses a list of containers +func unpauseCRIContainers(cr CommandRunner, root string, ids []string) error { + args := []string{"runc", "unpause"} + if root != "" { + args = append(args, "--root", root) } - var ids []string - for _, line := range strings.Split(rr.Stderr.String(), "\n") { - if line != "" { - ids = append(ids, line) + + for _, id := range ids { + cargs := append(args, id) + if _, err := cr.RunCmd(exec.Command("sudo", cargs...)); err != nil { + return errors.Wrap(err, "runc") } } - return ids, nil + return nil } // criCRIContainers kills a list of containers using crictl @@ -364,7 +140,7 @@ func killCRIContainers(cr CommandRunner, ids []string) error { args := append([]string{"crictl", "rm"}, ids...) c := exec.Command("sudo", args...) if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "kill cri containers.") + return errors.Wrap(err, "crictl") } return nil } @@ -378,10 +154,9 @@ func stopCRIContainers(cr CommandRunner, ids []string) error { args := append([]string{"crictl", "rm"}, ids...) c := exec.Command("sudo", args...) if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "stop cri containers") + return errors.Wrap(err, "crictl") } return nil - } // populateCRIConfig sets up /etc/crictl.yaml @@ -406,27 +181,6 @@ image-endpoint: unix://{{.Socket}} return nil } -// generateCRIOConfig sets up /etc/crio/crio.conf -func generateCRIOConfig(cr CommandRunner, imageRepository string, k8sVersion string) error { - cPath := crioConfigFile - t, err := template.New("crio.conf").Parse(crioConfigTemplate) - if err != nil { - return err - } - pauseImage := images.PauseImage(imageRepository, k8sVersion) - opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage} - var b bytes.Buffer - if err := t.Execute(&b, opts); err != nil { - return err - } - - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath)) - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "generateCRIOConfig.") - } - return nil -} - // criContainerLogCmd returns the command to retrieve the log for a container based on ID func criContainerLogCmd(id string, len int, follow bool) string { var cmd strings.Builder diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 763e52ac7395..e658494aafd3 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -17,15 +17,320 @@ limitations under the License. package cruntime import ( + "bytes" + "encoding/base64" "fmt" "os/exec" + "path" "strings" + "text/template" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/out" ) +const ( + // CRIOConfFile is the path to the CRI-O configuration + crioConfigFile = "/etc/crio/crio.conf" + crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration +# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime +# daemon, but in a TOML format that can be more easily modified and versioned. +# +# Please refer to crio.conf(5) for details of all configuration options. + +# CRI-O supports partial configuration reload during runtime, which can be +# done by sending SIGHUP to the running process. Currently supported options +# are explicitly mentioned with: 'This option supports live configuration +# reload'. + +# CRI-O reads its storage defaults from the containers-storage.conf(5) file +# located at /etc/containers/storage.conf. Modify this storage configuration if +# you want to change the system's defaults. If you want to modify storage just +# for CRI-O, you can change the storage configuration options here. +[crio] + +# Path to the "root directory". CRI-O stores all of its data, including +# containers images, in this directory. +root = "/var/lib/containers/storage" + +# Path to the "run directory". CRI-O stores all of its state in this directory. +runroot = "/var/run/containers/storage" + +# Storage driver used to manage the storage of images and containers. Please +# refer to containers-storage.conf(5) to see all available storage drivers. +storage_driver = "overlay" + +# List to pass options to the storage driver. Please refer to +# containers-storage.conf(5) to see all available storage options. +#storage_option = [ +#] + +# If set to false, in-memory locking will be used instead of file-based locking. +# **Deprecated** this option will be removed in the future. +file_locking = false + +# Path to the lock file. +# **Deprecated** this option will be removed in the future. +file_locking_path = "/run/crio.lock" + + +# The crio.api table contains settings for the kubelet/gRPC interface. +[crio.api] + +# Path to AF_LOCAL socket on which CRI-O will listen. +listen = "/var/run/crio/crio.sock" + +# IP address on which the stream server will listen. +stream_address = "127.0.0.1" + +# The port on which the stream server will listen. +stream_port = "0" + +# Enable encrypted TLS transport of the stream server. +stream_enable_tls = false + +# Path to the x509 certificate file used to serve the encrypted stream. This +# file can change, and CRI-O will automatically pick up the changes within 5 +# minutes. +stream_tls_cert = "" + +# Path to the key file used to serve the encrypted stream. This file can +# change, and CRI-O will automatically pick up the changes within 5 minutes. +stream_tls_key = "" + +# Path to the x509 CA(s) file used to verify and authenticate client +# communication with the encrypted stream. This file can change, and CRI-O will +# automatically pick up the changes within 5 minutes. +stream_tls_ca = "" + +# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_send_msg_size = 16777216 + +# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_recv_msg_size = 16777216 + +# The crio.runtime table contains settings pertaining to the OCI runtime used +# and options for how to set up and manage the OCI runtime. +[crio.runtime] + +# A list of ulimits to be set in containers by default, specified as +# "=:", for example: +# "nofile=1024:2048" +# If nothing is set here, settings will be inherited from the CRI-O daemon +#default_ulimits = [ +#] + +# default_runtime is the _name_ of the OCI runtime to be used as the default. +# The name is matched against the runtimes map below. +default_runtime = "runc" + +# If true, the runtime will not use pivot_root, but instead use MS_MOVE. +no_pivot = true + +# Path to the conmon binary, used for monitoring the OCI runtime. +conmon = "/usr/libexec/crio/conmon" + +# Cgroup setting for conmon +conmon_cgroup = "pod" + +# Environment variable list for the conmon process, used for passing necessary +# environment variables to conmon or the runtime. +conmon_env = [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] + +# If true, SELinux will be used for pod separation on the host. +selinux = false + +# Path to the seccomp.json profile which is used as the default seccomp profile +# for the runtime. If not specified, then the internal default seccomp profile +# will be used. +seccomp_profile = "" + +# Used to change the name of the default AppArmor profile of CRI-O. The default +# profile name is "crio-default-" followed by the version string of CRI-O. +apparmor_profile = "crio-default" + +# Cgroup management implementation used for the runtime. +cgroup_manager = "cgroupfs" + +# List of default capabilities for containers. If it is empty or commented out, +# only the capabilities defined in the containers json file by the user/kube +# will be added. +default_capabilities = [ + "CHOWN", + "DAC_OVERRIDE", + "FSETID", + "FOWNER", + "NET_RAW", + "SETGID", + "SETUID", + "SETPCAP", + "NET_BIND_SERVICE", + "SYS_CHROOT", + "KILL", +] + +# List of default sysctls. If it is empty or commented out, only the sysctls +# defined in the container json file by the user/kube will be added. +default_sysctls = [ +] + +# List of additional devices. specified as +# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". +#If it is empty or commented out, only the devices +# defined in the container json file by the user/kube will be added. +additional_devices = [ +] + +# Path to OCI hooks directories for automatically executed hooks. +hooks_dir = [ +] + +# List of default mounts for each container. **Deprecated:** this option will +# be removed in future versions in favor of default_mounts_file. +default_mounts = [ +] + +# Path to the file specifying the defaults mounts for each container. The +# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads +# its default mounts from the following two files: +# +# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the +# override file, where users can either add in their own default mounts, or +# override the default mounts shipped with the package. +# +# 2) /usr/share/containers/mounts.conf: This is the default file read for +# mounts. If you want CRI-O to read from a different, specific mounts file, +# you can change the default_mounts_file. Note, if this is done, CRI-O will +# only add mounts it finds in this file. +# +#default_mounts_file = "" + +# Maximum number of processes allowed in a container. +pids_limit = 1024 + +# Maximum sized allowed for the container log file. Negative numbers indicate +# that no size limit is imposed. If it is positive, it must be >= 8192 to +# match/exceed conmon's read buffer. The file is truncated and re-opened so the +# limit is never exceeded. +log_size_max = -1 + +# Whether container output should be logged to journald in addition to the kuberentes log file +log_to_journald = false + +# Path to directory in which container exit files are written to by conmon. +container_exits_dir = "/var/run/crio/exits" + +# Path to directory for container attach sockets. +container_attach_socket_dir = "/var/run/crio" + +# If set to true, all containers will run in read-only mode. +read_only = false + +# Changes the verbosity of the logs based on the level it is set to. Options +# are fatal, panic, error, warn, info, and debug. This option supports live +# configuration reload. +log_level = "error" + +# The default log directory where all logs will go unless directly specified by the kubelet +log_dir = "/var/log/crio/pods" + +# The UID mappings for the user namespace of each container. A range is +# specified in the form containerUID:HostUID:Size. Multiple ranges must be +# separated by comma. +uid_mappings = "" + +# The GID mappings for the user namespace of each container. A range is +# specified in the form containerGID:HostGID:Size. Multiple ranges must be +# separated by comma. +gid_mappings = "" + +# The minimal amount of time in seconds to wait before issuing a timeout +# regarding the proper termination of the container. +ctr_stop_timeout = 0 + +# ManageNetworkNSLifecycle determines whether we pin and remove network namespace +# and manage its lifecycle. +manage_network_ns_lifecycle = false + +# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. +# The runtime to use is picked based on the runtime_handler provided by the CRI. +# If no runtime_handler is provided, the runtime will be picked based on the level +# of trust of the workload. + +[crio.runtime.runtimes.runc] +runtime_path = "/usr/bin/runc" +runtime_type = "oci" +runtime_root = "/run/runc" + + +# The crio.image table contains settings pertaining to the management of OCI images. +# +# CRI-O reads its configured registries defaults from the system wide +# containers-registries.conf(5) located in /etc/containers/registries.conf. If +# you want to modify just CRI-O, you can change the registries configuration in +# this file. Otherwise, leave insecure_registries and registries commented out to +# use the system's defaults from /etc/containers/registries.conf. +[crio.image] + +# Default transport for pulling images from a remote container storage. +default_transport = "docker://" + +# The path to a file containing credentials necessary for pulling images from +# secure registries. The file is similar to that of /var/lib/kubelet/config.json +global_auth_file = "" + +# The image used to instantiate infra containers. +# This option supports live configuration reload. +pause_image = "{{ .PodInfraContainerImage }}" + +# The path to a file containing credentials specific for pulling the pause_image from +# above. The file is similar to that of /var/lib/kubelet/config.json +# This option supports live configuration reload. +pause_image_auth_file = "" + +# The command to run to have a container stay in the paused state. +# This option supports live configuration reload. +pause_command = "/pause" + +# Path to the file which decides what sort of policy we use when deciding +# whether or not to trust an image that we've pulled. It is not recommended that +# this option be used, as the default behavior of using the system-wide default +# policy (i.e., /etc/containers/policy.json) is most often preferred. Please +# refer to containers-policy.json(5) for more details. +signature_policy = "" + +# Controls how image volumes are handled. The valid values are mkdir, bind and +# ignore; the latter will ignore volumes entirely. +image_volumes = "mkdir" + +# List of registries to be used when pulling an unqualified image (e.g., +# "alpine:latest"). By default, registries is set to "docker.io" for +# compatibility reasons. Depending on your workload and usecase you may add more +# registries (e.g., "quay.io", "registry.fedoraproject.org", +# "registry.opensuse.org", etc.). +registries = [ + "docker.io" +] + + +# The crio.network table containers settings pertaining to the management of +# CNI plugins. +[crio.network] + +# Path to the directory where CNI configuration files are located. +network_dir = "/etc/cni/net.d/" + +# Paths to directories where CNI plugin binaries are located. +plugin_dirs = [ + "/opt/cni/bin/", +] +` +) + // CRIO contains CRIO runtime state type CRIO struct { Socket string @@ -140,8 +445,18 @@ func (r *CRIO) KubeletOptions() map[string]string { } // ListContainers returns a list of managed by this container runtime -func (r *CRIO) ListContainers(filter string) ([]string, error) { - return listCRIContainers(r.Runner, filter) +func (r *CRIO) ListContainers(o ListOptions) ([]string, error) { + return listCRIContainers(r.Runner, "", o) +} + +// PauseContainers pauses a running container based on ID +func (r *CRIO) PauseContainers(ids []string) error { + return pauseCRIContainers(r.Runner, "", ids) +} + +// PauseContainers pauses a running container based on ID +func (r *CRIO) UnpauseContainers(ids []string) error { + return unpauseCRIContainers(r.Runner, "", ids) } // KillContainers removes containers based on ID @@ -163,3 +478,24 @@ func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string { func (r *CRIO) SystemLogCmd(len int) string { return fmt.Sprintf("sudo journalctl -u crio -n %d", len) } + +// generateCRIOConfig sets up /etc/crio/crio.conf +func generateCRIOConfig(cr CommandRunner, imageRepository string, k8sVersion string) error { + cPath := crioConfigFile + t, err := template.New("crio.conf").Parse(crioConfigTemplate) + if err != nil { + return err + } + pauseImage := images.PauseImage(imageRepository, k8sVersion) + opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage} + var b bytes.Buffer + if err := t.Execute(&b, opts); err != nil { + return err + } + + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath)) + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "generateCRIOConfig.") + } + return nil +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 6566a8d8cc02..9ca3e078bfbc 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -27,6 +27,18 @@ import ( "k8s.io/minikube/pkg/minikube/out" ) +type ContainerState int + +const ( + All ContainerState = iota + Running + Paused +) + +func (cs ContainerState) String() string { + return [...]string{"All", "Running", "Paused"}[cs] +} + // CommandRunner is the subset of command.Runner this package consumes type CommandRunner interface { RunCmd(cmd *exec.Cmd) (*command.RunResult, error) @@ -60,11 +72,15 @@ type Manager interface { LoadImage(string) error // ListContainers returns a list of managed by this container runtime - ListContainers(string) ([]string, error) + ListContainers(ListOptions) ([]string, error) // KillContainers removes containers based on ID KillContainers([]string) error // StopContainers stops containers based on ID StopContainers([]string) error + // PauseContainers pauses containers based on ID + PauseContainers([]string) error + // UnpauseContainers unpauses containers based on ID + UnpauseContainers([]string) error // ContainerLogCmd returns the command to retrieve the log for a container based on ID ContainerLogCmd(string, int, bool) string // SystemLogCmd returns the command to return the system logs @@ -85,6 +101,13 @@ type Config struct { KubernetesVersion string } +type ListOptions struct { + // State is the container state to filter by (All, Running, Paused) + State ContainerState + // Name is a name filter + Name string +} + // New returns an appropriately configured runtime func New(c Config) (Manager, error) { switch c.Type { diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 3d80fca4595b..1bc2ab6920cb 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -121,11 +121,18 @@ func (r *Docker) KubeletOptions() map[string]string { } // ListContainers returns a list of containers -func (r *Docker) ListContainers(filter string) ([]string, error) { - filter = KubernetesContainerPrefix + filter - rr, err := r.Runner.RunCmd(exec.Command("docker", "ps", "-a", fmt.Sprintf("--filter=name=%s", filter), "--format=\"{{.ID}}\"")) +func (r *Docker) ListContainers(o ListOptions) ([]string, error) { + args := []string{"ps"} + switch o.State { + case All: + args = append(args, "-a") + case Paused: + args = append(args, "--filter status=paused") + } + args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format=\"{{.ID}}\"") + rr, err := r.Runner.RunCmd(exec.Command("docker", args...)) if err != nil { - return nil, errors.Wrapf(err, "docker ListContainers. ") + return nil, errors.Wrapf(err, "docker") } var ids []string for _, line := range strings.Split(rr.Stdout.String(), "\n") { @@ -159,7 +166,35 @@ func (r *Docker) StopContainers(ids []string) error { args := append([]string{"stop"}, ids...) c := exec.Command("docker", args...) if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "stopping containers docker.") + return errors.Wrap(err, "docker") + } + return nil +} + +// PauseContainers pauses a running container based on ID +func (r *Docker) PauseContainers(ids []string) error { + if len(ids) == 0 { + return nil + } + glog.Infof("Pausing containers: %s", ids) + args := append([]string{"pause"}, ids...) + c := exec.Command("docker", args...) + if _, err := r.Runner.RunCmd(c); err != nil { + return errors.Wrap(err, "docker") + } + return nil +} + +// UnpauseContainers unpauses a container based on ID +func (r *Docker) UnpauseContainers(ids []string) error { + if len(ids) == 0 { + return nil + } + glog.Infof("Unpausing containers: %s", ids) + args := append([]string{"unpause"}, ids...) + c := exec.Command("docker", args...) + if _, err := r.Runner.RunCmd(c); err != nil { + return errors.Wrap(err, "docker") } return nil } diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go new file mode 100644 index 000000000000..c8e0436a2f98 --- /dev/null +++ b/pkg/minikube/kubelet/kubelet.go @@ -0,0 +1,105 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "os/exec" + "strings" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/util/retry" +) + +// Stop idempotently stops the kubelet +func Stop(cr command.Runner) error { + glog.Infof("stopping kubelet ...") + stop := func() error { + cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service") + if rr, err := cr.RunCmd(cmd); err != nil { + glog.Errorf("temporary error for %q : %v", rr.Command(), err) + } + cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet") + rr, err := cr.RunCmd(cmd) + if err != nil { + glog.Errorf("temporary error: for %q : %v", rr.Command(), err) + } + if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") { + return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String()) + } + return nil + } + + if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil { + return errors.Wrapf(err, "error stopping kubelet") + } + + return nil +} + +// Start starts the kubelet +func Start(cr command.Runner) error { + glog.Infof("restarting kubelet.service ...") + c := exec.Command("sudo", "systemctl", "start", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return err + } + return nil +} + +// Restart restarts the kubelet +func Restart(cr command.Runner) error { + glog.Infof("restarting kubelet.service ...") + c := exec.Command("sudo", "systemctl", "restart", "kubelet.service") + if _, err := cr.RunCmd(c); err != nil { + return err + } + return nil +} + +// Check checks on the status of the kubelet +func Check(cr command.Runner) error { + glog.Infof("checking for running kubelet ...") + c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "check kubelet") + } + return nil +} + +// Disable disables the Kubelet +func Disable(cr command.Runner) error { + glog.Infof("disabling kubelet ...") + c := exec.Command("systemctl", "disable", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "disable") + } + return nil +} + +// Enable enables the Kubelet +func Enable(cr command.Runner) error { + glog.Infof("enabling kubelet ...") + c := exec.Command("systemctl", "enable", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "enable") + } + return nil +} diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go index f56230261900..ff5dcdacf084 100644 --- a/pkg/minikube/logs/logs.go +++ b/pkg/minikube/logs/logs.go @@ -170,7 +170,7 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner command.Run func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string { cmds := bs.LogCommands(bootstrapper.LogOptions{Lines: length, Follow: follow}) for _, pod := range importantPods { - ids, err := r.ListContainers(pod) + ids, err := r.ListContainers(cruntime.ListOptions{Name: pod}) if err != nil { glog.Errorf("Failed to list containers for %q: %v", pod, err) continue diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go index b14c9d5883e2..946ee16f0183 100644 --- a/pkg/minikube/out/style.go +++ b/pkg/minikube/out/style.go @@ -81,6 +81,8 @@ var styles = map[StyleEnum]style{ Celebration: {Prefix: "πŸŽ‰ "}, Workaround: {Prefix: "πŸ‘‰ ", LowPrefix: lowIndent}, Sparkle: {Prefix: "✨ "}, + Pause: {Prefix: "⏸️ "}, + Unpause: {Prefix: "⏯️ "}, // Specialized purpose styles ISODownload: {Prefix: "πŸ’Ώ "}, diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go index 0bc8ac182279..3787dbe90420 100644 --- a/pkg/minikube/out/style_enum.go +++ b/pkg/minikube/out/style_enum.go @@ -84,4 +84,6 @@ const ( Empty Workaround Sparkle + Pause + Unpause ) diff --git a/pkg/minikube/pause/pause.go b/pkg/minikube/pause/pause.go new file mode 100644 index 000000000000..8800fabfefd3 --- /dev/null +++ b/pkg/minikube/pause/pause.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pause + +import ( + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/kubelet" +) + +// Pause pauses a Kubernetes cluster +func Pause(cr cruntime.Manager, r command.Runner) error { + if err := kubelet.Disable(r); err != nil { + return errors.Wrap(err, "kubelet disable") + } + if err := kubelet.Stop(r); err != nil { + return errors.Wrap(err, "kubelet stop") + } + ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running}) + if err != nil { + return errors.Wrap(err, "list running") + } + if len(ids) == 0 { + glog.Warningf("no running containers to pause") + return nil + } + return cr.PauseContainers(ids) + +} + +// Unpause unpauses a Kubernetes cluster +func Unpause(cr cruntime.Manager, r command.Runner) error { + ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused}) + if err != nil { + return errors.Wrap(err, "list paused") + } + + if len(ids) == 0 { + glog.Warningf("no paused containers found") + } else { + if err := cr.UnpauseContainers(ids); err != nil { + return errors.Wrap(err, "unpause") + } + } + + if err := kubelet.Enable(r); err != nil { + return errors.Wrap(err, "kubelet enable") + } + if err := kubelet.Start(r); err != nil { + return errors.Wrap(err, "kubelet start") + } + return nil +} From 64ee657c49a125330e2edacfb12c554eccd85862 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 22 Nov 2019 16:12:24 -0800 Subject: [PATCH 02/14] Fix some bugs, add some logging, have a turkey --- cmd/minikube/cmd/pause.go | 8 +++++-- cmd/minikube/cmd/unpause.go | 22 ++++++++++------- pkg/minikube/config/config.go | 2 ++ pkg/minikube/cruntime/cri.go | 40 ++++++++++++++++++++----------- pkg/minikube/cruntime/cruntime.go | 2 +- pkg/minikube/cruntime/docker.go | 4 ++-- pkg/minikube/kubelet/kubelet.go | 4 ++-- 7 files changed, 53 insertions(+), 29 deletions(-) diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go index 731c5f87d0bc..dcfb53f82a72 100644 --- a/cmd/minikube/cmd/pause.go +++ b/cmd/minikube/cmd/pause.go @@ -17,7 +17,9 @@ limitations under the License. package cmd import ( + "github.com/golang/glog" "github.com/spf13/cobra" + "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" @@ -33,6 +35,7 @@ var pauseCmd = &cobra.Command{ Use: "pause", Short: "pause containers", Run: func(cmd *cobra.Command, args []string) { + cname := viper.GetString(config.MachineProfile) api, err := machine.NewAPIClient() if err != nil { exit.WithError("Error getting client", err) @@ -42,7 +45,8 @@ var pauseCmd = &cobra.Command{ if err != nil { exit.WithError("Error getting config", err) } - host, err := cluster.CheckIfHostExistsAndLoad(api, cc.Name) + glog.Infof("config: %+v", cc) + host, err := cluster.CheckIfHostExistsAndLoad(api, cname) if err != nil { exit.WithError("Error getting host", err) } @@ -52,7 +56,7 @@ var pauseCmd = &cobra.Command{ exit.WithError("Failed to get command runner", err) } - config := cruntime.Config{Type: cc.ContainerRuntime} + config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r} cr, err := cruntime.New(config) if err != nil { exit.WithError("Failed runtime", err) diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go index 3e45eaf71195..64d4eab3df5a 100644 --- a/cmd/minikube/cmd/unpause.go +++ b/cmd/minikube/cmd/unpause.go @@ -17,8 +17,11 @@ limitations under the License. package cmd import ( + "github.com/golang/glog" "github.com/spf13/cobra" + "github.com/spf13/viper" + "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/exit" @@ -32,22 +35,20 @@ var unpauseCmd = &cobra.Command{ Use: "unpause", Short: "unpause Kubernetes", Run: func(cmd *cobra.Command, args []string) { + cname := viper.GetString(config.MachineProfile) api, err := machine.NewAPIClient() if err != nil { exit.WithError("Error getting client", err) } defer api.Close() - cc, err := config.Load() if err != nil { exit.WithError("Error getting config", err) } - host, err := api.Load(cc.Name) - - config := cruntime.Config{Type: cc.ContainerRuntime} - cr, err := cruntime.New(config) + glog.Infof("config: %+v", cc) + host, err := cluster.CheckIfHostExistsAndLoad(api, cname) if err != nil { - exit.WithError("Failed runtime", err) + exit.WithError("Error getting host", err) } r, err := machine.CommandRunner(host) @@ -55,10 +56,15 @@ var unpauseCmd = &cobra.Command{ exit.WithError("Failed to get command runner", err) } - err = pause.Pause(cr, r) + config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r} + cr, err := cruntime.New(config) + if err != nil { + exit.WithError("Failed runtime", err) + } + err = pause.Unpause(cr, r) if err != nil { exit.WithError("Pause", err) } - out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name}) + out.T(out.Unpause, "The '{{.name}}' cluster is now unpaused", out.V{"name": cc.Name}) }, } diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index 7b7501ad158c..4c67669e93f8 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" + "github.com/golang/glog" "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/localpath" ) @@ -126,6 +127,7 @@ func encode(w io.Writer, m MinikubeConfig) error { // Load loads the kubernetes and machine config for the current machine func Load() (*MachineConfig, error) { machine := viper.GetString(MachineProfile) + glog.Infof("Load machine name = %q", machine) return DefaultLoader.LoadConfigFromFile(machine) } diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index ca77a58dabc9..0d543d276f3c 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -31,10 +31,8 @@ import ( // container maps to 'runc list -f json' type container struct { - pid int - id string - status string - annotations map[string]string + ID string + Status string } // listCRIContainers returns a list of containers @@ -55,7 +53,8 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, // Avoid an id named "" var ids []string seen := map[string]bool{} - for _, id := range strings.Split(rr.Stderr.String(), "\n") { + for _, id := range strings.Split(rr.Stdout.String(), "\n") { + glog.Infof("found id: %q", id) if id != "" && !seen[id] { ids = append(ids, id) seen[id] = true @@ -71,39 +70,51 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, // crictl does not understand paused pods cs := []container{} - args = []string{"runc", "list", "-f", "json"} + args = []string{"runc"} if root != "" { args = append(args, "--root", root) } - if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil { + args = append(args, "list", "-f", "json") + rr, err = cr.RunCmd(exec.Command("sudo", args...)) + if err != nil { return nil, errors.Wrap(err, "runc") } - - d := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())) + content := rr.Stdout.Bytes() + glog.Infof("JSON = %s", content) + d := json.NewDecoder(bytes.NewReader(content)) if err := d.Decode(&cs); err != nil { return nil, err } + if len(cs) == 0 { + return nil, fmt.Errorf("list returned 0 containers, but ps returned %d", len(ids)) + } + + glog.Infof("list returned %d containers", len(cs)) var fids []string for _, c := range cs { - if !seen[c.id] { + glog.Infof("container: %+v", c) + if !seen[c.ID] { + glog.Infof("skipping %s - not in ps", c.ID) continue } - if o.State.String() != c.status { + if o.State != All && o.State.String() != c.Status { + glog.Infof("skipping %s: state = %q, want %q", c, c.Status, o.State) continue } - fids = append(fids, c.id) + fids = append(fids, c.ID) } return fids, nil } // pauseCRIContainers pauses a list of containers func pauseCRIContainers(cr CommandRunner, root string, ids []string) error { - args := []string{"runc", "pause"} + args := []string{"runc"} if root != "" { args = append(args, "--root", root) } + args = append(args, "pause") for _, id := range ids { cargs := append(args, id) @@ -116,10 +127,11 @@ func pauseCRIContainers(cr CommandRunner, root string, ids []string) error { // unpauseCRIContainers pauses a list of containers func unpauseCRIContainers(cr CommandRunner, root string, ids []string) error { - args := []string{"runc", "unpause"} + args := []string{"runc"} if root != "" { args = append(args, "--root", root) } + args = append(args, "resume") for _, id := range ids { cargs := append(args, id) diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 9ca3e078bfbc..e07ba610812a 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -36,7 +36,7 @@ const ( ) func (cs ContainerState) String() string { - return [...]string{"All", "Running", "Paused"}[cs] + return [...]string{"all", "running", "paused"}[cs] } // CommandRunner is the subset of command.Runner this package consumes diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 1bc2ab6920cb..e72b3b7f9728 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -127,9 +127,9 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) { case All: args = append(args, "-a") case Paused: - args = append(args, "--filter status=paused") + args = append(args, "--filter", "status=paused") } - args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format=\"{{.ID}}\"") + args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format={{.ID}}") rr, err := r.Runner.RunCmd(exec.Command("docker", args...)) if err != nil { return nil, errors.Wrapf(err, "docker") diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go index c8e0436a2f98..315f492878cc 100644 --- a/pkg/minikube/kubelet/kubelet.go +++ b/pkg/minikube/kubelet/kubelet.go @@ -87,7 +87,7 @@ func Check(cr command.Runner) error { // Disable disables the Kubelet func Disable(cr command.Runner) error { glog.Infof("disabling kubelet ...") - c := exec.Command("systemctl", "disable", "kubelet") + c := exec.Command("sudo", "systemctl", "disable", "kubelet") if _, err := cr.RunCmd(c); err != nil { return errors.Wrap(err, "disable") } @@ -97,7 +97,7 @@ func Disable(cr command.Runner) error { // Enable enables the Kubelet func Enable(cr command.Runner) error { glog.Infof("enabling kubelet ...") - c := exec.Command("systemctl", "enable", "kubelet") + c := exec.Command("sudo", "systemctl", "enable", "kubelet") if _, err := cr.RunCmd(c); err != nil { return errors.Wrap(err, "enable") } From 7fe786aca460e6507bfc251119d5d37b1ac3e439 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 21 Jan 2020 19:46:05 -0800 Subject: [PATCH 03/14] Merge upstream --- pkg/minikube/config/config.go | 22 ++- pkg/minikube/config/profile.go.orig | 201 ---------------------------- pkg/minikube/cruntime/cri.go | 83 +++++++++--- pkg/minikube/out/style_enum.go | 1 + 4 files changed, 83 insertions(+), 224 deletions(-) delete mode 100644 pkg/minikube/config/profile.go.orig diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index 7b7501ad158c..7abd2a8d8119 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -24,7 +24,6 @@ import ( "io/ioutil" "os" - "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/localpath" ) @@ -124,14 +123,19 @@ func encode(w io.Writer, m MinikubeConfig) error { } // Load loads the kubernetes and machine config for the current machine -func Load() (*MachineConfig, error) { - machine := viper.GetString(MachineProfile) - return DefaultLoader.LoadConfigFromFile(machine) +func Load(profile string) (*MachineConfig, error) { + return DefaultLoader.LoadConfigFromFile(profile) +} + +// Write writes the kubernetes and machine config for the current machine +func Write(profile string, cc *MachineConfig) error { + return DefaultLoader.WriteConfigToFile(profile, cc) } // Loader loads the kubernetes and machine config based on the machine profile name type Loader interface { LoadConfigFromFile(profile string, miniHome ...string) (*MachineConfig, error) + WriteConfigToFile(profileName string, cc *MachineConfig, miniHome ...string) error } type simpleConfigLoader struct{} @@ -158,3 +162,13 @@ func (c *simpleConfigLoader) LoadConfigFromFile(profileName string, miniHome ... } return &cc, nil } + +func (c *simpleConfigLoader) WriteConfigToFile(profileName string, cc *MachineConfig, miniHome ...string) error { + // Move to profile package + path := profileFilePath(profileName, miniHome...) + contents, err := json.MarshalIndent(cc, "", " ") + if err != nil { + return err + } + return ioutil.WriteFile(path, contents, 0644) +} diff --git a/pkg/minikube/config/profile.go.orig b/pkg/minikube/config/profile.go.orig deleted file mode 100644 index 3546ec2e9341..000000000000 --- a/pkg/minikube/config/profile.go.orig +++ /dev/null @@ -1,201 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "encoding/json" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/golang/glog" - "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/util/lock" -) - -var keywords = []string{"start", "stop", "status", "delete", "config", "open", "profile", "addons", "cache", "logs"} - -// IsValid checks if the profile has the essential info needed for a profile -func (p *Profile) IsValid() bool { - if p.Config == nil { - return false - } - if len(p.Config) == 0 { - return false - } - // This will become a loop for multinode - if p.Config[0] == nil { - return false - } - if p.Config[0].VMDriver == "" { - return false - } - if p.Config[0].KubernetesConfig.KubernetesVersion == "" { - return false - } - return true -} - -// ProfileNameInReservedKeywords checks if the profile is an internal keywords -func ProfileNameInReservedKeywords(name string) bool { - for _, v := range keywords { - if strings.EqualFold(v, name) { - return true - } - } - return false -} - -// ProfileExists returns true if there is a profile config (regardless of being valid) -func ProfileExists(name string, miniHome ...string) bool { - miniPath := localpath.MiniPath() - if len(miniHome) > 0 { - miniPath = miniHome[0] - } - - p := profileFilePath(name, miniPath) - _, err := os.Stat(p) - return err == nil -} - -// CreateEmptyProfile creates an empty profile stores in $MINIKUBE_HOME/profiles//config.json -func CreateEmptyProfile(name string, miniHome ...string) error { - cfg := &MachineConfig{} - return CreateProfile(name, cfg, miniHome...) -} - -// CreateProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles//config.json -func CreateProfile(name string, cfg *MachineConfig, miniHome ...string) error { - data, err := json.MarshalIndent(cfg, "", " ") - if err != nil { - return err - } - path := profileFilePath(name, miniHome...) - glog.Infof("Saving config to %s ...", path) - if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { - return err - } - - // If no config file exists, don't worry about swapping paths - if _, err := os.Stat(path); os.IsNotExist(err) { - if err := lock.WriteFile(path, data, 0600); err != nil { - return err - } - return nil - } - - tf, err := ioutil.TempFile(filepath.Dir(path), "config.json.tmp") - if err != nil { - return err - } - defer os.Remove(tf.Name()) - - if err = lock.WriteFile(tf.Name(), data, 0600); err != nil { - return err - } - - if err = tf.Close(); err != nil { - return err - } - - if err = os.Remove(path); err != nil { - return err - } - - if err = os.Rename(tf.Name(), path); err != nil { - return err - } - return nil -} - -// DeleteProfile deletes a profile and removes the profile dir -func DeleteProfile(profile string, miniHome ...string) error { - miniPath := localpath.MiniPath() - if len(miniHome) > 0 { - miniPath = miniHome[0] - } - return os.RemoveAll(ProfileFolderPath(profile, miniPath)) -} - -// ListProfiles returns all valid and invalid (if any) minikube profiles -// invalidPs are the profiles that have a directory or config file but not usable -// invalidPs would be suggested to be deleted -func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, err error) { - pDirs, err := profileDirs(miniHome...) - if err != nil { - return nil, nil, err - } - for _, n := range pDirs { - p, err := LoadProfile(n, miniHome...) - if err != nil { - inValidPs = append(inValidPs, p) - continue - } - if !p.IsValid() { - inValidPs = append(inValidPs, p) - continue - } - validPs = append(validPs, p) - } - return validPs, inValidPs, nil -} - -// LoadProfile loads type Profile based on its name -func LoadProfile(name string, miniHome ...string) (*Profile, error) { - cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...) - p := &Profile{ - Name: name, - Config: []*MachineConfig{cfg}, - } - return p, err -} - -// profileDirs gets all the folders in the user's profiles folder regardless of valid or invalid config -func profileDirs(miniHome ...string) (dirs []string, err error) { - miniPath := localpath.MiniPath() - if len(miniHome) > 0 { - miniPath = miniHome[0] - } - pRootDir := filepath.Join(miniPath, "profiles") - items, err := ioutil.ReadDir(pRootDir) - for _, f := range items { - if f.IsDir() { - dirs = append(dirs, f.Name()) - } - } - return dirs, err -} - -// profileFilePath returns the Minikube profile config file -func profileFilePath(profile string, miniHome ...string) string { - miniPath := localpath.MiniPath() - if len(miniHome) > 0 { - miniPath = miniHome[0] - } - - return filepath.Join(miniPath, "profiles", profile, "config.json") -} - -// ProfileFolderPath returns path of profile folder -func ProfileFolderPath(profile string, miniHome ...string) string { - miniPath := localpath.MiniPath() - if len(miniHome) > 0 { - miniPath = miniHome[0] - } - return filepath.Join(miniPath, "profiles", profile) -} diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index ca77a58dabc9..b3618272a4f0 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -31,10 +31,8 @@ import ( // container maps to 'runc list -f json' type container struct { - pid int - id string - status string - annotations map[string]string + ID string + Status string } // listCRIContainers returns a list of containers @@ -55,7 +53,8 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, // Avoid an id named "" var ids []string seen := map[string]bool{} - for _, id := range strings.Split(rr.Stderr.String(), "\n") { + for _, id := range strings.Split(rr.Stdout.String(), "\n") { + glog.Infof("found id: %q", id) if id != "" && !seen[id] { ids = append(ids, id) seen[id] = true @@ -71,39 +70,51 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, // crictl does not understand paused pods cs := []container{} - args = []string{"runc", "list", "-f", "json"} + args = []string{"runc"} if root != "" { args = append(args, "--root", root) } - if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil { + args = append(args, "list", "-f", "json") + rr, err = cr.RunCmd(exec.Command("sudo", args...)) + if err != nil { return nil, errors.Wrap(err, "runc") } - - d := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())) + content := rr.Stdout.Bytes() + glog.Infof("JSON = %s", content) + d := json.NewDecoder(bytes.NewReader(content)) if err := d.Decode(&cs); err != nil { return nil, err } + if len(cs) == 0 { + return nil, fmt.Errorf("list returned 0 containers, but ps returned %d", len(ids)) + } + + glog.Infof("list returned %d containers", len(cs)) var fids []string for _, c := range cs { - if !seen[c.id] { + glog.Infof("container: %+v", c) + if !seen[c.ID] { + glog.Infof("skipping %s - not in ps", c.ID) continue } - if o.State.String() != c.status { + if o.State != All && o.State.String() != c.Status { + glog.Infof("skipping %s: state = %q, want %q", c, c.Status, o.State) continue } - fids = append(fids, c.id) + fids = append(fids, c.ID) } return fids, nil } -// pauseCRIContainers pauses a list of containers +// pauseContainers pauses a list of containers func pauseCRIContainers(cr CommandRunner, root string, ids []string) error { - args := []string{"runc", "pause"} + args := []string{"runc"} if root != "" { args = append(args, "--root", root) } + args = append(args, "pause") for _, id := range ids { cargs := append(args, id) @@ -114,12 +125,23 @@ func pauseCRIContainers(cr CommandRunner, root string, ids []string) error { return nil } +// getCrictlPath returns the absolute path of crictl +func getCrictlPath(cr CommandRunner) string { + cmd := "crictl" + rr, err := cr.RunCmd(exec.Command("which", cmd)) + if err != nil { + return cmd + } + return strings.Split(rr.Stdout.String(), "\n")[0] +} + // unpauseCRIContainers pauses a list of containers func unpauseCRIContainers(cr CommandRunner, root string, ids []string) error { - args := []string{"runc", "unpause"} + args := []string{"runc"} if root != "" { args = append(args, "--root", root) } + args = append(args, "resume") for _, id := range ids { cargs := append(args, id) @@ -137,7 +159,8 @@ func killCRIContainers(cr CommandRunner, ids []string) error { } glog.Infof("Killing containers: %s", ids) - args := append([]string{"crictl", "rm"}, ids...) + crictl := getCrictlPath(cr) + args := append([]string{crictl, "rm"}, ids...) c := exec.Command("sudo", args...) if _, err := cr.RunCmd(c); err != nil { return errors.Wrap(err, "crictl") @@ -151,7 +174,9 @@ func stopCRIContainers(cr CommandRunner, ids []string) error { return nil } glog.Infof("Stopping containers: %s", ids) - args := append([]string{"crictl", "rm"}, ids...) + + crictl := getCrictlPath(cr) + args := append([]string{crictl, "rm"}, ids...) c := exec.Command("sudo", args...) if _, err := cr.RunCmd(c); err != nil { return errors.Wrap(err, "crictl") @@ -181,10 +206,30 @@ image-endpoint: unix://{{.Socket}} return nil } +// getCRIInfo returns current information +func getCRIInfo(cr CommandRunner) (map[string]interface{}, error) { + args := []string{"crictl", "info"} + c := exec.Command("sudo", args...) + rr, err := cr.RunCmd(c) + if err != nil { + return nil, errors.Wrap(err, "get cri info") + } + info := rr.Stdout.String() + jsonMap := make(map[string]interface{}) + err = json.Unmarshal([]byte(info), &jsonMap) + if err != nil { + return nil, err + } + return jsonMap, nil +} + // criContainerLogCmd returns the command to retrieve the log for a container based on ID -func criContainerLogCmd(id string, len int, follow bool) string { +func criContainerLogCmd(cr CommandRunner, id string, len int, follow bool) string { + crictl := getCrictlPath(cr) var cmd strings.Builder - cmd.WriteString("sudo crictl logs ") + cmd.WriteString("sudo ") + cmd.WriteString(crictl) + cmd.WriteString(" logs ") if len > 0 { cmd.WriteString(fmt.Sprintf("--tail %d ", len)) } diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go index 3787dbe90420..0d0c6e9ae1f5 100644 --- a/pkg/minikube/out/style_enum.go +++ b/pkg/minikube/out/style_enum.go @@ -86,4 +86,5 @@ const ( Sparkle Pause Unpause + DryRun ) From be65e4c2ad9168e69761220029dde7681feab3a3 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 21 Jan 2020 19:46:42 -0800 Subject: [PATCH 04/14] Merge upstream --- cmd/minikube/cmd/pause.go | 21 +++++++++++++++++---- cmd/minikube/cmd/unpause.go | 31 +++++++++++++++++++++++-------- pkg/minikube/cruntime/cruntime.go | 13 ++++++++++++- pkg/minikube/cruntime/docker.go | 29 +++++++++++++++++++++++++++-- pkg/minikube/kubelet/kubelet.go | 4 ++-- 5 files changed, 81 insertions(+), 17 deletions(-) diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go index 731c5f87d0bc..797cffa25df0 100644 --- a/cmd/minikube/cmd/pause.go +++ b/cmd/minikube/cmd/pause.go @@ -17,7 +17,11 @@ limitations under the License. package cmd import ( + "os" + + "github.com/golang/glog" "github.com/spf13/cobra" + "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" @@ -33,16 +37,25 @@ var pauseCmd = &cobra.Command{ Use: "pause", Short: "pause containers", Run: func(cmd *cobra.Command, args []string) { + cname := viper.GetString(config.MachineProfile) api, err := machine.NewAPIClient() if err != nil { exit.WithError("Error getting client", err) } defer api.Close() - cc, err := config.Load() + cc, err := config.Load(cname) + + if err != nil && !os.IsNotExist(err) { + exit.WithError("Error loading profile config", err) + } + if err != nil { - exit.WithError("Error getting config", err) + out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname}) + os.Exit(1) } - host, err := cluster.CheckIfHostExistsAndLoad(api, cc.Name) + + glog.Infof("config: %+v", cc) + host, err := cluster.CheckIfHostExistsAndLoad(api, cname) if err != nil { exit.WithError("Error getting host", err) } @@ -52,7 +65,7 @@ var pauseCmd = &cobra.Command{ exit.WithError("Failed to get command runner", err) } - config := cruntime.Config{Type: cc.ContainerRuntime} + config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r} cr, err := cruntime.New(config) if err != nil { exit.WithError("Failed runtime", err) diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go index 3e45eaf71195..e0df53060eff 100644 --- a/cmd/minikube/cmd/unpause.go +++ b/cmd/minikube/cmd/unpause.go @@ -17,8 +17,13 @@ limitations under the License. package cmd import ( + "os" + + "github.com/golang/glog" "github.com/spf13/cobra" + "github.com/spf13/viper" + "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/exit" @@ -32,22 +37,27 @@ var unpauseCmd = &cobra.Command{ Use: "unpause", Short: "unpause Kubernetes", Run: func(cmd *cobra.Command, args []string) { + cname := viper.GetString(config.MachineProfile) api, err := machine.NewAPIClient() if err != nil { exit.WithError("Error getting client", err) } defer api.Close() + cc, err := config.Load(cname) + + if err != nil && !os.IsNotExist(err) { + exit.WithError("Error loading profile config", err) + } - cc, err := config.Load() if err != nil { - exit.WithError("Error getting config", err) + out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname}) + os.Exit(1) } - host, err := api.Load(cc.Name) - config := cruntime.Config{Type: cc.ContainerRuntime} - cr, err := cruntime.New(config) + glog.Infof("config: %+v", cc) + host, err := cluster.CheckIfHostExistsAndLoad(api, cname) if err != nil { - exit.WithError("Failed runtime", err) + exit.WithError("Error getting host", err) } r, err := machine.CommandRunner(host) @@ -55,10 +65,15 @@ var unpauseCmd = &cobra.Command{ exit.WithError("Failed to get command runner", err) } - err = pause.Pause(cr, r) + config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r} + cr, err := cruntime.New(config) + if err != nil { + exit.WithError("Failed runtime", err) + } + err = pause.Unpause(cr, r) if err != nil { exit.WithError("Pause", err) } - out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name}) + out.T(out.Unpause, "The '{{.name}}' cluster is now unpaused", out.V{"name": cc.Name}) }, } diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 9ca3e078bfbc..df816136789c 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -36,7 +36,7 @@ const ( ) func (cs ContainerState) String() string { - return [...]string{"All", "Running", "Paused"}[cs] + return [...]string{"all", "running", "paused"}[cs] } // CommandRunner is the subset of command.Runner this package consumes @@ -61,6 +61,8 @@ type Manager interface { // Style is an associated StyleEnum for Name() Style() out.StyleEnum + // CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") + CGroupDriver() (string, error) // KubeletOptions returns kubelet options for a runtime. KubeletOptions() map[string]string // SocketPath returns the path to the socket file for a given runtime @@ -71,6 +73,9 @@ type Manager interface { // Load an image idempotently into the runtime on a host LoadImage(string) error + // ImageExists takes image name and image sha checks if an it exists + ImageExists(string, string) bool + // ListContainers returns a list of managed by this container runtime ListContainers(ListOptions) ([]string, error) // KillContainers removes containers based on ID @@ -122,6 +127,12 @@ func New(c Config) (Manager, error) { } } +// ContainerStatusCommand works across container runtimes with good formatting +func ContainerStatusCommand() string { + // Fallback to 'docker ps' if it fails (none driver) + return "sudo `which crictl || echo crictl` ps -a || sudo docker ps -a" +} + // disableOthers disables all other runtimes except for me. func disableOthers(me Manager, cr CommandRunner) error { // valid values returned by manager.Name() diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 1bc2ab6920cb..0b5297f381d2 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -102,6 +102,20 @@ func (r *Docker) Disable() error { return nil } +// ImageExists checks if an image exists +func (r *Docker) ImageExists(name string, sha string) bool { + // expected output looks like [SHA_ALGO:SHA] + c := exec.Command("docker", "inspect", "--format='{{.Id}}'", name) + rr, err := r.Runner.RunCmd(c) + if err != nil { + return false + } + if !strings.Contains(rr.Output(), sha) { + return false + } + return true +} + // LoadImage loads an image into this runtime func (r *Docker) LoadImage(path string) error { glog.Infof("Loading image: %s", path) @@ -113,6 +127,17 @@ func (r *Docker) LoadImage(path string) error { } +// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") +func (r *Docker) CGroupDriver() (string, error) { + // Note: the server daemon has to be running, for this call to return successfully + c := exec.Command("docker", "info", "--format", "'{{.CgroupDriver}}'") + rr, err := r.Runner.RunCmd(c) + if err != nil { + return "", err + } + return strings.Split(rr.Stdout.String(), "\n")[0], nil +} + // KubeletOptions returns kubelet options for a runtime. func (r *Docker) KubeletOptions() map[string]string { return map[string]string{ @@ -127,9 +152,9 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) { case All: args = append(args, "-a") case Paused: - args = append(args, "--filter status=paused") + args = append(args, "--filter", "status=paused") } - args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format=\"{{.ID}}\"") + args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format={{.ID}}") rr, err := r.Runner.RunCmd(exec.Command("docker", args...)) if err != nil { return nil, errors.Wrapf(err, "docker") diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go index c8e0436a2f98..315f492878cc 100644 --- a/pkg/minikube/kubelet/kubelet.go +++ b/pkg/minikube/kubelet/kubelet.go @@ -87,7 +87,7 @@ func Check(cr command.Runner) error { // Disable disables the Kubelet func Disable(cr command.Runner) error { glog.Infof("disabling kubelet ...") - c := exec.Command("systemctl", "disable", "kubelet") + c := exec.Command("sudo", "systemctl", "disable", "kubelet") if _, err := cr.RunCmd(c); err != nil { return errors.Wrap(err, "disable") } @@ -97,7 +97,7 @@ func Disable(cr command.Runner) error { // Enable enables the Kubelet func Enable(cr command.Runner) error { glog.Infof("enabling kubelet ...") - c := exec.Command("systemctl", "enable", "kubelet") + c := exec.Command("sudo", "systemctl", "enable", "kubelet") if _, err := cr.RunCmd(c); err != nil { return errors.Wrap(err, "enable") } From 0f40eef9a7d556aee63e140a8b1db635d24fde77 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 21 Jan 2020 20:06:03 -0800 Subject: [PATCH 05/14] Merge conflicts --- pkg/minikube/cruntime/crio.go | 327 +--------------------------------- 1 file changed, 1 insertion(+), 326 deletions(-) diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 648e350dee26..dc4670cca765 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -17,13 +17,9 @@ limitations under the License. package cruntime import ( - "bytes" - "encoding/base64" "fmt" "os/exec" - "path" "strings" - "text/template" "github.com/golang/glog" "github.com/pkg/errors" @@ -33,302 +29,7 @@ import ( const ( // CRIOConfFile is the path to the CRI-O configuration - crioConfigFile = "/etc/crio/crio.conf" - crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration -# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime -# daemon, but in a TOML format that can be more easily modified and versioned. -# -# Please refer to crio.conf(5) for details of all configuration options. - -# CRI-O supports partial configuration reload during runtime, which can be -# done by sending SIGHUP to the running process. Currently supported options -# are explicitly mentioned with: 'This option supports live configuration -# reload'. - -# CRI-O reads its storage defaults from the containers-storage.conf(5) file -# located at /etc/containers/storage.conf. Modify this storage configuration if -# you want to change the system's defaults. If you want to modify storage just -# for CRI-O, you can change the storage configuration options here. -[crio] - -# Path to the "root directory". CRI-O stores all of its data, including -# containers images, in this directory. -root = "/var/lib/containers/storage" - -# Path to the "run directory". CRI-O stores all of its state in this directory. -runroot = "/var/run/containers/storage" - -# Storage driver used to manage the storage of images and containers. Please -# refer to containers-storage.conf(5) to see all available storage drivers. -storage_driver = "overlay" - -# List to pass options to the storage driver. Please refer to -# containers-storage.conf(5) to see all available storage options. -#storage_option = [ -#] - -# If set to false, in-memory locking will be used instead of file-based locking. -# **Deprecated** this option will be removed in the future. -file_locking = false - -# Path to the lock file. -# **Deprecated** this option will be removed in the future. -file_locking_path = "/run/crio.lock" - - -# The crio.api table contains settings for the kubelet/gRPC interface. -[crio.api] - -# Path to AF_LOCAL socket on which CRI-O will listen. -listen = "/var/run/crio/crio.sock" - -# IP address on which the stream server will listen. -stream_address = "127.0.0.1" - -# The port on which the stream server will listen. -stream_port = "0" - -# Enable encrypted TLS transport of the stream server. -stream_enable_tls = false - -# Path to the x509 certificate file used to serve the encrypted stream. This -# file can change, and CRI-O will automatically pick up the changes within 5 -# minutes. -stream_tls_cert = "" - -# Path to the key file used to serve the encrypted stream. This file can -# change, and CRI-O will automatically pick up the changes within 5 minutes. -stream_tls_key = "" - -# Path to the x509 CA(s) file used to verify and authenticate client -# communication with the encrypted stream. This file can change, and CRI-O will -# automatically pick up the changes within 5 minutes. -stream_tls_ca = "" - -# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. -grpc_max_send_msg_size = 16777216 - -# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. -grpc_max_recv_msg_size = 16777216 - -# The crio.runtime table contains settings pertaining to the OCI runtime used -# and options for how to set up and manage the OCI runtime. -[crio.runtime] - -# A list of ulimits to be set in containers by default, specified as -# "=:", for example: -# "nofile=1024:2048" -# If nothing is set here, settings will be inherited from the CRI-O daemon -#default_ulimits = [ -#] - -# default_runtime is the _name_ of the OCI runtime to be used as the default. -# The name is matched against the runtimes map below. -default_runtime = "runc" - -# If true, the runtime will not use pivot_root, but instead use MS_MOVE. -no_pivot = true - -# Path to the conmon binary, used for monitoring the OCI runtime. -conmon = "/usr/libexec/crio/conmon" - -# Cgroup setting for conmon -conmon_cgroup = "pod" - -# Environment variable list for the conmon process, used for passing necessary -# environment variables to conmon or the runtime. -conmon_env = [ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", -] - -# If true, SELinux will be used for pod separation on the host. -selinux = false - -# Path to the seccomp.json profile which is used as the default seccomp profile -# for the runtime. If not specified, then the internal default seccomp profile -# will be used. -seccomp_profile = "" - -# Used to change the name of the default AppArmor profile of CRI-O. The default -# profile name is "crio-default-" followed by the version string of CRI-O. -apparmor_profile = "crio-default" - -# Cgroup management implementation used for the runtime. -cgroup_manager = "cgroupfs" - -# List of default capabilities for containers. If it is empty or commented out, -# only the capabilities defined in the containers json file by the user/kube -# will be added. -default_capabilities = [ - "CHOWN", - "DAC_OVERRIDE", - "FSETID", - "FOWNER", - "NET_RAW", - "SETGID", - "SETUID", - "SETPCAP", - "NET_BIND_SERVICE", - "SYS_CHROOT", - "KILL", -] - -# List of default sysctls. If it is empty or commented out, only the sysctls -# defined in the container json file by the user/kube will be added. -default_sysctls = [ -] - -# List of additional devices. specified as -# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". -#If it is empty or commented out, only the devices -# defined in the container json file by the user/kube will be added. -additional_devices = [ -] - -# Path to OCI hooks directories for automatically executed hooks. -hooks_dir = [ -] - -# List of default mounts for each container. **Deprecated:** this option will -# be removed in future versions in favor of default_mounts_file. -default_mounts = [ -] - -# Path to the file specifying the defaults mounts for each container. The -# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads -# its default mounts from the following two files: -# -# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the -# override file, where users can either add in their own default mounts, or -# override the default mounts shipped with the package. -# -# 2) /usr/share/containers/mounts.conf: This is the default file read for -# mounts. If you want CRI-O to read from a different, specific mounts file, -# you can change the default_mounts_file. Note, if this is done, CRI-O will -# only add mounts it finds in this file. -# -#default_mounts_file = "" - -# Maximum number of processes allowed in a container. -pids_limit = 1024 - -# Maximum sized allowed for the container log file. Negative numbers indicate -# that no size limit is imposed. If it is positive, it must be >= 8192 to -# match/exceed conmon's read buffer. The file is truncated and re-opened so the -# limit is never exceeded. -log_size_max = -1 - -# Whether container output should be logged to journald in addition to the kuberentes log file -log_to_journald = false - -# Path to directory in which container exit files are written to by conmon. -container_exits_dir = "/var/run/crio/exits" - -# Path to directory for container attach sockets. -container_attach_socket_dir = "/var/run/crio" - -# If set to true, all containers will run in read-only mode. -read_only = false - -# Changes the verbosity of the logs based on the level it is set to. Options -# are fatal, panic, error, warn, info, and debug. This option supports live -# configuration reload. -log_level = "error" - -# The default log directory where all logs will go unless directly specified by the kubelet -log_dir = "/var/log/crio/pods" - -# The UID mappings for the user namespace of each container. A range is -# specified in the form containerUID:HostUID:Size. Multiple ranges must be -# separated by comma. -uid_mappings = "" - -# The GID mappings for the user namespace of each container. A range is -# specified in the form containerGID:HostGID:Size. Multiple ranges must be -# separated by comma. -gid_mappings = "" - -# The minimal amount of time in seconds to wait before issuing a timeout -# regarding the proper termination of the container. -ctr_stop_timeout = 0 - -# ManageNetworkNSLifecycle determines whether we pin and remove network namespace -# and manage its lifecycle. -manage_network_ns_lifecycle = false - -# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. -# The runtime to use is picked based on the runtime_handler provided by the CRI. -# If no runtime_handler is provided, the runtime will be picked based on the level -# of trust of the workload. - -[crio.runtime.runtimes.runc] -runtime_path = "/usr/bin/runc" -runtime_type = "oci" -runtime_root = "/run/runc" - - -# The crio.image table contains settings pertaining to the management of OCI images. -# -# CRI-O reads its configured registries defaults from the system wide -# containers-registries.conf(5) located in /etc/containers/registries.conf. If -# you want to modify just CRI-O, you can change the registries configuration in -# this file. Otherwise, leave insecure_registries and registries commented out to -# use the system's defaults from /etc/containers/registries.conf. -[crio.image] - -# Default transport for pulling images from a remote container storage. -default_transport = "docker://" - -# The path to a file containing credentials necessary for pulling images from -# secure registries. The file is similar to that of /var/lib/kubelet/config.json -global_auth_file = "" - -# The image used to instantiate infra containers. -# This option supports live configuration reload. -pause_image = "{{ .PodInfraContainerImage }}" - -# The path to a file containing credentials specific for pulling the pause_image from -# above. The file is similar to that of /var/lib/kubelet/config.json -# This option supports live configuration reload. -pause_image_auth_file = "" - -# The command to run to have a container stay in the paused state. -# This option supports live configuration reload. -pause_command = "/pause" - -# Path to the file which decides what sort of policy we use when deciding -# whether or not to trust an image that we've pulled. It is not recommended that -# this option be used, as the default behavior of using the system-wide default -# policy (i.e., /etc/containers/policy.json) is most often preferred. Please -# refer to containers-policy.json(5) for more details. -signature_policy = "" - -# Controls how image volumes are handled. The valid values are mkdir, bind and -# ignore; the latter will ignore volumes entirely. -image_volumes = "mkdir" - -# List of registries to be used when pulling an unqualified image (e.g., -# "alpine:latest"). By default, registries is set to "docker.io" for -# compatibility reasons. Depending on your workload and usecase you may add more -# registries (e.g., "quay.io", "registry.fedoraproject.org", -# "registry.opensuse.org", etc.). -registries = [ - "docker.io" -] - - -# The crio.network table containers settings pertaining to the management of -# CNI plugins. -[crio.network] - -# Path to the directory where CNI configuration files are located. -network_dir = "/etc/cni/net.d/" - -# Paths to directories where CNI plugin binaries are located. -plugin_dirs = [ - "/opt/cni/bin/", -] -` + crioConfigFile = "/etc/crio/crio.conf" ) // CRIO contains CRIO runtime state @@ -339,11 +40,6 @@ type CRIO struct { KubernetesVersion string } -const ( - // CRIOConfFile is the path to the CRI-O configuration - crioConfigFile = "/etc/crio/crio.conf" -) - // generateCRIOConfig sets up /etc/crio/crio.conf func generateCRIOConfig(cr CommandRunner, imageRepository string) error { cPath := crioConfigFile @@ -529,24 +225,3 @@ func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string { func (r *CRIO) SystemLogCmd(len int) string { return fmt.Sprintf("sudo journalctl -u crio -n %d", len) } - -// generateCRIOConfig sets up /etc/crio/crio.conf -func generateCRIOConfig(cr CommandRunner, imageRepository string) error { - cPath := crioConfigFile - t, err := template.New("crio.conf").Parse(crioConfigTemplate) - if err != nil { - return err - } - pauseImage := images.Pause(imageRepository) - opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage} - var b bytes.Buffer - if err := t.Execute(&b, opts); err != nil { - return err - } - - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath)) - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "generateCRIOConfig.") - } - return nil -} From 09b2780fb2e1503f071cbc20d7627cd518781c02 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 21 Jan 2020 22:06:05 -0800 Subject: [PATCH 06/14] Add support for pausing/unpausing specific namespaces --- cmd/minikube/cmd/pause.go | 98 +++++++++++++++--------- cmd/minikube/cmd/unpause.go | 21 ++++- pkg/minikube/{pause => cluster}/pause.go | 17 ++-- pkg/minikube/cruntime/cri.go | 41 +++++++--- pkg/minikube/cruntime/cruntime.go | 2 + pkg/minikube/cruntime/docker.go | 9 ++- 6 files changed, 132 insertions(+), 56 deletions(-) rename pkg/minikube/{pause => cluster}/pause.go (80%) diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go index 797cffa25df0..dc9e3b61d8ba 100644 --- a/cmd/minikube/cmd/pause.go +++ b/cmd/minikube/cmd/pause.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors All rights reserved. +Copyright 2020 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package cmd import ( "os" + "strings" "github.com/golang/glog" "github.com/spf13/cobra" @@ -29,52 +30,77 @@ import ( "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/minikube/pause" +) + +var ( + namespaces []string + allNamespaces bool ) // pauseCmd represents the docker-pause command var pauseCmd = &cobra.Command{ Use: "pause", Short: "pause containers", - Run: func(cmd *cobra.Command, args []string) { - cname := viper.GetString(config.MachineProfile) - api, err := machine.NewAPIClient() - if err != nil { - exit.WithError("Error getting client", err) - } - defer api.Close() - cc, err := config.Load(cname) + Run: runPause, +} - if err != nil && !os.IsNotExist(err) { - exit.WithError("Error loading profile config", err) - } +func runPause(cmd *cobra.Command, args []string) { + cname := viper.GetString(config.MachineProfile) + api, err := machine.NewAPIClient() + if err != nil { + exit.WithError("Error getting client", err) + } + defer api.Close() + cc, err := config.Load(cname) - if err != nil { - out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname}) - os.Exit(1) - } + if err != nil && !os.IsNotExist(err) { + exit.WithError("Error loading profile config", err) + } - glog.Infof("config: %+v", cc) - host, err := cluster.CheckIfHostExistsAndLoad(api, cname) - if err != nil { - exit.WithError("Error getting host", err) - } + if err != nil { + out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname}) + os.Exit(1) + } - r, err := machine.CommandRunner(host) - if err != nil { - exit.WithError("Failed to get command runner", err) - } + glog.Infof("config: %+v", cc) + host, err := cluster.CheckIfHostExistsAndLoad(api, cname) + if err != nil { + exit.WithError("Error getting host", err) + } - config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r} - cr, err := cruntime.New(config) - if err != nil { - exit.WithError("Failed runtime", err) - } + r, err := machine.CommandRunner(host) + if err != nil { + exit.WithError("Failed to get command runner", err) + } - err = pause.Pause(cr, r) - if err != nil { - exit.WithError("Pause", err) + config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r} + cr, err := cruntime.New(config) + if err != nil { + exit.WithError("Failed runtime", err) + } + + glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings()) + if allNamespaces { + namespaces = nil //all + } else { + if len(namespaces) == 0 { + exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces") } - out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name}) - }, + } + + err = cluster.Pause(cr, r, namespaces) + if err != nil { + exit.WithError("Pause", err) + } + + if namespaces == nil { + out.T(out.Pause, "Paused all namespaces in the '{{.name}}' cluster", out.V{"name": cc.Name}) + } else { + out.T(out.Pause, "Paused the following namespaces in '{{.name}}': {{.namespaces}}", out.V{"name": cc.Name, "namespaces": strings.Join(namespaces, ", ")}) + } +} + +func init() { + pauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to pause") + pauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, pause all namespaces") } diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go index dbaf18ffb47b..7791c54e0b8f 100644 --- a/cmd/minikube/cmd/unpause.go +++ b/cmd/minikube/cmd/unpause.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors All rights reserved. +Copyright 2020 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,7 +29,6 @@ import ( "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/minikube/pause" ) // unpauseCmd represents the docker-pause command @@ -69,10 +68,26 @@ var unpauseCmd = &cobra.Command{ if err != nil { exit.WithError("Failed runtime", err) } - err = pause.Unpause(cr, r) + + glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings()) + if allNamespaces { + namespaces = nil //all + } else { + if len(namespaces) == 0 { + exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces") + } + } + + err = cluster.Unpause(cr, r, namespaces) + if err != nil { exit.WithError("Pause", err) } out.T(out.Unpause, "The '{{.name}}' cluster is now unpaused", out.V{"name": cc.Name}) }, } + +func init() { + unpauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to unpause") + unpauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, unpause all namespaces") +} diff --git a/pkg/minikube/pause/pause.go b/pkg/minikube/cluster/pause.go similarity index 80% rename from pkg/minikube/pause/pause.go rename to pkg/minikube/cluster/pause.go index 8800fabfefd3..b963b4b002a4 100644 --- a/pkg/minikube/pause/pause.go +++ b/pkg/minikube/cluster/pause.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package pause +package cluster import ( "github.com/golang/glog" @@ -24,15 +24,22 @@ import ( "k8s.io/minikube/pkg/minikube/kubelet" ) +// DefaultNamespaces are namespaces used by minikube, including addons +var DefaultNamespaces = []string{ + "kube-system", + "kubernetes-dashboard", + "storage-gluster", +} + // Pause pauses a Kubernetes cluster -func Pause(cr cruntime.Manager, r command.Runner) error { +func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) error { if err := kubelet.Disable(r); err != nil { return errors.Wrap(err, "kubelet disable") } if err := kubelet.Stop(r); err != nil { return errors.Wrap(err, "kubelet stop") } - ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running}) + ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running, Namespaces: namespaces}) if err != nil { return errors.Wrap(err, "list running") } @@ -45,8 +52,8 @@ func Pause(cr cruntime.Manager, r command.Runner) error { } // Unpause unpauses a Kubernetes cluster -func Unpause(cr cruntime.Manager, r command.Runner) error { - ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused}) +func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) error { + ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused, Namespaces: namespaces}) if err != nil { return errors.Wrap(err, "list paused") } diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index b3618272a4f0..2cf50c78e299 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -27,6 +27,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/command" ) // container maps to 'runc list -f json' @@ -35,19 +36,37 @@ type container struct { Status string } -// listCRIContainers returns a list of containers -func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, error) { - // First use crictl, because it reliably matches names - args := []string{"crictl", "ps", "--quiet"} - if o.State == All { - args = append(args, "-a") - } +// crictlList returns the output of 'crictl ps' in an efficient manner +func crictlList(cr CommandRunner, root string, o ListOptions) (*command.RunResult, error) { + glog.Infof("listing CRI containers in root %s: %+v", root, o) + + // Use -a because otherwise paused containers are missed + baseCmd := []string{"crictl", "ps", "-a", "--quiet"} + if o.Name != "" { - args = append(args, fmt.Sprintf("--name=%s", o.Name)) + baseCmd = append(baseCmd, fmt.Sprintf("--name=%s", o.Name)) + } + + // shortcut for all namespaces + if len(o.Namespaces) == 0 { + return cr.RunCmd(exec.Command("sudo", baseCmd...)) } - rr, err := cr.RunCmd(exec.Command("sudo", args...)) + + // Gather containers for all namespaces without causing extraneous shells to be launched + cmds := []string{} + for _, ns := range o.Namespaces { + cmd := fmt.Sprintf("%s --label io.kubernetes.pod.namespace=%s", strings.Join(baseCmd, " "), ns) + cmds = append(cmds, cmd) + } + + return cr.RunCmd(exec.Command("sudo", "-s", "eval", strings.Join(cmds, "; "))) +} + +// listCRIContainers returns a list of containers +func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, error) { + rr, err := crictlList(cr, root, o) if err != nil { - return nil, err + return nil, errors.Wrap(err, "crictl list") } // Avoid an id named "" @@ -70,7 +89,7 @@ func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, // crictl does not understand paused pods cs := []container{} - args = []string{"runc"} + args := []string{"runc"} if root != "" { args = append(args, "--root", root) } diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index df816136789c..2705a012ce70 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -111,6 +111,8 @@ type ListOptions struct { State ContainerState // Name is a name filter Name string + // Namespaces is the namespaces to look into + Namespaces []string } // New returns an appropriately configured runtime diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 0b5297f381d2..a58cff35b11f 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -154,7 +154,14 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) { case Paused: args = append(args, "--filter", "status=paused") } - args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format={{.ID}}") + + nameFilter := KubernetesContainerPrefix + o.Name + if len(o.Namespaces) > 0 { + // Example result: k8s.*(kube-system|kubernetes-dashboard) + nameFilter = fmt.Sprintf("%s.*_(%s)_", nameFilter, strings.Join(o.Namespaces, "|")) + } + + args = append(args, fmt.Sprintf("--filter=name=%s", nameFilter), "--format={{.ID}}") rr, err := r.Runner.RunCmd(exec.Command("docker", args...)) if err != nil { return nil, errors.Wrapf(err, "docker") From e6999caea376744a27ced7c5ea28350cce7d4cbd Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 22 Jan 2020 09:48:16 -0800 Subject: [PATCH 07/14] Add paused state to apiserver status --- cmd/minikube/cmd/status.go | 165 ++++++++++--------- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 23 ++- 2 files changed, 108 insertions(+), 80 deletions(-) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 91376babb774..171cf82e5ad9 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -23,8 +23,10 @@ import ( "strings" "text/template" + "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/state" "github.com/golang/glog" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" @@ -40,16 +42,13 @@ import ( var statusFormat string var output string -// KubeconfigStatus represents the kubeconfig status -var KubeconfigStatus = struct { - Configured string - Misconfigured string -}{ - Configured: `Configured`, - Misconfigured: `Misconfigured`, -} +const ( + // Additional states used by kubeconfig + Configured = "Configured" // analagous to state.Saved + Misconfigured = "Misconfigured" // analagous to state.Error +) -// Status represents the status +// Status holds string representations of libmachine.state.State type Status struct { Host string Kubelet string @@ -81,7 +80,6 @@ var statusCmd = &cobra.Command{ exit.UsageT("Cannot use both --output and --format options") } - var returnCode = 0 api, err := machine.NewAPIClient() if err != nil { exit.WithCodeT(exit.Unavailable, "Error getting client: {{.error}}", out.V{"error": err}) @@ -89,81 +87,91 @@ var statusCmd = &cobra.Command{ defer api.Close() machineName := viper.GetString(config.MachineProfile) - - hostSt, err := cluster.GetHostStatus(api, machineName) + st, err := status(api, machineName) if err != nil { - exit.WithError("Error getting host status", err) - } - - kubeletSt := state.None.String() - kubeconfigSt := state.None.String() - apiserverSt := state.None.String() - - if hostSt == state.Running.String() { - clusterBootstrapper, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper)) - if err != nil { - exit.WithError("Error getting bootstrapper", err) - } - kubeletSt, err = clusterBootstrapper.GetKubeletStatus() - if err != nil { - glog.Warningf("kubelet err: %v", err) - returnCode |= clusterNotRunningStatusFlag - } else if kubeletSt != state.Running.String() { - returnCode |= clusterNotRunningStatusFlag - } - - ip, err := cluster.GetHostDriverIP(api, machineName) - if err != nil { - glog.Errorln("Error host driver ip status:", err) - } - - apiserverPort, err := kubeconfig.Port(machineName) - if err != nil { - // Fallback to presuming default apiserver port - apiserverPort = constants.APIServerPort - } - - apiserverSt, err = clusterBootstrapper.GetAPIServerStatus(ip, apiserverPort) - if err != nil { - glog.Errorln("Error apiserver status:", err) - } else if apiserverSt != state.Running.String() { - returnCode |= clusterNotRunningStatusFlag - } - - ks, err := kubeconfig.IsClusterInConfig(ip, machineName) - if err != nil { - glog.Errorln("Error kubeconfig status:", err) - } - if ks { - kubeconfigSt = KubeconfigStatus.Configured - } else { - kubeconfigSt = KubeconfigStatus.Misconfigured - returnCode |= k8sNotRunningStatusFlag - } - } else { - returnCode |= minikubeNotRunningStatusFlag - } - - status := Status{ - Host: hostSt, - Kubelet: kubeletSt, - APIServer: apiserverSt, - Kubeconfig: kubeconfigSt, + glog.Errorf("status error: %v", err) } switch strings.ToLower(output) { case "text": - printStatusText(status) + printStatusText(st) case "json": - printStatusJSON(status) + printStatusJSON(st) default: exit.WithCodeT(exit.BadUsage, fmt.Sprintf("invalid output format: %s. Valid values: 'text', 'json'", output)) } - os.Exit(returnCode) + os.Exit(exitCode(st)) }, } +func exitCode(st *Status) int { + c := 0 + if st.Host != state.Running.String() { + c |= minikubeNotRunningStatusFlag + } + if st.APIServer != state.Running.String() || st.Kubelet != state.Running.String() { + c |= clusterNotRunningStatusFlag + } + if st.Kubeconfig != Configured { + c |= k8sNotRunningStatusFlag + } + return c +} + +func status(api libmachine.API, name string) (*Status, error) { + st := &Status{} + hs, err := cluster.GetHostStatus(api, name) + if err != nil { + return st, errors.Wrap(err, "host") + } + st.Host = hs + if st.Host != state.Running.String() { + return st, nil + } + + bs, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper)) + if err != nil { + return st, errors.Wrap(err, "bootstrapper") + } + + st.Kubelet, err = bs.GetKubeletStatus() + if err != nil { + glog.Warningf("kubelet err: %v", err) + st.Kubelet = state.Error.String() + } + + ip, err := cluster.GetHostDriverIP(api, name) + if err != nil { + glog.Errorln("Error host driver ip status:", err) + st.APIServer = state.Error.String() + return st, err + } + + port, err := kubeconfig.Port(name) + if err != nil { + glog.Warningf("unable to get port: %v", err) + port = constants.APIServerPort + } + + st.APIServer, err = bs.GetAPIServerStatus(ip, port) + if err != nil { + glog.Errorln("Error apiserver status:", err) + st.APIServer = state.Error.String() + } + + ks, err := kubeconfig.IsClusterInConfig(ip, name) + if err != nil { + glog.Errorln("Error kubeconfig status:", err) + } + if ks { + st.Kubeconfig = Configured + } else { + st.Kubeconfig = Misconfigured + } + return st, nil +} + func init() { statusCmd.Flags().StringVarP(&statusFormat, "format", "f", defaultStatusFormat, `Go template format string for the status output. The format for Go templates can be found here: https://golang.org/pkg/text/template/ @@ -172,25 +180,24 @@ For the list accessible variables for the template, see the struct values here: `minikube status --output OUTPUT. json, text`) } -var printStatusText = func(status Status) { +var printStatusText = func(st *Status) { tmpl, err := template.New("status").Parse(statusFormat) if err != nil { exit.WithError("Error creating status template", err) } - err = tmpl.Execute(os.Stdout, status) + err = tmpl.Execute(os.Stdout, st) if err != nil { exit.WithError("Error executing status template", err) } - if status.Kubeconfig == KubeconfigStatus.Misconfigured { + if st.Kubeconfig == Misconfigured { out.WarningT("Warning: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`") } } -var printStatusJSON = func(status Status) { - - jsonString, err := json.Marshal(status) +var printStatusJSON = func(st *Status) { + js, err := json.Marshal(st) if err != nil { exit.WithError("Error converting status to json", err) } - out.String(string(jsonString)) + out.String(string(js)) } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 323e5e7609ab..61f8b6969758 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -18,6 +18,7 @@ package kubeadm import ( "os/exec" + "path" "fmt" "net" @@ -77,9 +78,11 @@ func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) { func (k *Bootstrapper) GetKubeletStatus() (string, error) { rr, err := k.c.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet")) if err != nil { - return "", errors.Wrapf(err, "getting kublet status. command: %q", rr.Command()) + // Do not return now, as we still have parsing to do! + glog.Warningf("%s returned error: %v", rr.Command(), err) } s := strings.TrimSpace(rr.Stdout.String()) + glog.Infof("kubelet is-active: %s", s) switch s { case "active": return state.Running.String(), nil @@ -93,6 +96,24 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) { // GetAPIServerStatus returns the api-server status func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) { + // sudo, in case hidepid is set + rr, err := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver")) + if err != nil { + return state.Stopped.String(), nil + } + pid := strings.TrimSpace(rr.Stdout.String()) + + // Check if apiserver is in an uninteruptible sleep + rr, err = k.c.RunCmd(exec.Command("sudo", "cut", "-d", " ", "-f3", path.Join("/proc", pid, "stat"))) + if err != nil { + return state.Error.String(), err + } + st := strings.TrimSpace(rr.Stdout.String()) + glog.Infof("apiserver process state: %s", st) + if st == "S" { + return state.Paused.String(), nil + } + return kverify.APIServerStatus(ip, apiserverPort) } From 50cd0dbfba630ddcb9f12c07817dd0a7353c60d8 Mon Sep 17 00:00:00 2001 From: tstromberg Date: Wed, 22 Jan 2020 13:55:01 -0800 Subject: [PATCH 08/14] First stab at freezer checks --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 25 ++++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 35044773886d..4582c6dbfa54 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -106,14 +106,29 @@ func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, } pid := strings.TrimSpace(rr.Stdout.String()) - // Check if apiserver is in an uninteruptible sleep - rr, err = k.c.RunCmd(exec.Command("sudo", "cut", "-d", " ", "-f3", path.Join("/proc", pid, "stat"))) + // Get the freezer cgroup entry for this pid + rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[-09]+:freezer:", path.Join("/proc", pid, "cgroup"))) if err != nil { return state.Error.String(), err } - st := strings.TrimSpace(rr.Stdout.String()) - glog.Infof("apiserver process state: %s", st) - if st == "S" { + freezer := strings.TrimSpace(rr.Stdout.String()) + glog.Infof("apiserver freezer: %q", freezer) + + fparts := strings.Split(freezer, ":") + if len(fparts) == 3 { + glog.Warningf("unable to parse freezer: %s", freezer) + return kverify.APIServerStatus(ip, apiserverPort) + } + + rr, err = k.c.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state"))) + if err != nil { + glog.Errorf("unable to get freezer state: %s", rr.Stderr) + return kverify.APIServerStatus(ip, apiserverPort) + } + + fs := strings.TrimSpace(rr.Stdout.String()) + glog.Infof("freezer state: %q", fs) + if fs == "FREEZING" || fs == "FROZEN" { return state.Paused.String(), nil } From 62a72f44e89fb08b9032b1ec3b1cf361212a73dd Mon Sep 17 00:00:00 2001 From: tstromberg Date: Wed, 22 Jan 2020 14:21:34 -0800 Subject: [PATCH 09/14] Address feedback --- cmd/minikube/cmd/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 171cf82e5ad9..471986c31f8c 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -48,7 +48,7 @@ const ( Misconfigured = "Misconfigured" // analagous to state.Error ) -// Status holds string representations of libmachine.state.State +// Status holds string representations of component states type Status struct { Host string Kubelet string From b81960e37c47f25bdb2b0f3e2b71883e24040e64 Mon Sep 17 00:00:00 2001 From: tstromberg Date: Wed, 22 Jan 2020 14:28:18 -0800 Subject: [PATCH 10/14] Fix none driver --- pkg/drivers/none/none.go | 64 +++++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index 9546cee2d625..c11a175ac6eb 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -19,17 +19,19 @@ package none import ( "fmt" "os/exec" + "strings" + "time" "github.com/docker/machine/libmachine/drivers" "github.com/docker/machine/libmachine/state" "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/net" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" pkgdrivers "k8s.io/minikube/pkg/drivers" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/vmpath" + "k8s.io/minikube/pkg/util/retry" ) // cleanupPaths are paths to be removed by cleanup, and are used by both kubeadm and minikube. @@ -121,7 +123,7 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { - if err := kubelet.Check(d.exec); err != nil { + if err := checkKubelet(d.exec); err != nil { glog.Infof("kubelet not running: %v", err) return state.Stopped, nil } @@ -130,12 +132,12 @@ func (d *Driver) GetState() (state.State, error) { // Kill stops a host forcefully, including any containers that we are managing. func (d *Driver) Kill() error { - if err := kubelet.TryStopKubelet(d.exec); err != nil { + if err := stopKubelet(d.exec); err != nil { return errors.Wrap(err, "kubelet") } // First try to gracefully stop containers - containers, err := d.runtime.ListContainers("") + containers, err := d.runtime.ListContainers(cruntime.ListOptions{}) if err != nil { return errors.Wrap(err, "containers") } @@ -147,7 +149,7 @@ func (d *Driver) Kill() error { return errors.Wrap(err, "stop") } - containers, err = d.runtime.ListContainers("") + containers, err = d.runtime.ListContainers(cruntime.ListOptions{}) if err != nil { return errors.Wrap(err, "containers") } @@ -175,7 +177,7 @@ func (d *Driver) Remove() error { // Restart a host func (d *Driver) Restart() error { - return kubelet.Restart(d.exec) + return restartKubelet(d.exec) } // Start a host @@ -194,10 +196,10 @@ func (d *Driver) Start() error { // Stop a host gracefully, including any containers that we are managing. func (d *Driver) Stop() error { - if err := kubelet.Stop(d.exec); err != nil { + if err := stopKubelet(d.exec); err != nil { return err } - containers, err := d.runtime.ListContainers("") + containers, err := d.runtime.ListContainers(cruntime.ListOptions{}) if err != nil { return errors.Wrap(err, "containers") } @@ -213,3 +215,49 @@ func (d *Driver) Stop() error { func (d *Driver) RunSSHCommandFromDriver() error { return fmt.Errorf("driver does not support ssh commands") } + +// stopKubelet idempotently stops the kubelet +func stopKubelet(cr command.Runner) error { + glog.Infof("stopping kubelet.service ...") + stop := func() error { + cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service") + if rr, err := cr.RunCmd(cmd); err != nil { + glog.Errorf("temporary error for %q : %v", rr.Command(), err) + } + cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet") + rr, err := cr.RunCmd(cmd) + if err != nil { + glog.Errorf("temporary error: for %q : %v", rr.Command(), err) + } + if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") { + return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String()) + } + return nil + } + + if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil { + return errors.Wrapf(err, "error stopping kubelet") + } + + return nil +} + +// restartKubelet restarts the kubelet +func restartKubelet(cr command.Runner) error { + glog.Infof("restarting kubelet.service ...") + c := exec.Command("sudo", "systemctl", "restart", "kubelet.service") + if _, err := cr.RunCmd(c); err != nil { + return err + } + return nil +} + +// checkKubelet returns an error if the kubelet is not running. +func checkKubelet(cr command.Runner) error { + glog.Infof("checking for running kubelet ...") + c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "check kubelet") + } + return nil +} From 22de8c110ad7d75f6ff8f3584cbbffef4db27874 Mon Sep 17 00:00:00 2001 From: tstromberg Date: Wed, 22 Jan 2020 15:07:44 -0800 Subject: [PATCH 11/14] Return the number of containers paused --- cmd/minikube/cmd/pause.go | 6 ++--- cmd/minikube/cmd/unpause.go | 12 ++++++--- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 11 +++++---- pkg/minikube/cluster/pause.go | 26 +++++++++++--------- pkg/minikube/cruntime/docker.go | 2 ++ 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go index dc9e3b61d8ba..ba805c0adb62 100644 --- a/cmd/minikube/cmd/pause.go +++ b/cmd/minikube/cmd/pause.go @@ -88,15 +88,15 @@ func runPause(cmd *cobra.Command, args []string) { } } - err = cluster.Pause(cr, r, namespaces) + ids, err := cluster.Pause(cr, r, namespaces) if err != nil { exit.WithError("Pause", err) } if namespaces == nil { - out.T(out.Pause, "Paused all namespaces in the '{{.name}}' cluster", out.V{"name": cc.Name}) + out.T(out.Unpause, "Paused kubelet and {{.count}} containers", out.V{"count": len(ids)}) } else { - out.T(out.Pause, "Paused the following namespaces in '{{.name}}': {{.namespaces}}", out.V{"name": cc.Name, "namespaces": strings.Join(namespaces, ", ")}) + out.T(out.Unpause, "Paused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")}) } } diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go index 7791c54e0b8f..6a38f0948b46 100644 --- a/cmd/minikube/cmd/unpause.go +++ b/cmd/minikube/cmd/unpause.go @@ -18,6 +18,7 @@ package cmd import ( "os" + "strings" "github.com/golang/glog" "github.com/spf13/cobra" @@ -78,12 +79,17 @@ var unpauseCmd = &cobra.Command{ } } - err = cluster.Unpause(cr, r, namespaces) - + ids, err := cluster.Unpause(cr, r, namespaces) if err != nil { exit.WithError("Pause", err) } - out.T(out.Unpause, "The '{{.name}}' cluster is now unpaused", out.V{"name": cc.Name}) + + if namespaces == nil { + out.T(out.Pause, "Unpaused kubelet and {{.count}} containers", out.V{"count": len(ids)}) + } else { + out.T(out.Pause, "Unpaused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")}) + } + }, } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 4582c6dbfa54..ad470c47dd78 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -107,16 +107,17 @@ func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, pid := strings.TrimSpace(rr.Stdout.String()) // Get the freezer cgroup entry for this pid - rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[-09]+:freezer:", path.Join("/proc", pid, "cgroup"))) + rr, err = k.c.RunCmd(exec.Command("sudo", "egrep", "^[0-9]+:freezer:", path.Join("/proc", pid, "cgroup"))) if err != nil { - return state.Error.String(), err + glog.Warningf("unable to find freezer cgroup: %v", err) + return kverify.APIServerStatus(ip, apiserverPort) + } freezer := strings.TrimSpace(rr.Stdout.String()) glog.Infof("apiserver freezer: %q", freezer) - fparts := strings.Split(freezer, ":") - if len(fparts) == 3 { - glog.Warningf("unable to parse freezer: %s", freezer) + if len(fparts) != 3 { + glog.Warningf("unable to parse freezer - found %d parts: %s", len(fparts), freezer) return kverify.APIServerStatus(ip, apiserverPort) } diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go index b963b4b002a4..6e4eaead45e2 100644 --- a/pkg/minikube/cluster/pause.go +++ b/pkg/minikube/cluster/pause.go @@ -32,45 +32,47 @@ var DefaultNamespaces = []string{ } // Pause pauses a Kubernetes cluster -func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) error { +func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) { + ids := []string{} + // Disable the kubelet so it does not attempt to restart paused pods if err := kubelet.Disable(r); err != nil { - return errors.Wrap(err, "kubelet disable") + return ids, errors.Wrap(err, "kubelet disable") } if err := kubelet.Stop(r); err != nil { - return errors.Wrap(err, "kubelet stop") + return ids, errors.Wrap(err, "kubelet stop") } ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running, Namespaces: namespaces}) if err != nil { - return errors.Wrap(err, "list running") + return ids, errors.Wrap(err, "list running") } if len(ids) == 0 { glog.Warningf("no running containers to pause") - return nil + return ids, nil } - return cr.PauseContainers(ids) + return ids, cr.PauseContainers(ids) } // Unpause unpauses a Kubernetes cluster -func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) error { +func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) { ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused, Namespaces: namespaces}) if err != nil { - return errors.Wrap(err, "list paused") + return ids, errors.Wrap(err, "list paused") } if len(ids) == 0 { glog.Warningf("no paused containers found") } else { if err := cr.UnpauseContainers(ids); err != nil { - return errors.Wrap(err, "unpause") + return ids, errors.Wrap(err, "unpause") } } if err := kubelet.Enable(r); err != nil { - return errors.Wrap(err, "kubelet enable") + return ids, errors.Wrap(err, "kubelet enable") } if err := kubelet.Start(r); err != nil { - return errors.Wrap(err, "kubelet start") + return ids, errors.Wrap(err, "kubelet start") } - return nil + return ids, nil } diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index a58cff35b11f..24fb506c94e1 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -151,6 +151,8 @@ func (r *Docker) ListContainers(o ListOptions) ([]string, error) { switch o.State { case All: args = append(args, "-a") + case Running: + args = append(args, "--filter", "status=running") case Paused: args = append(args, "--filter", "status=paused") } From 1a13055d0062a39576cdcc0e63b8f2f8d85322a4 Mon Sep 17 00:00:00 2001 From: tstromberg Date: Wed, 22 Jan 2020 15:50:09 -0800 Subject: [PATCH 12/14] Add integration tests --- cmd/minikube/cmd/pause.go | 6 +- cmd/minikube/cmd/status.go | 4 +- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 2 +- pkg/minikube/cluster/pause.go | 6 +- pkg/minikube/cruntime/cruntime_test.go | 89 +++++++++++--------- test/integration/helpers.go | 10 --- test/integration/start_stop_delete_test.go | 47 ++++++++++- 7 files changed, 98 insertions(+), 66 deletions(-) diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go index ba805c0adb62..54385a4d5c21 100644 --- a/cmd/minikube/cmd/pause.go +++ b/cmd/minikube/cmd/pause.go @@ -82,10 +82,8 @@ func runPause(cmd *cobra.Command, args []string) { glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings()) if allNamespaces { namespaces = nil //all - } else { - if len(namespaces) == 0 { - exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces") - } + } else if len(namespaces) == 0 { + exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces") } ids, err := cluster.Pause(cr, r, namespaces) diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 471986c31f8c..ba8e93c9e62f 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -44,8 +44,8 @@ var output string const ( // Additional states used by kubeconfig - Configured = "Configured" // analagous to state.Saved - Misconfigured = "Misconfigured" // analagous to state.Error + Configured = "Configured" // analogous to state.Saved + Misconfigured = "Misconfigured" // analogous to state.Error ) // Status holds string representations of component states diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index ad470c47dd78..5569dc67bacd 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -123,7 +123,7 @@ func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, rr, err = k.c.RunCmd(exec.Command("sudo", "cat", path.Join("/sys/fs/cgroup/freezer", fparts[2], "freezer.state"))) if err != nil { - glog.Errorf("unable to get freezer state: %s", rr.Stderr) + glog.Errorf("unable to get freezer state: %s", rr.Stderr.String()) return kverify.APIServerStatus(ip, apiserverPort) } diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go index 6e4eaead45e2..d2edcb18882d 100644 --- a/pkg/minikube/cluster/pause.go +++ b/pkg/minikube/cluster/pause.go @@ -62,10 +62,8 @@ func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]stri if len(ids) == 0 { glog.Warningf("no paused containers found") - } else { - if err := cr.UnpauseContainers(ids); err != nil { - return ids, errors.Wrap(err, "unpause") - } + } else if err := cr.UnpauseContainers(ids); err != nil { + return ids, errors.Wrap(err, "unpause") } if err := kubelet.Enable(r); err != nil { diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index cb92244ce37c..7764e1b7140a 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -118,9 +118,9 @@ func TestKubeletOptions(t *testing.T) { type serviceState int const ( - Exited serviceState = iota - Running - Restarted + SvcExited serviceState = iota + SvcRunning + SvcRestarted ) // FakeRunner is a command runner that isn't very smart. @@ -262,6 +262,7 @@ func (f *FakeRunner) containerd(args []string, _ bool) (string, error) { // crictl is a fake implementation of crictl func (f *FakeRunner) crictl(args []string, _ bool) (string, error) { + f.t.Logf("crictl args: %s", args) switch cmd := args[0]; cmd { case "info": return `{ @@ -273,9 +274,21 @@ func (f *FakeRunner) crictl(args []string, _ bool) (string, error) { "golang": "go1.11.13" }`, nil case "ps": + fmt.Printf("args %d: %v\n", len(args), args) + if len(args) != 4 { + f.t.Logf("crictl all") + ids := []string{} + for id := range f.containers { + ids = append(ids, id) + } + f.t.Logf("fake crictl: Found containers: %v", ids) + return strings.Join(ids, "\n"), nil + } + // crictl ps -a --name=apiserver --state=Running --quiet - if args[1] == "-a" && strings.HasPrefix(args[2], "--name") { - fname := strings.Split(args[2], "=")[1] + if args[1] == "-a" && strings.HasPrefix(args[3], "--name") { + fname := strings.Split(args[3], "=")[1] + f.t.Logf("crictl filter for %s", fname) ids := []string{} f.t.Logf("fake crictl: Looking for containers matching %q", fname) for id, cname := range f.containers { @@ -285,14 +298,6 @@ func (f *FakeRunner) crictl(args []string, _ bool) (string, error) { } f.t.Logf("fake crictl: Found containers: %v", ids) return strings.Join(ids, "\n"), nil - } else if args[1] == "-a" { - ids := []string{} - for id := range f.containers { - ids = append(ids, id) - } - f.t.Logf("fake crictl: Found containers: %v", ids) - return strings.Join(ids, "\n"), nil - } case "stop": for _, id := range args[1:] { @@ -341,23 +346,23 @@ func (f *FakeRunner) systemctl(args []string, root bool) (string, error) { // no if !root { return out, fmt.Errorf("not root") } - f.services[svc] = Exited + f.services[svc] = SvcExited f.t.Logf("fake systemctl: stopped %s", svc) case "start": if !root { return out, fmt.Errorf("not root") } - f.services[svc] = Running + f.services[svc] = SvcRunning f.t.Logf("fake systemctl: started %s", svc) case "restart": if !root { return out, fmt.Errorf("not root") } - f.services[svc] = Restarted - f.t.Logf("fake systemctl: restarted %s", svc) + f.services[svc] = SvcRestarted + f.t.Logf("fake systemctl: SvcRestarted %s", svc) case "is-active": f.t.Logf("fake systemctl: %s is-status: %v", svc, state) - if state == Running { + if state == SvcRunning { return out, nil } return out, fmt.Errorf("%s in state: %v", svc, state) @@ -403,11 +408,11 @@ func TestVersion(t *testing.T) { // defaultServices reflects the default boot state for the minikube VM var defaultServices = map[string]serviceState{ - "docker": Running, - "docker.socket": Running, - "crio": Exited, - "crio-shutdown": Exited, - "containerd": Exited, + "docker": SvcRunning, + "docker.socket": SvcRunning, + "crio": SvcExited, + "crio-shutdown": SvcExited, + "containerd": SvcExited, } func TestDisable(t *testing.T) { @@ -446,25 +451,25 @@ func TestEnable(t *testing.T) { want map[string]serviceState }{ {"docker", map[string]serviceState{ - "docker": Running, - "docker.socket": Running, - "containerd": Exited, - "crio": Exited, - "crio-shutdown": Exited, + "docker": SvcRunning, + "docker.socket": SvcRunning, + "containerd": SvcExited, + "crio": SvcExited, + "crio-shutdown": SvcExited, }}, {"containerd", map[string]serviceState{ - "docker": Exited, - "docker.socket": Exited, - "containerd": Restarted, - "crio": Exited, - "crio-shutdown": Exited, + "docker": SvcExited, + "docker.socket": SvcExited, + "containerd": SvcRestarted, + "crio": SvcExited, + "crio-shutdown": SvcExited, }}, {"crio", map[string]serviceState{ - "docker": Exited, - "docker.socket": Exited, - "containerd": Exited, - "crio": Restarted, - "crio-shutdown": Exited, + "docker": SvcExited, + "docker.socket": SvcExited, + "containerd": SvcExited, + "crio": SvcRestarted, + "crio-shutdown": SvcExited, }}, } for _, tc := range tests { @@ -516,7 +521,7 @@ func TestContainerFunctions(t *testing.T) { } // Get the list of apiservers - got, err := cr.ListContainers("apiserver") + got, err := cr.ListContainers(ListOptions{Name: "apiserver"}) if err != nil { t.Fatalf("ListContainers: %v", err) } @@ -529,7 +534,7 @@ func TestContainerFunctions(t *testing.T) { if err := cr.StopContainers(got); err != nil { t.Fatalf("stop failed: %v", err) } - got, err = cr.ListContainers("apiserver") + got, err = cr.ListContainers(ListOptions{Name: "apiserver"}) if err != nil { t.Fatalf("ListContainers: %v", err) } @@ -539,7 +544,7 @@ func TestContainerFunctions(t *testing.T) { } // Get the list of everything else. - got, err = cr.ListContainers("") + got, err = cr.ListContainers(ListOptions{}) if err != nil { t.Fatalf("ListContainers: %v", err) } @@ -552,7 +557,7 @@ func TestContainerFunctions(t *testing.T) { if err := cr.KillContainers(got); err != nil { t.Errorf("KillContainers: %v", err) } - got, err = cr.ListContainers("") + got, err = cr.ListContainers(ListOptions{}) if err != nil { t.Fatalf("ListContainers: %v", err) } diff --git a/test/integration/helpers.go b/test/integration/helpers.go index 8d145da091bf..1453f84cdbf0 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -321,16 +321,6 @@ func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, n } } -// Status returns the minikube cluster status as a string -func Status(ctx context.Context, t *testing.T, path string, profile string) string { - t.Helper() - rr, err := Run(t, exec.CommandContext(ctx, path, "status", "--format={{.Host}}", "-p", profile)) - if err != nil { - t.Logf("status error: %v (may be ok)", err) - } - return strings.TrimSpace(rr.Stdout.String()) -} - // MaybeParallel sets that the test should run in parallel func MaybeParallel(t *testing.T) { t.Helper() diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index ad1b6832b5ff..847e518fe9f1 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -36,6 +36,16 @@ import ( "k8s.io/minikube/pkg/minikube/constants" ) +// status returns a minikube component status as a string +func status(ctx context.Context, t *testing.T, path string, profile string, key string) string { + t.Helper() + rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile)) + if err != nil { + t.Logf("status error: %v (may be ok)", err) + } + return strings.TrimSpace(rr.Stdout.String()) +} + func TestStartStop(t *testing.T) { MaybeParallel(t) @@ -135,7 +145,7 @@ func TestStartStop(t *testing.T) { t.Errorf("%s failed: %v", rr.Args, err) } - got := Status(ctx, t, Target(), profile) + got := status(ctx, t, Target(), profile, "Host") if got != state.Stopped.String() { t.Errorf("status = %q; want = %q", got, state.Stopped) } @@ -187,9 +197,40 @@ func TestStartStop(t *testing.T) { t.Fatalf("wait: %v", err) } - got = Status(ctx, t, Target(), profile) + got = status(ctx, t, Target(), profile, "Host") + if got != state.Running.String() { + t.Errorf("host status = %q; want = %q", got, state.Running) + } + + // Can this runtime be paused and unpaused? + rr, err = Run(t, exec.CommandContext(ctx, Target(), "pause", "-p", profile, "--alsologtostderr", "-v=1")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + got = status(ctx, t, Target(), profile, "APIServer") + if got != state.Paused.String() { + t.Errorf("apiserver status = %q; want = %q", got, state.Paused) + } + + got = status(ctx, t, Target(), profile, "Kubelet") + if got != state.Stopped.String() { + t.Errorf("kubelet status = %q; want = %q", got, state.Stopped) + } + + rr, err = Run(t, exec.CommandContext(ctx, Target(), "unpause", "-p", profile, "--alsologtostderr", "-v=1")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + got = status(ctx, t, Target(), profile, "APIServer") + if got != state.Running.String() { + t.Errorf("apiserver status = %q; want = %q", got, state.Running) + } + + got = status(ctx, t, Target(), profile, "Kubelet") if got != state.Running.String() { - t.Errorf("status = %q; want = %q", got, state.Running) + t.Errorf("kubelet status = %q; want = %q", got, state.Running) } if *cleanup { From 9fc5db914c248244856034d4b9b7d6f6405250ba Mon Sep 17 00:00:00 2001 From: tstromberg Date: Wed, 22 Jan 2020 16:39:26 -0800 Subject: [PATCH 13/14] Decrease cylomatic complexity of TestStartStop --- test/integration/start_stop_delete_test.go | 209 +++++++++++---------- 1 file changed, 113 insertions(+), 96 deletions(-) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 847e518fe9f1..2ea0a44aa840 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -100,44 +100,14 @@ func TestStartStop(t *testing.T) { startArgs := append([]string{"start", "-p", profile, "--alsologtostderr", "-v=3", "--wait=true"}, tc.args...) startArgs = append(startArgs, StartArgs()...) startArgs = append(startArgs, fmt.Sprintf("--kubernetes-version=%s", tc.version)) + rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...)) if err != nil { - // Fatal so that we may collect logs before stop/delete steps t.Fatalf("%s failed: %v", rr.Args, err) } - // SADNESS: 0/1 nodes are available: 1 node(s) had taints that the pod didn't tolerate. - if strings.Contains(tc.name, "cni") { - t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(") - } else { - // schedule a pod to assert persistence - rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml"))) - if err != nil { - t.Fatalf("%s failed: %v", rr.Args, err) - } - - // 8 minutes, because 4 is not enough for images to pull in all cases. - names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 8*time.Minute) - if err != nil { - t.Fatalf("wait: %v", err) - } - - // Use this pod to confirm that the runtime resource limits are sane - rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "ulimit -n")) - if err != nil { - t.Fatalf("ulimit: %v", err) - } - - got, err := strconv.ParseInt(strings.TrimSpace(rr.Stdout.String()), 10, 64) - if err != nil { - t.Errorf("ParseInt(%q): %v", rr.Stdout.String(), err) - } - - // Arbitrary value set by some container runtimes. If higher, apps like MySQL may make bad decisions. - expected := int64(1048576) - if got != expected { - t.Errorf("'ulimit -n' returned %d, expected %d", got, expected) - } + if !strings.Contains(tc.name, "cni") { + testPodScheduling(ctx, t, profile) } rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile, "--alsologtostderr", "-v=3")) @@ -156,41 +126,6 @@ func TestStartStop(t *testing.T) { t.Fatalf("%s failed: %v", rr.Args, err) } - // Make sure that kubeadm did not need to pull in additional images - if !NoneDriver() { - rr, err = Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "sudo crictl images -o json")) - if err != nil { - t.Errorf("%s failed: %v", rr.Args, err) - } - jv := map[string][]struct { - Tags []string `json:"repoTags"` - }{} - err = json.Unmarshal(rr.Stdout.Bytes(), &jv) - if err != nil { - t.Errorf("images unmarshal: %v", err) - } - gotImages := []string{} - for _, img := range jv["images"] { - for _, i := range img.Tags { - if defaultImage(i) { - // Remove docker.io for naming consistency between container runtimes - gotImages = append(gotImages, strings.TrimPrefix(i, "docker.io/")) - } else { - t.Logf("Found non-minikube image: %s", i) - } - } - } - want, err := images.Kubeadm("", tc.version) - if err != nil { - t.Errorf("kubeadm images: %v", tc.version) - } - sort.Strings(want) - sort.Strings(gotImages) - if diff := cmp.Diff(want, gotImages); diff != "" { - t.Errorf("%s images mismatch (-want +got):\n%s", tc.version, diff) - } - } - if strings.Contains(tc.name, "cni") { t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(") } else if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 4*time.Minute); err != nil { @@ -202,36 +137,11 @@ func TestStartStop(t *testing.T) { t.Errorf("host status = %q; want = %q", got, state.Running) } - // Can this runtime be paused and unpaused? - rr, err = Run(t, exec.CommandContext(ctx, Target(), "pause", "-p", profile, "--alsologtostderr", "-v=1")) - if err != nil { - t.Fatalf("%s failed: %v", rr.Args, err) - } - - got = status(ctx, t, Target(), profile, "APIServer") - if got != state.Paused.String() { - t.Errorf("apiserver status = %q; want = %q", got, state.Paused) - } - - got = status(ctx, t, Target(), profile, "Kubelet") - if got != state.Stopped.String() { - t.Errorf("kubelet status = %q; want = %q", got, state.Stopped) - } - - rr, err = Run(t, exec.CommandContext(ctx, Target(), "unpause", "-p", profile, "--alsologtostderr", "-v=1")) - if err != nil { - t.Fatalf("%s failed: %v", rr.Args, err) - } - - got = status(ctx, t, Target(), profile, "APIServer") - if got != state.Running.String() { - t.Errorf("apiserver status = %q; want = %q", got, state.Running) + if !NoneDriver() { + testPulledImages(ctx, t, profile, tc.version) } - got = status(ctx, t, Target(), profile, "Kubelet") - if got != state.Running.String() { - t.Errorf("kubelet status = %q; want = %q", got, state.Running) - } + testPause(ctx, t, profile) if *cleanup { // Normally handled by cleanuprofile, but not fatal there @@ -245,6 +155,113 @@ func TestStartStop(t *testing.T) { }) } +// testPodScheduling asserts that this configuration can schedule new pods +func testPodScheduling(ctx context.Context, t *testing.T, profile string) { + t.Helper() + + // schedule a pod to assert persistence + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml"))) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + // 8 minutes, because 4 is not enough for images to pull in all cases. + names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 8*time.Minute) + if err != nil { + t.Fatalf("wait: %v", err) + } + + // Use this pod to confirm that the runtime resource limits are sane + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "ulimit -n")) + if err != nil { + t.Fatalf("ulimit: %v", err) + } + + got, err := strconv.ParseInt(strings.TrimSpace(rr.Stdout.String()), 10, 64) + if err != nil { + t.Errorf("ParseInt(%q): %v", rr.Stdout.String(), err) + } + + // Arbitrary value set by some container runtimes. If higher, apps like MySQL may make bad decisions. + expected := int64(1048576) + if got != expected { + t.Errorf("'ulimit -n' returned %d, expected %d", got, expected) + } +} + +// testPulledImages asserts that this configuration pulls only expected images +func testPulledImages(ctx context.Context, t *testing.T, profile string, version string) { + t.Helper() + + rr, err := Run(t, exec.CommandContext(ctx, Target(), "ssh", "-p", profile, "sudo crictl images -o json")) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + jv := map[string][]struct { + Tags []string `json:"repoTags"` + }{} + err = json.Unmarshal(rr.Stdout.Bytes(), &jv) + if err != nil { + t.Errorf("images unmarshal: %v", err) + } + gotImages := []string{} + for _, img := range jv["images"] { + for _, i := range img.Tags { + if defaultImage(i) { + // Remove docker.io for naming consistency between container runtimes + gotImages = append(gotImages, strings.TrimPrefix(i, "docker.io/")) + } else { + t.Logf("Found non-minikube image: %s", i) + } + } + } + want, err := images.Kubeadm("", version) + if err != nil { + t.Errorf("kubeadm images: %v", version) + } + sort.Strings(want) + sort.Strings(gotImages) + if diff := cmp.Diff(want, gotImages); diff != "" { + t.Errorf("%s images mismatch (-want +got):\n%s", version, diff) + } +} + +// testPause asserts that this configuration can be paused and unpaused +func testPause(ctx context.Context, t *testing.T, profile string) { + t.Helper() + + rr, err := Run(t, exec.CommandContext(ctx, Target(), "pause", "-p", profile, "--alsologtostderr", "-v=1")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + got := status(ctx, t, Target(), profile, "APIServer") + if got != state.Paused.String() { + t.Errorf("apiserver status = %q; want = %q", got, state.Paused) + } + + got = status(ctx, t, Target(), profile, "Kubelet") + if got != state.Stopped.String() { + t.Errorf("kubelet status = %q; want = %q", got, state.Stopped) + } + + rr, err = Run(t, exec.CommandContext(ctx, Target(), "unpause", "-p", profile, "--alsologtostderr", "-v=1")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + got = status(ctx, t, Target(), profile, "APIServer") + if got != state.Running.String() { + t.Errorf("apiserver status = %q; want = %q", got, state.Running) + } + + got = status(ctx, t, Target(), profile, "Kubelet") + if got != state.Running.String() { + t.Errorf("kubelet status = %q; want = %q", got, state.Running) + } + +} + // defaultImage returns true if this image is expected in a default minikube install func defaultImage(name string) bool { if strings.Contains(name, ":latest") { From a9bc13b403dc03e4d5155288f4fcd6dbcdfa3d20 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 23 Jan 2020 15:44:30 -0800 Subject: [PATCH 14/14] Address pause code review feedback --- cmd/minikube/cmd/pause.go | 3 +-- cmd/minikube/cmd/status.go | 32 +++++++++++++++++++------------- cmd/minikube/cmd/unpause.go | 3 +-- pkg/minikube/cluster/pause.go | 1 + 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go index 54385a4d5c21..e44cc79ed7c1 100644 --- a/cmd/minikube/cmd/pause.go +++ b/cmd/minikube/cmd/pause.go @@ -73,8 +73,7 @@ func runPause(cmd *cobra.Command, args []string) { exit.WithError("Failed to get command runner", err) } - config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r} - cr, err := cruntime.New(config) + cr, err := cruntime.New(cruntime.Config{Type: cc.ContainerRuntime, Runner: r}) if err != nil { exit.WithError("Failed runtime", err) } diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index ba8e93c9e62f..64a20bcddfd1 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -19,6 +19,7 @@ package cmd import ( "encoding/json" "fmt" + "io" "os" "strings" "text/template" @@ -94,9 +95,13 @@ var statusCmd = &cobra.Command{ switch strings.ToLower(output) { case "text": - printStatusText(st) + if err := statusText(st, os.Stdout); err != nil { + exit.WithError("status text failure", err) + } case "json": - printStatusJSON(st) + if err := statusJSON(st, os.Stdout); err != nil { + exit.WithError("status json failure", err) + } default: exit.WithCodeT(exit.BadUsage, fmt.Sprintf("invalid output format: %s. Valid values: 'text', 'json'", output)) } @@ -160,14 +165,13 @@ func status(api libmachine.API, name string) (*Status, error) { st.APIServer = state.Error.String() } + st.Kubeconfig = Misconfigured ks, err := kubeconfig.IsClusterInConfig(ip, name) if err != nil { glog.Errorln("Error kubeconfig status:", err) } if ks { st.Kubeconfig = Configured - } else { - st.Kubeconfig = Misconfigured } return st, nil } @@ -180,24 +184,26 @@ For the list accessible variables for the template, see the struct values here: `minikube status --output OUTPUT. json, text`) } -var printStatusText = func(st *Status) { +func statusText(st *Status, w io.Writer) error { tmpl, err := template.New("status").Parse(statusFormat) if err != nil { - exit.WithError("Error creating status template", err) + return err } - err = tmpl.Execute(os.Stdout, st) - if err != nil { - exit.WithError("Error executing status template", err) + if err := tmpl.Execute(w, st); err != nil { + return err } if st.Kubeconfig == Misconfigured { - out.WarningT("Warning: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`") + _, err := w.Write([]byte("\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`\n")) + return err } + return nil } -var printStatusJSON = func(st *Status) { +func statusJSON(st *Status, w io.Writer) error { js, err := json.Marshal(st) if err != nil { - exit.WithError("Error converting status to json", err) + return err } - out.String(string(js)) + _, err = w.Write(js) + return err } diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go index 6a38f0948b46..a0d282072131 100644 --- a/cmd/minikube/cmd/unpause.go +++ b/cmd/minikube/cmd/unpause.go @@ -64,8 +64,7 @@ var unpauseCmd = &cobra.Command{ exit.WithError("Failed to get command runner", err) } - config := cruntime.Config{Type: cc.ContainerRuntime, Runner: r} - cr, err := cruntime.New(config) + cr, err := cruntime.New(cruntime.Config{Type: cc.ContainerRuntime, Runner: r}) if err != nil { exit.WithError("Failed runtime", err) } diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go index d2edcb18882d..2f98cf6de36d 100644 --- a/pkg/minikube/cluster/pause.go +++ b/pkg/minikube/cluster/pause.go @@ -29,6 +29,7 @@ var DefaultNamespaces = []string{ "kube-system", "kubernetes-dashboard", "storage-gluster", + "istio-operator", } // Pause pauses a Kubernetes cluster