Merge remote-tracking branch 'origin/master' into check-gcr-access

cmd/minikube/cmd/cache.go

@@ -66,20 +66,43 @@ var deleteCacheCmd = &cobra.Command{
 	},
 }
 
-// LoadCachedImagesInConfigFile loads the images currently in the config file (minikube start)
-func LoadCachedImagesInConfigFile() error {
+func imagesInConfigFile() ([]string, error) {
 	configFile, err := config.ReadConfig()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if values, ok := configFile[constants.Cache]; ok {
 		var images []string
 		for key := range values.(map[string]interface{}) {
 			images = append(images, key)
 		}
-		return machine.CacheAndLoadImages(images)
+		return images, nil
+	}
+	return []string{}, nil
+}
+
+// CacheImagesInConfigFile caches the images currently in the config file (minikube start)
+func CacheImagesInConfigFile() error {
+	images, err := imagesInConfigFile()
+	if err != nil {
+		return err
+	}
+	if len(images) == 0 {
+		return nil
+	}
+	return machine.CacheImages(images, constants.ImageCacheDir)
+}
+
+// LoadCachedImagesInConfigFile loads the images currently in the config file (minikube start)
+func LoadCachedImagesInConfigFile() error {
+	images, err := imagesInConfigFile()
+	if err != nil {
+		return err
+	}
+	if len(images) == 0 {
+		return nil
 	}
-	return nil
+	return machine.CacheAndLoadImages(images)
 }
 
 func init() {

cmd/minikube/cmd/config/util.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -132,6 +133,11 @@ func EnableOrDisableAddon(name string, val string) error {
 	}
 
 	data := assets.GenerateTemplateData(cfg.KubernetesConfig)
+	return enableOrDisableAddonInternal(addon, cmd, data, enable)
+}
+
+func enableOrDisableAddonInternal(addon *assets.Addon, cmd bootstrapper.CommandRunner, data interface{}, enable bool) error {
+	var err error
 	if enable {
 		for _, addon := range addon.Assets {
 			var addonFile assets.CopyableFile

cmd/minikube/cmd/env.go

@@ -26,7 +26,9 @@ import (
 	"text/template"
 
 	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/log"
 	"github.com/docker/machine/libmachine/shell"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/pkg/errors"
@@ -301,13 +303,31 @@ func (EnvNoProxyGetter) GetNoProxyVar() (string, string) {
 	return noProxyVar, noProxyValue
 }
 
+// same as drivers.RunSSHCommandFromDriver, but allows errors
+func runSSHCommandFromDriver(d drivers.Driver, command string) (string, error) {
+	client, err := drivers.GetSSHClientFromDriver(d)
+	if err != nil {
+		return "", err
+	}
+
+	log.Debugf("About to run SSH command:\n%s", command)
+	output, err := client.Output(command)
+	log.Debugf("SSH cmd err, output: %v: %s", err, output)
+	return output, err
+}
+
+// same as host.RunSSHCommand, but allows errors
+func runSSHCommand(h *host.Host, command string) (string, error) {
+	return runSSHCommandFromDriver(h.Driver, command)
+}
+
 // GetDockerActive checks if Docker is active
 func GetDockerActive(host *host.Host) (bool, error) {
 	statusCmd := `sudo systemctl is-active docker`
-	status, err := host.RunSSHCommand(statusCmd)
+	status, err := runSSHCommand(host, statusCmd)
 	// systemd returns error code on inactive
 	s := strings.TrimSpace(status)
-	return err == nil && s == "active", err
+	return err == nil && s == "active", nil
 }
 
 // envCmd represents the docker-env command

cmd/minikube/cmd/start.go

@@ -91,6 +91,7 @@ const (
 	hidden                = "hidden"
 	embedCerts            = "embed-certs"
 	noVTXCheck            = "no-vtx-check"
+	downloadOnly          = "download-only"
 )
 
 var (
@@ -140,6 +141,7 @@ func init() {
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\"")
 	startCmd.Flags().String(featureGates, "", "A set of key=value pairs that describe feature gates for alpha/experimental features.")
+	startCmd.Flags().Bool(downloadOnly, false, "If true, only download and cache files for later use - don't install or start anything.")
 	startCmd.Flags().Bool(cacheImages, true, "If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.")
 	startCmd.Flags().Var(&extraOptions, "extra-config",
 		`A set of key=value pairs that describe configuration that may be passed to different components.
@@ -201,11 +203,17 @@ func runStart(cmd *cobra.Command, args []string) {
 		console.OutStyle("success", "using image repository %s", config.KubernetesConfig.ImageRepository)
 	}
 
-	if viper.GetString(vmDriver) == constants.DriverNone {
-		// Optimization: images will be persistently loaded into the host's container runtime, so no need to duplicate work.
+	// For non-"none", the ISO is required to boot, so block until it is downloaded
+	if viper.GetString(vmDriver) != constants.DriverNone {
+		if err := cluster.CacheISO(config.MachineConfig); err != nil {
+			exit.WithError("Failed to cache ISO", err)
+		}
+	} else {
+		// With "none", images are persistently stored in Docker, so internal caching isn't necessary.
 		viper.Set(cacheImages, false)
 	}
 
+	// Now that the ISO is downloaded, pull images in the background while the VM boots.
 	var cacheGroup errgroup.Group
 	beginCacheImages(&cacheGroup, config.KubernetesConfig.ImageRepository, k8sVersion)
 
@@ -219,6 +227,20 @@ func runStart(cmd *cobra.Command, args []string) {
 	if err != nil {
 		exit.WithError("Failed to get machine client", err)
 	}
+
+	// If --download-only, complete the remaining downloads and exit.
+	if viper.GetBool(downloadOnly) {
+		if err := doCacheBinaries(k8sVersion); err != nil {
+			exit.WithError("Failed to cache binaries", err)
+		}
+		waitCacheImages(&cacheGroup)
+		if err := CacheImagesInConfigFile(); err != nil {
+			exit.WithError("Failed to cache images", err)
+		}
+		console.OutStyle("check", "Download complete!")
+		return
+	}
+
 	host, preexisting := startHost(m, config.MachineConfig)
 
 	ip := validateNetwork(host)
@@ -235,12 +257,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	cr := configureRuntimes(host, runner)
 
 	// prepareHostEnvironment uses the downloaded images, so we need to wait for background task completion.
-	if viper.GetBool(cacheImages) {
-		console.OutStyle("waiting", "Waiting for image downloads to complete ...")
-		if err := cacheGroup.Wait(); err != nil {
-			glog.Errorln("Error caching images: ", err)
-		}
-	}
+	waitCacheImages(&cacheGroup)
 
 	bs := prepareHostEnvironment(m, config.KubernetesConfig)
 
@@ -260,16 +277,20 @@ func runStart(cmd *cobra.Command, args []string) {
 		prepareNone()
 	}
 
+	showKubectlConnectInfo(kubeconfig)
+	console.OutStyle("ready", "Done! Thank you for using minikube!")
+}
+
+func showKubectlConnectInfo(kubeconfig *pkgutil.KubeConfigSetup) {
 	if kubeconfig.KeepContext {
 		console.OutStyle("kubectl", "To connect to this cluster, use: kubectl --context=%s", kubeconfig.ClusterName)
 	} else {
 		console.OutStyle("kubectl", "kubectl is now configured to use %q", cfg.GetMachineName())
 	}
-	_, err = exec.LookPath("kubectl")
+	_, err := exec.LookPath("kubectl")
 	if err != nil {
 		console.OutStyle("tip", "For best results, install kubectl: https://kubernetes.io/docs/tasks/tools/install-kubectl/")
 	}
-	console.OutStyle("ready", "Done! Thank you for using minikube!")
 }
 
 func selectImageRepository(config cfg.Config) (bool, string, error) {
@@ -341,6 +362,11 @@ func validateConfig() {
 	}
 }
 
+// doCacheBinaries caches Kubernetes binaries in the foreground
+func doCacheBinaries(k8sVersion string) error {
+	return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper))
+}
+
 // beginCacheImages caches Docker images in the background
 func beginCacheImages(g *errgroup.Group, imageRepository string, k8sVersion string) {
 	if !viper.GetBool(cacheImages) {
@@ -352,6 +378,17 @@ func beginCacheImages(g *errgroup.Group, imageRepository string, k8sVersion stri
 	})
 }
 
+// waitCacheImages blocks until the image cache jobs complete
+func waitCacheImages(g *errgroup.Group) {
+	if !viper.GetBool(cacheImages) {
+		return
+	}
+	console.OutStyle("waiting", "Waiting for image downloads to complete ...")
+	if err := g.Wait(); err != nil {
+		glog.Errorln("Error caching images: ", err)
+	}
+}
+
 // generateConfig generates cfg.Config based on flags and supplied arguments
 func generateConfig(cmd *cobra.Command, k8sVersion string) (cfg.Config, error) {
 	r, err := cruntime.New(cruntime.Config{Type: viper.GetString(containerRuntime)})

deploy/addons/registry/registry-rc.yaml

@@ -23,3 +23,6 @@ spec:
         ports:
         - containerPort: 5000
           protocol: TCP
+        env:
+        - name: REGISTRY_STORAGE_DELETE_ENABLED
+          value: true  

docs/vmdriver-none.md

@@ -73,7 +73,7 @@ We'll cover these in detail below:
 With the `none` driver, minikube will overwrite the following system paths:
 
 * /usr/bin/kubeadm - Updated to match the exact version of Kubernetes selected
-* /usr/bin/kubectl - Updated to match the exact version of Kubernetes selected
+* /usr/bin/kubelet - Updated to match the exact version of Kubernetes selected
 * /etc/kubernetes - configuration files
 
 These paths will be erased when running `minikube delete`:
@@ -101,5 +101,12 @@ Some environment variables may be useful for using the `none` driver:
 * minikube with the `none` driver has a confusing permissions model, as some commands need to be run as root ("start"), and others by a regular user ("dashboard")
 * CoreDNS detects resolver loop, goes into CrashloopBackoff - [#3511](https://github.com/kubernetes/minikube/issues/3511)
 * Some versions of Linux have a version of docker that is newer then what Kubernetes expects. To overwrite this, run minikube with the following parameters: `sudo -E minikube start --vm-driver=none --kubernetes-version v1.11.8 --extra-config kubeadm.ignore-preflight-errors=SystemVerification`
-* On Ubuntu 18.04 (and probably others), because of how `systemd-resolve` is configured by default, one needs to bypass the default `resolv.conf` file and use a different one instead: `sudo -E minikube --vm-driver=none start --extra-config=kubelet.resolv-conf=/run/systemd/resolve/resolv.conf`
+* On Ubuntu 18.04 (and probably others), because of how `systemd-resolve` is configured by default, one needs to bypass the default `resolv.conf` file and use a different one instead.
+  - In this case, you should use this file: `/run/systemd/resolve/resolv.conf`
+  - `sudo -E minikube --vm-driver=none start --extra-config=kubelet.resolv-conf=/run/systemd/resolve/resolv.conf`
+  - Apperently, though, if `resolve.conf` is too big (about 10 lines!!!), one gets the following error: `Waiting for pods: apiserver proxy! Error restarting cluster: wait: waiting for k8s-app=kube-proxy: timed out waiting for the condition`
+  - This error happens in Kubernetes 0.11.x, 0.12.x and 0.13.x, but *not* in 0.14.x
+  - If that's your case, try this:
+  - `grep -E "^nameserver" /run/systemd/resolve/resolv.conf  |head -n 3 > /tmp/resolv.conf && sudo -E minikube --vm-driver=none start --extra-config=kubelet.resolv-conf=/tmp/resolv.conf`
+
 * [Full list of open 'none' driver issues](https://github.com/kubernetes/minikube/labels/co%2Fnone-driver)

pkg/minikube/bootstrapper/bootstrapper.go

@@ -51,6 +51,16 @@ const (
 	BootstrapperTypeKubeadm = "kubeadm"
 )
 
+// GetCachedBinaryList returns the list of binaries
+func GetCachedBinaryList(bootstrapper string) []string {
+	switch bootstrapper {
+	case BootstrapperTypeKubeadm:
+		return constants.GetKubeadmCachedBinaries()
+	default:
+		return []string{}
+	}
+}
+
 // GetCachedImageList returns the list of images for a version
 func GetCachedImageList(imageRepository string, version string, bootstrapper string) []string {
 	switch bootstrapper {