1.18.x persist install test option

changelog/v1.18.5/k8s-e2e-skip-install-update.yaml

@@ -0,0 +1,11 @@
+changelog:
+  - type: FIX
+    issueLink: https://github.com/solo-io/solo-projects/issues/7432
+    resolvesIssue: true
+    description: >-
+      Add `PERSIST_INSTALL` environment variable to control Gloo installation while running e2e tests (both new and old versions).
+      If set to `true`, the the installation of Gloo will be skipped if it is already installed, and will install Gloo if not already
+      installed. When set to `true`, teardown will also be skipped.
+
+      The TEAR_DOWN flag will now also be usable with the new kubernetes e2e tests, and common logic is now beign used to control
+      Gloo installtion and teardown for both new and old e2e tests.

test/helpers/glooctl_debug_dump.go

@@ -8,6 +8,11 @@ import (
 	"github.com/solo-io/go-utils/testutils"
 )
 
+const (
+	// TearDown is used to TearDown assets after a test completes. This is used in kube2e tests to uninstall
+	DefaultTearDown = false
+)
+
 func RegisterGlooDebugLogPrintHandlerAndClearLogs() {
 	_ = os.Remove(cliutil.GetLogsPath())
 	RegisterGlooDebugLogPrintHandler()

test/kube2e/README.md

@@ -80,11 +80,9 @@ The below table contains the environment variables that can be used to configure
 
 | Name             | Default | Description                                                                                                                                                                                                                                        |
 |------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| KUBE2E_TESTS     | gateway | Name of the test suite to be run. Options: `'gateway', 'gloo', 'ingress', 'helm', 'glooctl', 'upgrade', 'istio'`                                                                                                                                   |
-| DEBUG            | 0       | Set to 1 for debug log output                                                                                                                                                                                                                      |
-| WAIT_ON_FAIL     | 0       | Set to 1 to prevent Ginkgo from cleaning up the Gloo Edge installation in case of failure. Useful to exec into inspect resources created by the test. A command to resume the test run (and thus clean up resources) will be logged to the output. |
-| TEAR_DOWN        | false   | Set to true to uninstall Gloo after the test suite completes                                                                                                                                                                                       |
-| RELEASED_VERSION | ''      | Used by nightlies to tests a specific released version. 'LATEST' will find the latest release                                                                                                                                                      |
+| TEAR_DOWN        | true   | Set to true to uninstall Gloo after the test suite completes                                                                                                                                                                                       |
+| SKIP_INSTALL     | false   | Set to true to use previously installed Gloo for the tests                                                                                                                                                                                        |
+| PERSIST_INSTALL  | false   | Set to true to conditionally install Gloo if it is not already installed and leave Gloo installed after the test run. Useful when running tests repeatedly. If TEAR_DOWN or SKIP_INSTALL values are defined (not just using the default values) and conflict with PERSIST_INSTALL behavior, the TEAR_DOWN or SKIP_INSTALL values will be considered to be more specific and will take precedence.                      |
 | CLUSTER_NAME     | kind    | Used to control which Kind cluster to run the tests inside | 
 
 #### Common Test Errors

test/kube2e/gateway/gateway_suite_test.go

@@ -73,7 +73,7 @@ func StartTestHelper() {
 	kubeCli = kubectl.NewCli().WithReceiver(GinkgoWriter)
 
 	// Allow skipping of install step for running multiple times
-	if !kubeutils2.ShouldSkipInstall() {
+	if !kubeutils2.ShouldSkipInstall(testHelper.IsGlooInstalled(ctx)) {
 		installGloo()
 	}
 
@@ -88,7 +88,7 @@ func StartTestHelper() {
 }
 
 func TearDownTestHelper() {
-	if kubeutils2.ShouldTearDown() {
+	if kubeutils2.ShouldTearDown(helpers.DefaultTearDown) {
 		uninstallGloo()
 	}
 	cancel()

test/kube2e/gloo/gloo_suite_test.go

@@ -72,7 +72,7 @@ var _ = BeforeSuite(func() {
 	skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, namespaces))
 
 	// Allow skipping of install step for running multiple times
-	if !glootestutils.ShouldSkipInstall() {
+	if !glootestutils.ShouldSkipInstall(testHelper.IsGlooInstalled(ctx)) {
 		installGloo()
 	}
 
@@ -97,7 +97,7 @@ var _ = BeforeSuite(func() {
 var _ = AfterSuite(func() {
 	defer cancel()
 
-	if glootestutils.ShouldTearDown() {
+	if glootestutils.ShouldTearDown(helpers.DefaultTearDown) {
 		uninstallGloo()
 	}
 })

test/kube2e/helper/install.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/rotisserie/eris"
 	"github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl"
+	"github.com/solo-io/gloo/test/testutils"
 	"github.com/solo-io/go-utils/log"
 	"github.com/solo-io/go-utils/testutils/exec"
 	"github.com/solo-io/k8s-utils/testutils/kube"
@@ -356,3 +357,7 @@ func GenerateVariantValuesFile(variant string) (string, error) {
 
 	return tmpFile.Name(), nil
 }
+
+func (h *SoloTestHelper) IsGlooInstalled(ctx context.Context) bool {
+	return testutils.CheckResourcesOk(ctx, h.InstallNamespace) == nil
+}

test/kubernetes/e2e/README.md

@@ -53,6 +53,17 @@ Some tests may require environment variables to be set. Some required env vars a
 
 - Istio features: Require `ISTIO_VERSION` to be set. The tests running in CI use `ISTIO_VERSION="${ISTIO_VERSION:-1.19.9}"` to default to a specific version of Istio.
 
+### Optional Environment Variables
+| Name             | Default | Description                                                                                                                                                                                                                                        |
+|------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| KUBE2E_TESTS     | gateway | Name of the test suite to be run. Options: `'gateway', 'gloo', 'ingress', 'helm', 'glooctl', 'upgrade', 'istio'`                                                                                                                                   |
+| DEBUG            | 0       | Set to 1 for debug log output                                                                                                                                                                                                                      |
+| TEAR_DOWN        | false   | Set to true to uninstall Gloo after the test suite completes                                                                                                                                                                                       |
+| SKIP_INSTALL     | false   | Set to true to use previously installed Gloo for the tests                                                                                                                                                                                        |
+| PERSIST_INSTALL  | false   | Set to true to conditionally install Gloo if it is not already installed and leave Gloo installed after the test run. Useful when running tests repeatedly. If TEAR_DOWN or SKIP_INSTALL values are defined and conflict with PERSIST_INSTALL behavior, the TEAR_DOWN or SKIP_INSTALL values will be considered to be more specific and will take precedence                      |
+| RELEASED_VERSION | ''      | Used by nightlies to tests a specific released version. 'LATEST' will find the latest release                                                                                                                                                      |
+| CLUSTER_NAME     | kind    | Used to control which Kind cluster to run the tests inside | 
+
 ## Debugging
 
 Refer to the [Debugging guide](./debugging.md) for more information on how to debug tests.

test/kubernetes/e2e/test.go

@@ -21,6 +21,10 @@ import (
 	"github.com/solo-io/gloo/test/testutils"
 )
 
+const (
+	defaultTearDown = true
+)
+
 // MustTestHelper returns the SoloTestHelper used for e2e tests
 // The SoloTestHelper is a wrapper around `glooctl` and we should eventually phase it out
 // in favor of using the exact tool that users rely on
@@ -192,8 +196,13 @@ func (i *TestInstallation) InstallGlooGatewayWithTestHelper(ctx context.Context,
 	i.InstallGlooGateway(ctx, installFn)
 }
 
+func (i *TestInstallation) isGatewayInstalled(ctx context.Context) bool {
+	return i.Assertions.CheckResourcesOk(ctx) == nil
+}
+
 func (i *TestInstallation) InstallGlooGateway(ctx context.Context, installFn func(ctx context.Context) error) {
-	if !testutils.ShouldSkipInstall() {
+
+	if !testutils.ShouldSkipInstall(i.isGatewayInstalled(ctx)) {
 		err := installFn(ctx)
 		i.Assertions.Require.NoError(err)
 		i.Assertions.EventuallyInstallationSucceeded(ctx)
@@ -217,7 +226,7 @@ func (i *TestInstallation) UninstallGlooGatewayWithTestHelper(ctx context.Contex
 }
 
 func (i *TestInstallation) UninstallGlooGateway(ctx context.Context, uninstallFn func(ctx context.Context) error) {
-	if testutils.ShouldSkipInstall() {
+	if !testutils.ShouldTearDown(defaultTearDown) {
 		return
 	}
 	err := uninstallFn(ctx)

test/kubernetes/testutils/assertions/glooctl.go

@@ -5,30 +5,23 @@ import (
 	"time"
 
 	. "github.com/onsi/gomega"
-	"github.com/solo-io/solo-kit/pkg/api/v1/resources/core"
 
-	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check"
-	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/version"
-	"github.com/solo-io/gloo/projects/gloo/cli/pkg/printers"
+
+	"github.com/solo-io/gloo/test/testutils"
 )
 
+// CheckResourcesOk asserts that `glooctl check` succeeds
+func (p *Provider) CheckResourcesOk(ctx context.Context) error {
+	return testutils.CheckResourcesOk(ctx, p.glooGatewayContext.InstallNamespace)
+}
+
 // EventuallyCheckResourcesOk asserts that `glooctl check` eventually responds Ok
 func (p *Provider) EventuallyCheckResourcesOk(ctx context.Context) {
 	p.expectGlooGatewayContextDefined()
 
 	p.Gomega.Eventually(func(innerG Gomega) {
-		contextWithCancel, cancel := context.WithCancel(ctx)
-		defer cancel()
-		opts := &options.Options{
-			Metadata: core.Metadata{
-				Namespace: p.glooGatewayContext.InstallNamespace,
-			},
-			Top: options.Top{
-				Ctx: contextWithCancel,
-			},
-		}
-		err := check.CheckResources(contextWithCancel, printers.P{}, opts)
+		err := p.CheckResourcesOk(ctx)
 		innerG.Expect(err).NotTo(HaveOccurred())
 	}).
 		WithContext(ctx).

test/testutils/env.go

@@ -1,6 +1,8 @@
 package testutils
 
 import (
+	"log"
+
 	"github.com/solo-io/gloo/pkg/utils/envutils"
 )
 
@@ -13,6 +15,10 @@ const (
 	// installation from a previous run
 	SkipInstall = "SKIP_INSTALL"
 
+	// PersistInstall can be used when running Kube suites consecutively. It will install Gloo if it is not installed,
+	// and it won't tear down after the test suite completes
+	PersistInstall = "PERSIST_INSTALL"
+
 	// InstallNamespace is the namespace in which Gloo is installed
 	InstallNamespace = "INSTALL_NAMESPACE"
 
@@ -87,15 +93,39 @@ const (
 
 // ShouldTearDown returns true if any assets that were created before a test (for example Gloo being installed)
 // should be torn down after the test.
-func ShouldTearDown() bool {
-	return IsEnvTruthy(TearDown)
+func ShouldTearDown(defaultTearDown bool) bool {
+	if IsEnvTruthy(TearDown) && IsEnvTruthy(PersistInstall) {
+		log.Printf("Warning: %s and %s are both set to true. This is likely a mistake.  Gloo will be uninstalled.", TearDown, PersistInstall)
+	}
+
+	// Start with the default tear down value
+	tearDown := defaultTearDown
+
+	// If TearDown is defined, use that value
+	if envutils.IsEnvDefined(TearDown) {
+		tearDown = IsEnvTruthy(TearDown)
+	} else if IsEnvTruthy(PersistInstall) {
+		// If TearDown is not defined, and persist install is truthy, don't tear down
+		tearDown = false
+	}
+
+	return tearDown
 }
 
 // ShouldSkipInstall returns true if any assets that need to be created before a test (for example Gloo being installed)
 // should be skipped. This is typically used in tandem with ShouldTearDown when running consecutive tests and skipping
 // both the tear down and install of Gloo Edge.
-func ShouldSkipInstall() bool {
-	return IsEnvTruthy(SkipInstall)
+func ShouldSkipInstall(alreadyInstalled bool) bool {
+	// Skip install if:
+	// - SkipInstall is true
+	// OR
+	// - PersistInstall is true and Gloo is already installed
+
+	// If SkipInstall is true and PersistInstall is false, warn the user
+	if IsEnvTruthy(SkipInstall) && (envutils.IsEnvDefined(PersistInstall) && !IsEnvTruthy(PersistInstall)) {
+		log.Printf("Warning: %s is set to true, but %s is set to false. This is likely a mistake. Gloo will be installed.", SkipInstall, PersistInstall)
+	}
+	return IsEnvTruthy(SkipInstall) || (IsEnvTruthy(PersistInstall) && alreadyInstalled)
 }
 
 // ShouldSkipIstioInstall returns true if any assets that need to be created before a test (for example Gloo being installed)

test/testutils/install.go

@@ -0,0 +1,30 @@
+package testutils
+
+import (
+	"context"
+
+	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check"
+	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options"
+	"github.com/solo-io/gloo/projects/gloo/cli/pkg/printers"
+	"github.com/solo-io/solo-kit/pkg/api/v1/resources/core"
+)
+
+// CheckResourcesOk checks if the resources are ok, similar to `glooctl check`
+func CheckResourcesOk(ctx context.Context, installNamespace string) error {
+	contextWithCancel, cancel := context.WithCancel(ctx)
+	defer cancel()
+	opts := &options.Options{
+		Metadata: core.Metadata{
+			Namespace: installNamespace,
+		},
+		Top: options.Top{
+			Ctx: contextWithCancel,
+		},
+	}
+	return check.CheckResources(contextWithCancel, printers.P{}, opts)
+}
+
+// IsGlooInstalled checks if gloo is installed by checking the resources
+func IsGlooInstalled(ctx context.Context, installNamespace string) bool {
+	return CheckResourcesOk(ctx, installNamespace) == nil
+}