From 05b4e9546aaa14f842019ab0f0ef99c6eaf2c380 Mon Sep 17 00:00:00 2001 From: guruprasad Date: Fri, 20 Dec 2024 16:32:55 +0530 Subject: [PATCH] update integration tests --- CONTRIBUTING.md | 102 +++++++++++------ pom.xml | 22 +++- .../computeengine/client/ClientUtil.java | 15 +++ .../integration/ComputeClientIT.java | 2 +- ...CloudMultipleMatchingConfigurationsIT.java | 33 +++++- ...ComputeEngineCloudNoSnapshotCreatedIT.java | 2 +- .../ComputeEngineCloudNonStandardJavaIT.java | 22 +--- .../ComputeEngineCloudRestartPreemptedIT.java | 61 +++++++--- .../ComputeEngineCloudSnapshotCreatedIT.java | 9 +- .../ComputeEngineCloudWorkerCreatedIT.java | 23 ++++ .../ComputeEngineCloudWorkerFailedIT.java | 3 +- .../computeengine/integration/ITUtil.java | 108 ++++++++++++------ .../integration/configuration-as-code-it.yml | 7 +- ...iguration-as-code-non-standard-java-it.yml | 7 +- testimages/linux/install-java.sh | 15 +++ testimages/linux/jre.pkr.hcl | 46 ++++++++ testimages/linux/setup-gce-image.sh | 37 ++++++ 17 files changed, 388 insertions(+), 126 deletions(-) create mode 100644 testimages/linux/install-java.sh create mode 100644 testimages/linux/jre.pkr.hcl create mode 100644 testimages/linux/setup-gce-image.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3dca8c24..b698fc21 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,45 +54,75 @@ An essential part of getting your change through is to make sure all existing te * Make sure you are at the directory where pom.xml is located. ##### Unit Tests -* Run the following: - -``` -mvn test -``` +* Run the following: + ``` + mvn test + ``` ##### Integration Tests -* By default, the integration tests are not executed. In case you are interested in executing - them, disable the `skipITs` property. -* The following environment variables are required to run the integration tests. 5, 6, and 7 are - only required when running a windows integration test. - -1. GOOGLE_PROJECT_ID -1. GOOGLE_CREDENTIALS -1. GOOGLE_REGION -1. GOOGLE_ZONE -1. GOOGLE_BOOT_DISK_PROJECT_ID -1. GOOGLE_BOOT_DISK_IMAGE_NAME -1. GOOGLE_JENKINS_PASSWORD -1. GOOGLE_SA_NAME - -* Run the following: -``` -mvn verify -DskipITs=false -``` + +Integration tests actually provision instances in a GCP project, run pipeline, take snapshot etc. +Therefore, they are disabled in the CI and expected to be executed by contributors in their laptop itself. + +Reasons for disabling integration test in CI, +* getting provisioning GCP infra is not possible +* even if we did get a GCP infra setup in the CI, it is risky to expose that, as someone can abuse the CI. + +By default, integration tests are skipped from the maven goals, need to enable using the `skipITs` property. + +Steps to execute integration test +* Prepare VM images + (ideally we should automate this see idea [here](https://github.com/jenkinsci/google-compute-engine-plugin/pull/492#discussion_r1892705637)) + The jenkins agent images need to have java installed. We have a packer script to create the image and upload to your configured GCP project. + The scripts are located in [testimages/linux](./testimages/linux) + Navigate to the directory and execute, + ```bash + bash setup-gce-image.sh + ``` + If you want to execute the integration tests for `non-standard-java` location, then create non-standard-java image as, + ```bash + bash setup-gce-image.sh non-standard-java + ``` + If you want to delete the images or recreate them, use the arguments `--recreate` or `--delete`. + +* Create a service account with relevant access - See [Refer to IAM Credentials](Home.md#iam-credentials) + +* Export these mandatory environment variable + ```bash + export GOOGLE_PROJECT_ID=your-project-id + export GOOGLE_CREDENTIALS=/path/to/sa-key.json + export GOOGLE_REGION=us-central1 + export GOOGLE_ZONE=us-central1-a + export GOOGLE_SA_NAME=jenkins-agent-sa + ``` +* Run the integration tests as, + * Run all the tests + ```bash + mvn verify -DskipITs=false -Djenkins.test.timeout=1200 + ``` + * Run a specific test class + ```bash + mvn clean test -Dtest=ComputeEngineCloudRestartPreemptedIT + ``` + * Run a specific test method + ```bash + mvn clean test -Dtest=ComputeEngineCloudRestartPreemptedIT#testIfNodeWasPreempted + ``` + You can also debug the tests with surefire by passing `-Dmaven.surefire.debug=true` and in your IDE connect to remote debug port `8000`. ###### Windows Integration Test * By default, the integration tests only use linux based agents for testing. If you make a windows-related change, or otherwise want to test that a change still works for windows agents, - run the tests with the flag `-Dit.windows=true` like this: -```bash -mvn verify -Dit.windows=true -``` - -Make sure you have these extra environment variables configured: - * GOOGLE_BOOT_DISK_PROJECT_ID will be the same as your project id. - * GOOGLE_BOOT_DISK_IMAGE_NAME will be the name of the image you created using packer in Google - cloud console. - * GOOGLE_JENKINS_PASSWORD will be the password you set when creating the image with packer, used - for password based ssh authentication. - * More information on building your baseline windows image can be found [here](WINDOWS.md) - and an example powershell script for setup can be found [here](windows-it-install.ps1). + run the tests with the flag `-Dit.windows=true` like this: + ```bash + mvn verify -Dit.windows=true + ``` +* You need to prepare the windows image before running the tests. + * More information on building your baseline windows image can be found [here](WINDOWS.md) + and an example powershell script for setup can be found [here](windows-it-install.ps1). + * In addition to the environment variables mentioned in the previous section, also export these variables too, + ```bash + export GOOGLE_BOOT_DISK_PROJECT_ID=your-project-id # will be the same as your project id + expot GOOGLE_BOOT_DISK_IMAGE_NAME=windows-image-name # will be the name of the image you created using packer in Google cloud console + export GOOGLE_JENKINS_PASSWORD=password # will be the password you set when creating the image with packer, used for password based ssh authentication. + ``` diff --git a/pom.xml b/pom.xml index f3d480a9..5c1e0450 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,27 @@ ${powershell.version} test - + + + org.jenkins-ci.plugins.workflow + workflow-cps + test + + + org.jenkins-ci.plugins.workflow + workflow-durable-task-step + test + + + org.jenkins-ci.plugins.workflow + workflow-job + test + org.mockito mockito-core diff --git a/src/main/java/com/google/jenkins/plugins/computeengine/client/ClientUtil.java b/src/main/java/com/google/jenkins/plugins/computeengine/client/ClientUtil.java index cfcd5381..5b38aa91 100644 --- a/src/main/java/com/google/jenkins/plugins/computeengine/client/ClientUtil.java +++ b/src/main/java/com/google/jenkins/plugins/computeengine/client/ClientUtil.java @@ -13,6 +13,7 @@ import com.google.jenkins.plugins.credentials.oauth.GoogleOAuth2Credentials; import com.google.jenkins.plugins.credentials.oauth.GoogleRobotCredentials; import hudson.AbortException; +import hudson.Main; import hudson.model.ItemGroup; import hudson.security.ACL; import java.io.IOException; @@ -73,6 +74,20 @@ private static GoogleRobotCredentials getRobotCredentials( ItemGroup itemGroup, List domainRequirements, String credentialsId) throws AbortException { + /* During the integration tests, the parameter `credentialId` is equal to the `` which is set + by the environment variable. But the actual credential created within Jenkins is having `id` as a random + UUID. So the `CredentialsMatchers.firstOrNull` was returning `null` due to `CredentialsMatchers.withId + (credentialsId)` + */ + if (Main.isUnitTest) { + var credentialList = CredentialsProvider.lookupCredentials( + GoogleOAuth2Credentials.class, itemGroup, ACL.SYSTEM, domainRequirements); + if (!credentialList.isEmpty()) { + return (GoogleRobotCredentials) credentialList.get(0); + } + return null; + } + GoogleOAuth2Credentials credentials = CredentialsMatchers.firstOrNull( CredentialsProvider.lookupCredentials( GoogleOAuth2Credentials.class, itemGroup, ACL.SYSTEM, domainRequirements), diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeClientIT.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeClientIT.java index 19bc4b34..da1d08c9 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeClientIT.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeClientIT.java @@ -58,7 +58,7 @@ public static void teardown() throws IOException { @Test public void testGetImage() throws Exception { - Image image = client.getImage("debian-cloud", "debian-9-stretch-v20180820"); + Image image = client.getImage("debian-cloud", "debian-12-bookworm-v20241210"); assertNotNull(image); assertEquals("READY", image.getStatus()); } diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudMultipleMatchingConfigurationsIT.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudMultipleMatchingConfigurationsIT.java index 366ab813..5fa2159d 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudMultipleMatchingConfigurationsIT.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudMultipleMatchingConfigurationsIT.java @@ -19,15 +19,17 @@ import static com.google.jenkins.plugins.computeengine.integration.ITUtil.LABEL; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.NULL_TEMPLATE; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.NUM_EXECUTORS; +import static com.google.jenkins.plugins.computeengine.integration.ITUtil.PROJECT_ID; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.TEST_TIMEOUT_MULTIPLIER; +import static com.google.jenkins.plugins.computeengine.integration.ITUtil.ZONE; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.getLabel; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.initClient; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.initCloud; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.initCredentials; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.instanceConfigurationBuilder; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.teardownResources; +import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import com.google.cloud.graphite.platforms.plugin.client.ComputeClient; @@ -52,8 +54,7 @@ /** * Integration test suite for {@link ComputeEngineCloud}. Verifies that instances can be created - * with multiple matching {@link InstanceConfiguration} and that these instances are properly - * provisioned. + * with multiple matching {@link InstanceConfiguration} and that these instances are provisioned in round-robin fashion. */ public class ComputeEngineCloudMultipleMatchingConfigurationsIT { private static Logger log = Logger.getLogger(ComputeEngineCloudMultipleMatchingConfigurationsIT.class.getName()); @@ -105,16 +106,18 @@ public static void teardown() throws IOException { } @Test - public void testMultipleLabelsProvisionedWithLabels() { + public void testMultipleLabelsProvisionedWithLabels() throws IOException { assertEquals(2, planned.size()); final Iterator iterator = planned.iterator(); PlannedNode firstNode = iterator.next(); PlannedNode secondNode = iterator.next(); if (checkOneNode(firstNode, DESC_1)) { - assertTrue(checkOneNode(secondNode, DESC_2)); + assertDescription(firstNode, DESC_1); + assertDescription(secondNode, DESC_2); } else if (checkOneNode(secondNode, DESC_1)) { - assertTrue(checkOneNode(firstNode, DESC_2)); + assertDescription(secondNode, DESC_1); + assertDescription(firstNode, DESC_2); } else { fail("Nodes did not have expected values"); } @@ -125,4 +128,22 @@ private boolean checkOneNode(PlannedNode plannedNode, String desc) { Node node = jenkinsRule.jenkins.getNode(name); return desc.equals(node.getNodeDescription()); } + + private void assertDescription(PlannedNode plannedNode, String agentDesc) throws IOException { + String agentName = plannedNode.displayName; + + assertEquals( + "Jenkins agent description is incorrect", + agentDesc, + jenkinsRule.jenkins.getNode(agentName).getNodeDescription()); + + // verify the corresponding instance is actually provisioned in GCP + // and also has matching description + await("Instance is in running status").timeout(2, TimeUnit.MINUTES).until(() -> "RUNNING" + .equals(client.getInstance(PROJECT_ID, ZONE, agentName).getStatus())); + assertEquals( + "Instance description is incorrect", + agentDesc, + client.getInstance(PROJECT_ID, ZONE, agentName).getDescription()); + } } diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudNoSnapshotCreatedIT.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudNoSnapshotCreatedIT.java index 54037184..0f1bd691 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudNoSnapshotCreatedIT.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudNoSnapshotCreatedIT.java @@ -59,7 +59,7 @@ * instance is terminated but no snapshot is created. */ public class ComputeEngineCloudNoSnapshotCreatedIT { - private static Logger log = Logger.getLogger(ComputeEngineCloudNoSnapshotCreatedIT.class.getName()); + private static final Logger log = Logger.getLogger(ComputeEngineCloudNoSnapshotCreatedIT.class.getName()); @ClassRule public static Timeout timeout = new Timeout(7 * TEST_TIMEOUT_MULTIPLIER, TimeUnit.MINUTES); diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudNonStandardJavaIT.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudNonStandardJavaIT.java index 6dd6917e..fd8cf6b3 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudNonStandardJavaIT.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudNonStandardJavaIT.java @@ -16,11 +16,11 @@ package com.google.jenkins.plugins.computeengine.integration; +import static com.google.jenkins.plugins.computeengine.integration.ITUtil.BOOT_DISK_IMAGE_NAME; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.LABEL; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.NULL_TEMPLATE; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.NUM_EXECUTORS; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.PROJECT_ID; -import static com.google.jenkins.plugins.computeengine.integration.ITUtil.TEST_TIMEOUT_MULTIPLIER; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.ZONE; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.getLabel; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.initClient; @@ -42,37 +42,24 @@ import java.io.IOException; import java.util.Collection; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -import org.junit.rules.Timeout; import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.recipes.WithTimeout; /** * Integration test suite for {@link ComputeEngineCloudNonStandardJavaIT}. Verifies that instances * can be created using a non-standard Java executable path. */ public class ComputeEngineCloudNonStandardJavaIT { - private static final String NON_STANDARD_JAVA_STARTUP_SCRIPT = "#!/bin/bash\n" - + "sudo su -\n" - + "/etc/init.d/ssh stop\n" - + "echo \"deb http://http.debian.net/debian stretch-backports main\" >> /etc/apt/sources.list\n" - + "apt-get -y update\n" - + "apt-get -y install -t stretch-backports openjdk-8-jdk\n" - + "update-java-alternatives -s java-1.8.0-openjdk-amd64\n" - + "mv /usr/bin/java /usr/bin/non-standard-java\n" - + "/etc/init.d/ssh start"; private static final String NON_STANDARD_JAVA_PATH = "/usr/bin/non-standard-java"; private static Logger log = Logger.getLogger(ComputeEngineCloudNonStandardJavaIT.class.getName()); - @ClassRule - public static Timeout timeout = new Timeout(5 * TEST_TIMEOUT_MULTIPLIER, TimeUnit.MINUTES); - @ClassRule public static JenkinsRule jenkinsRule = new JenkinsRule(); @@ -88,9 +75,8 @@ public static void init() throws Exception { initCredentials(jenkinsRule); ComputeEngineCloud cloud = initCloud(jenkinsRule); client = initClient(jenkinsRule, label, log); - InstanceConfiguration instanceConfiguration = instanceConfigurationBuilder() - .startupScript(NON_STANDARD_JAVA_STARTUP_SCRIPT) + .bootDiskSourceImageName(BOOT_DISK_IMAGE_NAME + "-non-standard-java") .numExecutorsStr(NUM_EXECUTORS) .labels(LABEL) .oneShot(false) @@ -112,11 +98,13 @@ public static void teardown() throws IOException { teardownResources(client, label, log); } + @WithTimeout(300) @Test public void testWorkerCreatedOnePlannedNode() { assertEquals(1, planned.size()); } + @WithTimeout(300) @Test public void testInstanceStatusRunning() { assertEquals("RUNNING", instance.getStatus()); diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudRestartPreemptedIT.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudRestartPreemptedIT.java index 6115ddaa..73974688 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudRestartPreemptedIT.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudRestartPreemptedIT.java @@ -20,7 +20,6 @@ import static com.google.jenkins.plugins.computeengine.integration.ITUtil.NULL_TEMPLATE; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.NUM_EXECUTORS; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.PROJECT_ID; -import static com.google.jenkins.plugins.computeengine.integration.ITUtil.TEST_TIMEOUT_MULTIPLIER; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.ZONE; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.execute; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.getLabel; @@ -57,17 +56,19 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -import org.junit.rules.Timeout; import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.PrefixedOutputStream; +import org.jvnet.hudson.test.TailLog; +import org.jvnet.hudson.test.recipes.WithTimeout; /** - * Integration test suite for {@link ComputeEngineCloud}. Verifies that instances with preempted - * flag will be restarted when preempted. + * Integration test suite for {@link ComputeEngineCloud}. Verifies that the build is rescheduled and + * completed successfully, when the agent was provisioned with Preemptible Vm, and the agent is preempted during an + * ongoing build. See {@code PreemptedCheckCallable} being attached in {@link ComputeEngineComputer} and + * {@code ComputeEngineRetentionStrategy#rescheduleTask} usages. */ @Log public class ComputeEngineCloudRestartPreemptedIT { - @ClassRule - public static Timeout timeout = new Timeout(20 * TEST_TIMEOUT_MULTIPLIER, TimeUnit.MINUTES); @ClassRule public static JenkinsRule jenkinsRule = new JenkinsRule(); @@ -100,7 +101,23 @@ public static void teardown() throws IOException { teardownResources(client, label, log); } + /** + * This test works, the logs are also clear until the preemption event occurs. The {@code ComputeEngineCloud} + * launch logs may seem confusing as they differ when we run the test multiple times. + *

+ * After the preemption event occurred, even though the executors in the preempted agent are terminated + * by the {@code ComputeEngineComputer#getPreemptedStatus}, but for some reason the {@code ComputeEngineCloud} logs + * (sometimes and not always) show that Jenkins still attempts to connect to that stopping VM. Due to this attempt + * to connect to the stopping VM, you might see some IOException, or host null because no network interface found + * etc. + *

+ * However note that these errors have nothing to do with the task being rescheduled, a new VM agent does get + * provisioned and build is rescheduled on there, and succeeds. + *

+ * It is just that in the test logs, the logs get mixed up and may seem confusing. + */ @Test + @WithTimeout(1200) public void testIfNodeWasPreempted() throws Exception { Collection planned = cloud.provision(new LabelAtom(LABEL), 1); Iterator iterator = planned.iterator(); @@ -112,24 +129,34 @@ public void testIfNodeWasPreempted() throws Exception { ComputeEngineComputer computer = (ComputeEngineComputer) node.toComputer(); assertTrue("Configuration was set as preemptible but saw as not", computer.getPreemptible()); - FreeStyleProject project = jenkinsRule.createFreeStyleProject(); + FreeStyleProject project = jenkinsRule.createFreeStyleProject("p"); Builder step = execute(Commands.SLEEP, "60"); project.getBuildersList().add(step); project.setAssignedLabel(new LabelAtom(LABEL)); - QueueTaskFuture taskFuture = project.scheduleBuild2(0); + FreeStyleBuild freeStyleBuild; + // build1 that gets failed due to preemption + try (var tailLog = new TailLog(jenkinsRule, "p", 1).withColor(PrefixedOutputStream.Color.MAGENTA)) { + QueueTaskFuture taskFuture = project.scheduleBuild2(0); - Awaitility.await().timeout(7, TimeUnit.MINUTES).until(() -> computer.getLog() - .contains("listening to metadata for preemption event")); + Awaitility.await().timeout(7, TimeUnit.MINUTES).until(() -> computer.getLog() + .contains("listening to metadata for preemption event")); - client.simulateMaintenanceEvent(PROJECT_ID, ZONE, name); - Awaitility.await().timeout(8, TimeUnit.MINUTES).until(computer::getPreempted); + client.simulateMaintenanceEvent(PROJECT_ID, ZONE, name); + Awaitility.await().timeout(8, TimeUnit.MINUTES).until(computer::getPreempted); - FreeStyleBuild freeStyleBuild = taskFuture.get(); - assertEquals(FAILURE, freeStyleBuild.getResult()); + freeStyleBuild = taskFuture.get(); + assertEquals(FAILURE, freeStyleBuild.getResult()); + tailLog.waitForCompletion(); + } Awaitility.await().timeout(5, TimeUnit.MINUTES).until(() -> freeStyleBuild.getNextBuild() != null); - FreeStyleBuild nextBuild = freeStyleBuild.getNextBuild(); - Awaitility.await().timeout(5, TimeUnit.MINUTES).until(() -> nextBuild.getResult() != null); - assertEquals(SUCCESS, nextBuild.getResult()); + + // build2 gets automatically scheduled and succeeds + try (var tailLog = new TailLog(jenkinsRule, "p", 2).withColor(PrefixedOutputStream.Color.YELLOW)) { + FreeStyleBuild nextBuild = freeStyleBuild.getNextBuild(); + Awaitility.await().timeout(5, TimeUnit.MINUTES).until(() -> nextBuild.getResult() != null); + assertEquals(SUCCESS, nextBuild.getResult()); + tailLog.waitForCompletion(); + } } } diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudSnapshotCreatedIT.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudSnapshotCreatedIT.java index 30fda848..c3c90b0f 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudSnapshotCreatedIT.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudSnapshotCreatedIT.java @@ -21,7 +21,6 @@ import static com.google.jenkins.plugins.computeengine.integration.ITUtil.PROJECT_ID; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.SNAPSHOT_LABEL; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.SNAPSHOT_TIMEOUT; -import static com.google.jenkins.plugins.computeengine.integration.ITUtil.TEST_TIMEOUT_MULTIPLIER; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.execute; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.getLabel; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.initClient; @@ -53,8 +52,8 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -import org.junit.rules.Timeout; import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.recipes.WithTimeout; /** * Integration test suite for {@link ComputeEngineCloud}. Verifies that when configured to use one @@ -64,9 +63,6 @@ public class ComputeEngineCloudSnapshotCreatedIT { private static Logger log = Logger.getLogger(ComputeEngineCloudSnapshotCreatedIT.class.getName()); - @ClassRule - public static Timeout timeout = new Timeout(15 * TEST_TIMEOUT_MULTIPLIER, TimeUnit.MINUTES); - @ClassRule public static JenkinsRule jenkinsRule = new JenkinsRule(); @@ -121,16 +117,19 @@ public static void teardown() throws IOException { } /** Tests snapshot is created when we have failure builds for given node */ + @WithTimeout(1200) @Test public void testSnapshotCreatedNotNull() { assertNotNull(createdSnapshot); } + @WithTimeout(1200) @Test public void testSnapshotCreatedStatusReady() { assertEquals("READY", createdSnapshot.getStatus()); } + @WithTimeout(1200) @Test public void testSnapshotCreatedOneShotInstanceDeleted() { Awaitility.await() diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudWorkerCreatedIT.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudWorkerCreatedIT.java index 6d54f2f5..ae586c48 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudWorkerCreatedIT.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudWorkerCreatedIT.java @@ -28,6 +28,9 @@ import static com.google.jenkins.plugins.computeengine.integration.ITUtil.initCredentials; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.instanceConfigurationBuilder; import static com.google.jenkins.plugins.computeengine.integration.ITUtil.teardownResources; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -44,12 +47,17 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.Timeout; import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.PrefixedOutputStream; +import org.jvnet.hudson.test.TailLog; +import org.jvnet.hudson.test.recipes.WithTimeout; /** * Integration test suite for {@link ComputeEngineCloud}. This verifies the default case for an @@ -138,4 +146,19 @@ public void testGuestAttributesEnabled() { assertTrue(guestAttributes.isPresent()); assertEquals(guestAttributes.get(), "TRUE"); } + + @WithTimeout(300) + @Test + public void testWorkerCanExecuteBuild() throws Exception { + var p = jenkinsRule.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("node('" + LABEL + "') { sh 'date' }", true)); + try (var tailLog = new TailLog(jenkinsRule, "p", 1).withColor(PrefixedOutputStream.Color.MAGENTA)) { + var r = jenkinsRule.buildAndAssertSuccess(p); + tailLog.waitForCompletion(); + assertThat( + "Build did not run on GCP agent", + JenkinsRule.getLog(r), + is(containsString("Running on " + instance.getName()))); + } + } } diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudWorkerFailedIT.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudWorkerFailedIT.java index f3c8cb23..6a4833d5 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudWorkerFailedIT.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ComputeEngineCloudWorkerFailedIT.java @@ -74,7 +74,8 @@ public static void init() throws Exception { // This configuration creates an instance with no Java installed. cloud.setConfigurations(ImmutableList.of(instanceConfigurationBuilder() - .startupScript("") + .bootDiskSourceImageProject("debian-cloud") + .bootDiskSourceImageName("projects/debian-cloud/global/images/family/debian-12") .numExecutorsStr(NUM_EXECUTORS) .labels(LABEL) .oneShot(false) diff --git a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ITUtil.java b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ITUtil.java index 1810ac00..451ec380 100644 --- a/src/test/java/com/google/jenkins/plugins/computeengine/integration/ITUtil.java +++ b/src/test/java/com/google/jenkins/plugins/computeengine/integration/ITUtil.java @@ -63,6 +63,7 @@ import com.google.jenkins.plugins.computeengine.ssh.GoogleKeyPair; import com.google.jenkins.plugins.credentials.oauth.GoogleRobotPrivateKeyCredentials; import com.google.jenkins.plugins.credentials.oauth.JsonServiceAccountConfig; +import hudson.model.FreeStyleBuild; import hudson.model.Node; import hudson.plugins.powershell.PowerShell; import hudson.tasks.Builder; @@ -76,24 +77,18 @@ import java.util.Map; import java.util.logging.Logger; import jenkins.util.SystemProperties; +import lombok.extern.java.Log; import org.apache.commons.lang.SystemUtils; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; import org.jvnet.hudson.test.JenkinsRule; /** Common logic and constants used throughout the integration tests. */ +@Log class ITUtil { static final boolean windows = Boolean.parseBoolean(SystemProperties.getString(ITUtil.class.getName() + ".windows", "false")); static final boolean customssh = Boolean.parseBoolean(SystemProperties.getString(ITUtil.class.getName() + ".customssh", "false")); - private static final String DEB_JAVA_STARTUP_SCRIPT = "#!/bin/bash\n" - + "/etc/init.d/ssh stop\n" - + "echo \"deb http://http.debian.net/debian stretch-backports main\" | \\\n" - + " sudo tee --append /etc/apt/sources.list > /dev/null\n" - + "apt-get -y update\n" - + "apt-get -y install -t stretch-backports openjdk-8-jdk\n" - + "update-java-alternatives -s java-1.8.0-openjdk-amd64\n" - + "/etc/init.d/ssh start"; - static final String PROJECT_ID = System.getenv("GOOGLE_PROJECT_ID"); private static final String CREDENTIALS = loadCredentialsString(); static final String CLOUD_NAME = "integration"; @@ -111,13 +106,9 @@ class ITUtil { private static final String CONFIG_DESC = "integration"; private static final String BOOT_DISK_TYPE = ZONE_BASE + "/diskTypes/pd-ssd"; private static final boolean BOOT_DISK_AUTODELETE = true; - private static final String BOOT_DISK_PROJECT_ID = - windows ? System.getenv("GOOGLE_BOOT_DISK_PROJECT_ID") : "debian-cloud"; - private static final String BOOT_DISK_IMAGE_NAME = windows - ? String.format( - "projects/%s/global/images/%s", BOOT_DISK_PROJECT_ID, System.getenv("GOOGLE_BOOT_DISK_IMAGE_NAME")) - : "projects/debian-cloud/global/images/family/debian-9"; - private static final String BOOT_DISK_SIZE_GB_STR = windows ? "50" : "10"; + private static final String BOOT_DISK_PROJECT_ID = bootDiskProject(); + static final String BOOT_DISK_IMAGE_NAME = bootDiskImageName(); + private static final String BOOT_DISK_SIZE_GB_STR = windows ? "50" : "20"; private static final Node.Mode NODE_MODE = Node.Mode.EXCLUSIVE; private static final String ACCELERATOR_NAME = ""; private static final String ACCELERATOR_COUNT = ""; @@ -134,24 +125,9 @@ class ITUtil { static final int SNAPSHOT_TIMEOUT = windows ? 600 : 300; private static final GoogleKeyCredential SSH_KEY = GoogleKeyPair.generate(RUN_AS_USER); static final String SSH_PRIVATE_KEY = Secret.toString(SSH_KEY.getPrivateKey()); - private static final String WINDOWS_STARTUP_SCRIPT = "Stop-Service sshd\n" - + "$ConfiguredPublicKey = " - + "\"" - + ((GoogleKeyPair) SSH_KEY).getPublicKey().trim().substring(RUN_AS_USER.length() + 1) - + "\"\n" - + "Write-Output \"Second phase\"\n" - + "# We are in the second phase of startup where we need to set up authorized_keys for the specified user.\n" - + "# Create the .ssh folder and authorized_keys file.\n" - + "Set-Content -Path $env:PROGRAMDATA\\ssh\\administrators_authorized_keys -Value $ConfiguredPublicKey\n" - + "icacls $env:PROGRAMDATA\\ssh\\administrators_authorized_keys /inheritance:r\n" - + "icacls $env:PROGRAMDATA\\ssh\\administrators_authorized_keys /grant SYSTEM:`(F`)\n" - + "icacls $env:PROGRAMDATA\\ssh\\administrators_authorized_keys /grant BUILTIN\\Administrators:`(F`)\n" - + "Restart-Service sshd"; - private static final String STARTUP_SCRIPT = windows ? WINDOWS_STARTUP_SCRIPT : DEB_JAVA_STARTUP_SCRIPT; static final int TEST_TIMEOUT_MULTIPLIER = (SystemUtils.IS_OS_WINDOWS || windows) ? 3 : 1; static final String CONFIG_AS_CODE_PATH = windows ? "configuration-as-code-windows-it.yml" : "configuration-as-code-it.yml"; - private static String windowsPrivateKeyCredentialsId; private static String customsshPrivateKeyCredentialsId; @@ -160,6 +136,50 @@ static String format(String s) { return String.format(s, PROJECT_ID); } + private static String bootDiskImageName() { + String projectId = System.getenv("GOOGLE_BOOT_DISK_PROJECT_ID"); + if (projectId == null) { + projectId = PROJECT_ID; + } + + String imageName = System.getenv("GOOGLE_BOOT_DISK_IMAGE_NAME"); + if (imageName == null) { + imageName = "jenkins-gce-integration-test-jre"; /* Image built with packer, see the `setup-gce-image.sh` */ + } + + String imageFqn = String.format("projects/%s/global/images/%s", projectId, imageName); + log.info("Using boot disk image: " + imageFqn); + + return imageFqn; + } + + private static String bootDiskProject() { + String project = System.getenv("GOOGLE_BOOT_DISK_PROJECT_ID"); + project = project != null ? project : PROJECT_ID; + log.info("Using boot disk project: " + project); + return project; + } + + private static String startUpScript() { + if (!windows) { // there is no need for a startup script in linux + return ""; + } + + return "Stop-Service sshd\n" + + "$ConfiguredPublicKey = " + + "\"" + + ((GoogleKeyPair) SSH_KEY).getPublicKey().trim().substring(RUN_AS_USER.length() + 1) + + "\"\n" + + "Write-Output \"Second phase\"\n" + + "# We are in the second phase of startup where we need to set up authorized_keys for the specified user.\n" + + "# Create the .ssh folder and authorized_keys file.\n" + + "Set-Content -Path $env:PROGRAMDATA\\ssh\\administrators_authorized_keys -Value $ConfiguredPublicKey\n" + + "icacls $env:PROGRAMDATA\\ssh\\administrators_authorized_keys /inheritance:r\n" + + "icacls $env:PROGRAMDATA\\ssh\\administrators_authorized_keys /grant SYSTEM:`(F`)\n" + + "icacls $env:PROGRAMDATA\\ssh\\administrators_authorized_keys /grant BUILTIN\\Administrators:`(F`)\n" + + "Restart-Service sshd"; + } + private static String loadCredentialsString() { String creds = System.getenv("GOOGLE_CREDENTIALS"); if (!Strings.isNullOrEmpty(creds)) { @@ -278,7 +298,7 @@ static InstanceTemplate createTemplate(Map googleLabels, String windows ? METADATA_WINDOWS_STARTUP_SCRIPT_KEY : METADATA_LINUX_STARTUP_SCRIPT_KEY) - .setValue(STARTUP_SCRIPT), + .setValue(startUpScript()), new Metadata.Items() .setKey(InstanceConfiguration.SSH_METADATA_KEY) .setValue(((GoogleKeyPair) SSH_KEY).getPublicKey())))); @@ -329,7 +349,7 @@ static InstanceConfiguration.Builder instanceConfigurationBuilder() { .mode(NODE_MODE) .acceleratorConfiguration(new AcceleratorConfiguration(ACCELERATOR_NAME, ACCELERATOR_COUNT)) .runAsUser(RUN_AS_USER) - .startupScript(STARTUP_SCRIPT) + .startupScript(startUpScript()) .javaExecPath("java -Dhudson.remoting.Launcher.pingIntervalSec=-1"); } @@ -359,4 +379,26 @@ private static void safeDelete(String instanceId, boolean waitForCompletion, Com log.warning(String.format("Error deleting instance %s: %s", instanceId, e.getMessage())); } } + + public static String printLogsAndGetAgentName(Object build) throws IOException { + List logs; + if (build instanceof FreeStyleBuild) { + logs = ((FreeStyleBuild) build).getLog(1000); + } else if (build instanceof WorkflowRun) { + logs = ((WorkflowRun) build).getLog(1000); + } else { + throw new IllegalArgumentException("Unsupported build type"); + } + + String agentName = null; + for (String line : logs) { + if (line.contains("Building remotely on")) { + agentName = line.split(" ")[3]; + } else if (line.contains("Running on")) { + agentName = line.split(" ")[2]; + } + log.info(line); + } + return agentName; + } } diff --git a/src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-it.yml b/src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-it.yml index 068580c7..f6315ec9 100644 --- a/src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-it.yml +++ b/src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-it.yml @@ -24,7 +24,6 @@ jenkins: machineType: "https://www.googleapis.com/compute/v1/projects/${env.GOOGLE_PROJECT_ID}/zones/${env.GOOGLE_ZONE}/machineTypes/n1-standard-1" preemptible: false minCpuPlatform: '' # tried not setting, added when 'saved' in UI - startupScript: "#!/bin/bash\n/etc/init.d/ssh stop\necho \"deb http://http.debian.net/debian stretch-backports main\" | \\\n sudo tee --append /etc/apt/sources.list > /dev/null\napt-get -y update\napt-get -y install -t stretch-backports openjdk-8-jdk\nupdate-java-alternatives -s java-1.8.0-openjdk-amd64\n/etc/init.d/ssh start" networkConfiguration: autofilled: network: default @@ -34,9 +33,9 @@ jenkins: singleStack: externalIPV4Address: true useInternalAddress: false - bootDiskSourceImageProject: debian-cloud - bootDiskSourceImageName: "projects/debian-cloud/global/images/family/debian-9" + bootDiskSourceImageProject: ${env.GOOGLE_PROJECT_ID} + bootDiskSourceImageName: "projects/${env.GOOGLE_PROJECT_ID}/global/images/jenkins-gce-integration-test-jre" bootDiskType: "https://www.googleapis.com/compute/v1/projects/${env.GOOGLE_PROJECT_ID}/zones/${env.GOOGLE_ZONE}/diskTypes/pd-ssd" - bootDiskSizeGbStr: 10 + bootDiskSizeGbStr: 20 bootDiskAutoDelete: true serviceAccountEmail: "${env.GOOGLE_SA_NAME}@${env.GOOGLE_PROJECT_ID}.iam.gserviceaccount.com" diff --git a/src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-non-standard-java-it.yml b/src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-non-standard-java-it.yml index 8ce0d72e..decccc83 100644 --- a/src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-non-standard-java-it.yml +++ b/src/test/resources/com/google/jenkins/plugins/computeengine/integration/configuration-as-code-non-standard-java-it.yml @@ -23,7 +23,6 @@ jenkins: machineType: "https://www.googleapis.com/compute/v1/projects/${env.GOOGLE_PROJECT_ID}/zones/${env.GOOGLE_ZONE}/machineTypes/n1-standard-1" preemptible: false minCpuPlatform: '' # tried not setting, added when 'saved' in UI - startupScript: "#!/bin/bash\nsudo su-\n/etc/init.d/ssh stop\necho \"deb http://http.debian.net/debian stretch-backports main\" >> /etc/apt/sources.list\napt-get -y update\napt-get -y install -t stretch-backports openjdk-8-jdk\nupdate-java-alternatives -s java-1.8.0-openjdk-amd64\nln -s /usr/bin/java /usr/bin/non-standard-java\n/etc/init.d/ssh start" javaExecPath: 'non-standard-java' networkConfiguration: autofilled: @@ -34,9 +33,9 @@ jenkins: singleStack: externalIPV4Address: true useInternalAddress: false - bootDiskSourceImageProject: debian-cloud - bootDiskSourceImageName: "projects/debian-cloud/global/images/family/debian-9" + bootDiskSourceImageProject: ${env.GOOGLE_PROJECT_ID} + bootDiskSourceImageName: "projects/${env.GOOGLE_PROJECT_ID}/global/images/jenkins-gce-integration-test-jre-non-standard-java" bootDiskType: "https://www.googleapis.com/compute/v1/projects/${env.GOOGLE_PROJECT_ID}/zones/${env.GOOGLE_ZONE}/diskTypes/pd-ssd" - bootDiskSizeGbStr: 10 + bootDiskSizeGbStr: 20 bootDiskAutoDelete: true serviceAccountEmail: "${env.GOOGLE_SA_NAME}@${env.GOOGLE_PROJECT_ID}.iam.gserviceaccount.com" diff --git a/testimages/linux/install-java.sh b/testimages/linux/install-java.sh new file mode 100644 index 00000000..eb2e22af --- /dev/null +++ b/testimages/linux/install-java.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euxo pipefail + +echo "AGENT_IMAGE value is: $AGENT_IMAGE" +if [ -z "$AGENT_IMAGE" ]; then + echo "AGENT_IMAGE variable is not set" + exit 1 +fi + +sudo apt-get update && sudo apt-get install openjdk-17-jre-headless -y && java -version + +if [[ "$AGENT_IMAGE" == *non-standard-java ]]; then + sudo mv /usr/bin/java /usr/bin/non-standard-java + /usr/bin/non-standard-java -version +fi \ No newline at end of file diff --git a/testimages/linux/jre.pkr.hcl b/testimages/linux/jre.pkr.hcl new file mode 100644 index 00000000..3a364e47 --- /dev/null +++ b/testimages/linux/jre.pkr.hcl @@ -0,0 +1,46 @@ +# https://github.com/jenkinsci/google-compute-engine-plugin/pull/186#issuecomment-1664345279 + +packer { + required_plugins { + googlecompute = { + version = ">= 1.1.1" + source = "github.com/hashicorp/googlecompute" + } + } +} + +variable "project" { + type = string +} + +variable "region" { + type = string +} + +variable "zone" { + type = string +} + +variable "agent_image" { + type = string +} + +source "googlecompute" "base" { + project_id = var.project + zone = var.zone + image_storage_locations = [var.region] + source_image_project_id = ["debian-cloud"] + source_image_family = "debian-12" + image_name = var.agent_image + ssh_username = "jenkins" +} + +build { + sources = ["sources.googlecompute.base"] + provisioner "shell" { + script = "./install-java.sh" + environment_vars = [ + "AGENT_IMAGE=${var.agent_image}" + ] + } +} \ No newline at end of file diff --git a/testimages/linux/setup-gce-image.sh b/testimages/linux/setup-gce-image.sh new file mode 100644 index 00000000..670e3f37 --- /dev/null +++ b/testimages/linux/setup-gce-image.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euxo pipefail + +project=${GOOGLE_PROJECT_ID:-$(gcloud config get-value core/project)} +region=${GOOGLE_REGION:-$(gcloud config get-value compute/region)} +zone=${GOOGLE_ZONE:-$(gcloud config get-value compute/zone)} + +current_dir=$(dirname "$0") +pushd $current_dir + +agent_image="jenkins-gce-integration-test-jre" +if [ "${1:-}" == "non-standard-java" ]; then + agent_image="${agent_image}-non-standard-java" + shift +fi + +case "${1:-}" in + --recreate) + if gcloud compute images describe $agent_image; then + gcloud compute images delete $agent_image --project=$project --quiet + fi + ;; + --delete) + if gcloud compute images describe $agent_image; then + gcloud compute images delete $agent_image --project=$project --quiet + exit 0 + fi + ;; +esac + +if ! gcloud compute images describe $agent_image --project=$project; then + packer init ./ + packer build -var project=$project -var region=$region -var zone=$zone -var agent_image=$agent_image ./ + gcloud compute images describe $agent_image --project=$project +fi + +popd \ No newline at end of file