From 59bee05f8a935f4c8ac9fcf0da47bab60c425e92 Mon Sep 17 00:00:00 2001 From: Slawomir Jaranowski Date: Thu, 26 Jan 2023 23:17:31 +0100 Subject: [PATCH] [MDEPLOY-206] Support parallel deployment for deployAtEnd Each whole module will be deployed by resolver in one thread. We have parallel deployment on module level. --- .../deploy-at-end-parallel/invoker.properties | 18 +++ src/it/deploy-at-end-parallel/module1/pom.xml | 25 ++++ src/it/deploy-at-end-parallel/module2/pom.xml | 25 ++++ src/it/deploy-at-end-parallel/module3/pom.xml | 25 ++++ src/it/deploy-at-end-parallel/module4/pom.xml | 25 ++++ src/it/deploy-at-end-parallel/pom.xml | 103 ++++++++++++++ src/it/deploy-at-end-parallel/setup.groovy | 26 ++++ src/it/deploy-at-end-parallel/verify.groovy | 37 +++++ .../maven/plugins/deploy/DeployMojo.java | 129 ++++++++++++++---- .../maven/plugins/deploy/DeployMojoTest.java | 7 +- 10 files changed, 388 insertions(+), 32 deletions(-) create mode 100644 src/it/deploy-at-end-parallel/invoker.properties create mode 100644 src/it/deploy-at-end-parallel/module1/pom.xml create mode 100644 src/it/deploy-at-end-parallel/module2/pom.xml create mode 100644 src/it/deploy-at-end-parallel/module3/pom.xml create mode 100644 src/it/deploy-at-end-parallel/module4/pom.xml create mode 100644 src/it/deploy-at-end-parallel/pom.xml create mode 100644 src/it/deploy-at-end-parallel/setup.groovy create mode 100644 src/it/deploy-at-end-parallel/verify.groovy diff --git a/src/it/deploy-at-end-parallel/invoker.properties b/src/it/deploy-at-end-parallel/invoker.properties new file mode 100644 index 00000000..4a11915f --- /dev/null +++ b/src/it/deploy-at-end-parallel/invoker.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +invoker.mavenOpts = -Dorg.slf4j.simpleLogger.showThreadName=true \ No newline at end of file diff --git a/src/it/deploy-at-end-parallel/module1/pom.xml b/src/it/deploy-at-end-parallel/module1/pom.xml new file mode 100644 index 00000000..4466ec7d --- /dev/null +++ b/src/it/deploy-at-end-parallel/module1/pom.xml @@ -0,0 +1,25 @@ + + + + + + 4.0.0 + + + org.apache.maven.its.deploy.dae.parallel + dae + 1.0 + + module1 + + diff --git a/src/it/deploy-at-end-parallel/module2/pom.xml b/src/it/deploy-at-end-parallel/module2/pom.xml new file mode 100644 index 00000000..d38b6718 --- /dev/null +++ b/src/it/deploy-at-end-parallel/module2/pom.xml @@ -0,0 +1,25 @@ + + + + + + 4.0.0 + + + org.apache.maven.its.deploy.dae.parallel + dae + 1.0 + + module2 + + diff --git a/src/it/deploy-at-end-parallel/module3/pom.xml b/src/it/deploy-at-end-parallel/module3/pom.xml new file mode 100644 index 00000000..be6d3437 --- /dev/null +++ b/src/it/deploy-at-end-parallel/module3/pom.xml @@ -0,0 +1,25 @@ + + + + + + 4.0.0 + + + org.apache.maven.its.deploy.dae.parallel + dae + 1.0 + + module3 + + diff --git a/src/it/deploy-at-end-parallel/module4/pom.xml b/src/it/deploy-at-end-parallel/module4/pom.xml new file mode 100644 index 00000000..a3d83823 --- /dev/null +++ b/src/it/deploy-at-end-parallel/module4/pom.xml @@ -0,0 +1,25 @@ + + + + + + 4.0.0 + + + org.apache.maven.its.deploy.dae.parallel + dae + 1.0 + + module4 + + diff --git a/src/it/deploy-at-end-parallel/pom.xml b/src/it/deploy-at-end-parallel/pom.xml new file mode 100644 index 00000000..3e783511 --- /dev/null +++ b/src/it/deploy-at-end-parallel/pom.xml @@ -0,0 +1,103 @@ + + + + + + 4.0.0 + + org.apache.maven.its.deploy.dae.parallel + dae + 1.0 + pom + + + Tests deployment at end with parallel. + + + + true + + + + + it + file:///@remoteRepo@/deploy-at-end-parallel + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + @mavenCompilerPluginVersion@ + + + org.apache.maven.plugins + maven-deploy-plugin + @project.version@ + + true + 4 + + + + org.apache.maven.plugins + maven-install-plugin + @mavenInstallPluginVersion@ + + + org.apache.maven.plugins + maven-jar-plugin + @mavenJarPluginVersion@ + + + org.apache.maven.plugins + maven-resources-plugin + @mavenResourcesPluginVersion@ + + + org.apache.maven.plugins + maven-source-plugin + @mavenSourcePluginVersion@ + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + @mavenSurefirePluginVersion@ + + + + + + module1 + module2 + module3 + module4 + + diff --git a/src/it/deploy-at-end-parallel/setup.groovy b/src/it/deploy-at-end-parallel/setup.groovy new file mode 100644 index 00000000..3fd36199 --- /dev/null +++ b/src/it/deploy-at-end-parallel/setup.groovy @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +def mockrepo = new File( remoteRepo, "deploy-at-end-parallel") +mockrepo.deleteDir() + +assert !mockrepo.exists() + +mockrepo.mkdirs() \ No newline at end of file diff --git a/src/it/deploy-at-end-parallel/verify.groovy b/src/it/deploy-at-end-parallel/verify.groovy new file mode 100644 index 00000000..845a6951 --- /dev/null +++ b/src/it/deploy-at-end-parallel/verify.groovy @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/dae/1.0/dae-1.0.pom" ).exists() +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/module1/1.0/module1-1.0.pom" ).exists() +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/module1/1.0/module1-1.0.jar" ).exists() +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/module2/1.0/module2-1.0.pom" ).exists() +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/module2/1.0/module2-1.0.jar" ).exists() +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/module3/1.0/module3-1.0.pom" ).exists() +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/module3/1.0/module3-1.0.jar" ).exists() +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/module4/1.0/module4-1.0.pom" ).exists() +assert new File( remoteRepo, "deploy-at-end-parallel/org/apache/maven/its/deploy/dae/parallel/module4/1.0/module4-1.0.jar" ).exists() + +File buildLog = new File( basedir, 'build.log' ) +assert buildLog.exists() +assert buildLog.text.contains( "[INFO] Deferring deploy for org.apache.maven.its.deploy.dae.parallel:dae:1.0 at end" ) +assert buildLog.text.contains( "[INFO] Deferring deploy for org.apache.maven.its.deploy.dae.parallel:module1:1.0 at end" ) +assert buildLog.text.contains( "[INFO] Deferring deploy for org.apache.maven.its.deploy.dae.parallel:module2:1.0 at end" ) +assert buildLog.text.contains( "[INFO] Deferring deploy for org.apache.maven.its.deploy.dae.parallel:module3:1.0 at end" ) +assert buildLog.text.contains( "[INFO] Deferring deploy for org.apache.maven.its.deploy.dae.parallel:module4:1.0 at end" ) + diff --git a/src/main/java/org/apache/maven/plugins/deploy/DeployMojo.java b/src/main/java/org/apache/maven/plugins/deploy/DeployMojo.java index 806b3f33..98d5ee99 100644 --- a/src/main/java/org/apache/maven/plugins/deploy/DeployMojo.java +++ b/src/main/java/org/apache/maven/plugins/deploy/DeployMojo.java @@ -23,6 +23,10 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -71,6 +75,14 @@ public class DeployMojo extends AbstractDeployMojo { @Parameter(defaultValue = "false", property = "deployAtEnd") private boolean deployAtEnd; + /** + * The number of parallel threads which will be used for deployment with the {@code deployAtEnd} feature. + * + * @since 3.1.0 + */ + @Parameter(property = "maven.deploy.parallelThreads", defaultValue = "1") + private int parallelThreads; + /** * Specifies an alternative repository to which the project artifacts should be deployed (other than those specified * in <distributionManagement>).
@@ -202,32 +214,93 @@ public void execute() throws MojoExecutionException, MojoFailureException { List allProjectsUsingPlugin = getAllProjectsUsingPlugin(); if (allProjectsMarked(allProjectsUsingPlugin)) { - Map requests = new LinkedHashMap<>(); - - // collect all arifacts from all modules to deploy - // requests are grouped by used remote repository - for (MavenProject reactorProject : allProjectsUsingPlugin) { - Map pluginContext = session.getPluginContext(pluginDescriptor, reactorProject); - State state = getState(pluginContext); - if (state == State.TO_BE_DEPLOYED) { - - RemoteRepository deploymentRepository = getDeploymentRepository( - reactorProject, - getPluginContextValue(pluginContext, DEPLOY_ALT_SNAPSHOT_DEPLOYMENT_REPOSITORY), - getPluginContextValue(pluginContext, DEPLOY_ALT_RELEASE_DEPLOYMENT_REPOSITORY), - getPluginContextValue(pluginContext, DEPLOY_ALT_DEPLOYMENT_REPOSITORY)); - - DeployRequest request = requests.computeIfAbsent(deploymentRepository, repo -> { - DeployRequest newRequest = new DeployRequest(); - newRequest.setRepository(repo); - return newRequest; - }); - processProject(reactorProject, request); - } + if (parallelThreads <= 1) { + deployAllAtOnce(allProjectsUsingPlugin); + } else { + deployInParallel(allProjectsUsingPlugin); } - // finally execute all deployments request, lets resolver to optimize deployment - for (DeployRequest request : requests.values()) { - deploy(request); + } + } + + private void deployAllAtOnce(List allProjectsUsingPlugin) throws MojoExecutionException { + Map requests = new LinkedHashMap<>(); + + // collect all arifacts from all modules to deploy + // requests are grouped by used remote repository + for (MavenProject reactorProject : allProjectsUsingPlugin) { + Map pluginContext = session.getPluginContext(pluginDescriptor, reactorProject); + State state = getState(pluginContext); + if (state == State.TO_BE_DEPLOYED) { + + RemoteRepository deploymentRepository = getDeploymentRepository( + reactorProject, + getPluginContextValue(pluginContext, DEPLOY_ALT_SNAPSHOT_DEPLOYMENT_REPOSITORY), + getPluginContextValue(pluginContext, DEPLOY_ALT_RELEASE_DEPLOYMENT_REPOSITORY), + getPluginContextValue(pluginContext, DEPLOY_ALT_DEPLOYMENT_REPOSITORY)); + + DeployRequest request = requests.computeIfAbsent(deploymentRepository, repo -> { + DeployRequest newRequest = new DeployRequest(); + newRequest.setRepository(repo); + return newRequest; + }); + processProject(reactorProject, request); + } + } + // finally execute all deployments request, lets resolver to optimize deployment + for (DeployRequest request : requests.values()) { + deploy(request); + } + } + + private void deployInParallel(List allProjectsUsingPlugin) throws MojoExecutionException { + + List> requestTasks = new ArrayList<>(); + AtomicBoolean hasError = new AtomicBoolean(false); + + for (MavenProject reactorProject : allProjectsUsingPlugin) { + Map pluginContext = session.getPluginContext(pluginDescriptor, reactorProject); + State state = getState(pluginContext); + if (state == State.TO_BE_DEPLOYED) { + + RemoteRepository deploymentRepository = getDeploymentRepository( + reactorProject, + getPluginContextValue(pluginContext, DEPLOY_ALT_SNAPSHOT_DEPLOYMENT_REPOSITORY), + getPluginContextValue(pluginContext, DEPLOY_ALT_RELEASE_DEPLOYMENT_REPOSITORY), + getPluginContextValue(pluginContext, DEPLOY_ALT_DEPLOYMENT_REPOSITORY)); + + DeployRequest request = new DeployRequest(); + request.setRepository(deploymentRepository); + + processProject(reactorProject, request); + + requestTasks.add(() -> { + try { + if (!hasError.get()) { + deploy(request); + } + } catch (MojoExecutionException e) { + hasError.set(true); + getLog().error(e.getMessage(), e); + } + return null; + }); + } + } + + if (!requestTasks.isEmpty()) { + ExecutorService executorService = Executors.newFixedThreadPool(parallelThreads); + + try { + executorService.invokeAll(requestTasks); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new MojoExecutionException(e.getMessage(), e); + } + + executorService.shutdown(); + + if (hasError.get()) { + throw new MojoExecutionException("Deployment errors"); } } } @@ -303,7 +376,7 @@ RemoteRepository getDeploymentRepository( final String altSnapshotDeploymentRepository, final String altReleaseDeploymentRepository, final String altDeploymentRepository) - throws MojoExecutionException, MojoFailureException { + throws MojoExecutionException { RemoteRepository repo = null; String altDeploymentRepo; @@ -330,7 +403,7 @@ RemoteRepository getDeploymentRepository( + "\" instead."); repo = getRemoteRepository(id, url); } else { - throw new MojoFailureException( + throw new MojoExecutionException( altDeploymentRepo, "Invalid legacy syntax and layout for repository.", "Invalid legacy syntax and layout for alternative repository. Use \"" + id + "::" + url @@ -340,7 +413,7 @@ RemoteRepository getDeploymentRepository( matcher = ALT_REPO_SYNTAX_PATTERN.matcher(altDeploymentRepo); if (!matcher.matches()) { - throw new MojoFailureException( + throw new MojoExecutionException( altDeploymentRepo, "Invalid syntax for repository.", "Invalid syntax for alternative repository. Use \"id::url\"."); diff --git a/src/test/java/org/apache/maven/plugins/deploy/DeployMojoTest.java b/src/test/java/org/apache/maven/plugins/deploy/DeployMojoTest.java index b99d7176..c43bdf5c 100644 --- a/src/test/java/org/apache/maven/plugins/deploy/DeployMojoTest.java +++ b/src/test/java/org/apache/maven/plugins/deploy/DeployMojoTest.java @@ -28,7 +28,6 @@ import org.apache.maven.execution.MavenSession; import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugin.MojoFailureException; import org.apache.maven.plugin.descriptor.PluginDescriptor; import org.apache.maven.plugin.testing.AbstractMojoTestCase; import org.apache.maven.plugin.testing.stubs.MavenProjectStub; @@ -581,7 +580,7 @@ public void testLegacyAltDeploymentRepositoryWithLegacyLayout() throws Exception try { mojo.getDeploymentRepository(project, null, null, "altDeploymentRepository::legacy::http://localhost"); fail("Should throw: Invalid legacy syntax and layout for repository."); - } catch (MojoFailureException e) { + } catch (MojoExecutionException e) { assertEquals(e.getMessage(), "Invalid legacy syntax and layout for repository."); assertEquals( e.getLongMessage(), @@ -602,7 +601,7 @@ public void testInsaneAltDeploymentRepository() throws Exception { mojo.getDeploymentRepository( project, null, null, "altDeploymentRepository::hey::wow::foo::http://localhost"); fail("Should throw: Invalid legacy syntax and layout for repository."); - } catch (MojoFailureException e) { + } catch (MojoExecutionException e) { assertEquals(e.getMessage(), "Invalid legacy syntax and layout for repository."); assertEquals( e.getLongMessage(), @@ -638,7 +637,7 @@ public void testLegacyScmSvnAltDeploymentRepository() throws Exception { mojo.getDeploymentRepository( project, null, null, "altDeploymentRepository::legacy::scm:svn:http://localhost"); fail("Should throw: Invalid legacy syntax and layout for repository."); - } catch (MojoFailureException e) { + } catch (MojoExecutionException e) { assertEquals(e.getMessage(), "Invalid legacy syntax and layout for repository."); assertEquals( e.getLongMessage(),