From bf0d2effbf7bc819f67ff26cb4cda6abcb0b5865 Mon Sep 17 00:00:00 2001 From: Prashant Kumar Date: Thu, 30 Sep 2021 13:18:36 -0700 Subject: [PATCH 1/5] Migrate bookkeepr-server:test to gradle run unit tests excepts org.apache.bookkeeper.bookie. org.apache.bookkeeper.client org.apache.bookkeeper.replication org.apache.bookkeeper.tls. --- .github/workflows/remaining-tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/remaining-tests.yml b/.github/workflows/remaining-tests.yml index 6176aa42ec0..3b2ca345ac6 100644 --- a/.github/workflows/remaining-tests.yml +++ b/.github/workflows/remaining-tests.yml @@ -29,9 +29,6 @@ on: - 'site/**' workflow_dispatch: -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - jobs: test: @@ -49,4 +46,4 @@ jobs: with: java-version: 1.8 - name: Run remaining tests - run: mvn -B -nsu -am -pl bookkeeper-server clean install test -Dtest="!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + run: ./gradlew bookkeeper-server:test -PexcludeTests="**org.apache.bookkeeper.bookie.*, **org.apache.bookkeeper.client.*, **org.apache.bookkeeper.replication.*, **org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain \ No newline at end of file From 46770d2266102cdcc7b7238d4e51d0c5d1213899 Mon Sep 17 00:00:00 2001 From: Prashant Kumar Date: Fri, 1 Oct 2021 10:56:22 -0700 Subject: [PATCH 2/5] Rest of the tests --- .github/workflows/compatibility-check-java11.yml | 4 ++-- .github/workflows/compatibility-check-java8.yml | 4 ++-- .github/workflows/integration-tests.yml | 13 ++++++------- .github/workflows/pr-validation.yml | 8 ++++++-- .github/workflows/replication-tests.yml | 2 +- .github/workflows/stream-tests.yml | 2 +- .github/workflows/tls-tests.yml | 5 +---- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index 4a49a9aeaa9..a1e90ad2e92 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -48,5 +48,5 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.11 - - name: Build with Maven - run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + - name: Build with gradle + run: ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml index fda71a59317..e80fa0be8f2 100644 --- a/.github/workflows/compatibility-check-java8.yml +++ b/.github/workflows/compatibility-check-java8.yml @@ -48,5 +48,5 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build with Maven - run: mvn clean package -B -nsu spotbugs:check -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + - name: Build with gradle + run: ./gradlew test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 97a3c82702c..9cda4d88076 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -48,11 +48,10 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build with Maven - run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - name: Run metadata driver tests - run: mvn -B -nsu -f metadata-drivers/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - - - name: Run all integration tests - run: mvn -B -nsu -f tests/pom.xml test -DintegrationTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + - name: run cluster integration test + run: ./gradlew :tests:integration:cluster:test -Dtestlogger.theme=plain + - name: run smoke test + run: ./gradlew tests:integration:smoke -Dtestlogger.theme=plain + - name: run standalone test + run: ./gradlew tests:integration:standalone -Dtestlogger.theme=plain diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index edcebaa484b..7b491384415 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -48,7 +48,11 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Validate pull request style - run: mvn clean -B -nsu apache-rat:check checkstyle:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + - name: Validate pull request style validate license headers + run: ./gradlew rat + - name: Validate pull request style validate style + run: ./gradlew checkstyleMain -x microbenchmarks:checkstyleMain + - name: Validate pull request style validate style + run: ./gradlew spotbugsMain - name: Check license files run: dev/check-all-licenses diff --git a/.github/workflows/replication-tests.yml b/.github/workflows/replication-tests.yml index 8dd6ea6ee6c..c5bd88ea081 100644 --- a/.github/workflows/replication-tests.yml +++ b/.github/workflows/replication-tests.yml @@ -49,4 +49,4 @@ jobs: with: java-version: 1.8 - name: Run replication tests - run: mvn -B -nsu -am -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.replication.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" -Dtestlogger.theme=plain diff --git a/.github/workflows/stream-tests.yml b/.github/workflows/stream-tests.yml index ca4a7074f5a..87eff403a33 100644 --- a/.github/workflows/stream-tests.yml +++ b/.github/workflows/stream-tests.yml @@ -50,4 +50,4 @@ jobs: - name: Build with Maven run: mvn -B -nsu clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: Run StreamStorage tests - run: mvn -B -nsu -f stream/pom.xml verify -DstreamTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + run: ./gradlew build -x bookkeeper-server:test -x checkstyleMain -x checkstyleTest -x spotbugsTest -x spotbugsMain -x rat -x signDistTar -Dtestlogger.theme=plain diff --git a/.github/workflows/tls-tests.yml b/.github/workflows/tls-tests.yml index 45748c25fb7..00fb6f3d9db 100644 --- a/.github/workflows/tls-tests.yml +++ b/.github/workflows/tls-tests.yml @@ -29,9 +29,6 @@ on: - 'site/**' workflow_dispatch: -env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 - jobs: test: @@ -49,4 +46,4 @@ jobs: with: java-version: 1.8 - name: Run tls tests - run: mvn -B -am -nsu -pl bookkeeper-server clean install test -Dtest="org.apache.bookkeeper.tls.**" -DfailIfNoTests=false -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain From 1571edcacf03871f0c2dfdc24b2f3d65bae26b01 Mon Sep 17 00:00:00 2001 From: Prashant Kumar Date: Tue, 5 Oct 2021 13:15:21 -0700 Subject: [PATCH 3/5] Exclude target directory from RAT plugin --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 6bb45a614e9..19b52bd5db6 100644 --- a/build.gradle +++ b/build.gradle @@ -101,6 +101,7 @@ allprojects { '**/out/**', '**/generated/**', '**/build/**', + '**/target/**', '**/.idea/**', '.git/**/*', '.github/**/*', From 6f78290751a76212098114b44708e98850643fd2 Mon Sep 17 00:00:00 2001 From: Prashant Kumar Date: Tue, 5 Oct 2021 13:23:13 -0700 Subject: [PATCH 4/5] Not needed --- .../bookkeeper/tools/cli/helpers/CookieCommandTestBase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/CookieCommandTestBase.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/CookieCommandTestBase.java index f6f66b6e2fa..d8ba2fb2236 100644 --- a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/CookieCommandTestBase.java +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/helpers/CookieCommandTestBase.java @@ -36,7 +36,6 @@ /** * A test base for testing cookie commands. */ -@RunWith(PowerMockRunner.class) @PrepareForTest({ MetadataDrivers.class }) public class CookieCommandTestBase extends CommandTestBase { From 5411bc98b3cc8eeb87c9990aa17bde5bfdfe1631 Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Tue, 5 Oct 2021 18:39:38 -0700 Subject: [PATCH 5/5] Building bookkeeper with gradle on java11 --- build.gradle | 1 + .../main/resources/bookkeeper/checkstyle.xml | 2 +- circe-checksum/build.gradle | 4 +-- .../com/scurrilous/circe/crc/Sse42Crc32C.java | 25 ++++++++++++++----- cpu-affinity/build.gradle | 10 +++++--- .../util/affinity/impl/CpuAffinityJni.java | 3 --- dependencies.gradle | 2 ++ stream/distributedlog/core/build.gradle | 1 + stream/proto/build.gradle | 1 + stream/server/build.gradle | 1 + stream/tests-common/build.gradle | 1 + 11 files changed, 35 insertions(+), 16 deletions(-) diff --git a/build.gradle b/build.gradle index 19b52bd5db6..2c25bc235f7 100644 --- a/build.gradle +++ b/build.gradle @@ -73,6 +73,7 @@ allprojects { checkstyle { toolVersion "${checkStyleVersion}" configFile file("$rootDir/buildtools/src/main/resources/bookkeeper/checkstyle.xml") + configDirectory = file("$rootDir/buildtools/src/main/resources/bookkeeper") checkstyleMain { source ='src/main/java' } diff --git a/buildtools/src/main/resources/bookkeeper/checkstyle.xml b/buildtools/src/main/resources/bookkeeper/checkstyle.xml index b79bfdd3f24..fda4fb619d6 100644 --- a/buildtools/src/main/resources/bookkeeper/checkstyle.xml +++ b/buildtools/src/main/resources/bookkeeper/checkstyle.xml @@ -60,7 +60,7 @@ page at http://checkstyle.sourceforge.net/config.html --> - + diff --git a/circe-checksum/build.gradle b/circe-checksum/build.gradle index 17258fee0b2..f469ef299a5 100644 --- a/circe-checksum/build.gradle +++ b/circe-checksum/build.gradle @@ -34,8 +34,8 @@ task generateJniHeaders(type:Exec) { } dependsOn classes def classpath = sourceSets.main.output.classesDirs.join(":") - commandLine("javah", "-d", javahOutputDir, "-classpath", - classpath, "com.scurrilous.circe.crc.Sse42Crc32C") + commandLine("javac", "-h", javahOutputDir, "-classpath", + classpath, "./src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java") } jar { diff --git a/circe-checksum/src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java b/circe-checksum/src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java index 22379a0e32d..70b7cf6b113 100644 --- a/circe-checksum/src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java +++ b/circe-checksum/src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java @@ -18,6 +18,7 @@ import static com.scurrilous.circe.utils.NativeUtils.loadLibraryFromJar; import static com.scurrilous.circe.utils.NativeUtils.libType; +import java.lang.ref.Cleaner; import java.nio.ByteBuffer; import com.scurrilous.circe.IncrementalIntHash; import com.scurrilous.circe.impl.AbstractIncrementalIntHash; @@ -54,6 +55,23 @@ public static boolean isSupported() { config = 0; } + // A cleaner similar to a finalizer(deprecated in java11) + private static final Cleaner cleaner = Cleaner.create(); + + static class State implements Runnable { + private final long config; + State(long config) { + // initialize State needed for cleaning action + this.config = config; + } + + public void run() { + // cleanup action accessing State, executed at most once + if (this.config != 0) + freeConfig(this.config); + } + } + public Sse42Crc32C(int[] chunkWords) { if (chunkWords.length == 0) { config = 0; @@ -62,12 +80,7 @@ public Sse42Crc32C(int[] chunkWords) { if (config == 0) throw new RuntimeException("CRC32C configuration allocation failed"); } - } - - @Override - protected void finalize() { - if (config != 0) - freeConfig(config); + cleaner.register(config, new State(config)); } @Override diff --git a/cpu-affinity/build.gradle b/cpu-affinity/build.gradle index 981fd6df381..d4249918230 100644 --- a/cpu-affinity/build.gradle +++ b/cpu-affinity/build.gradle @@ -16,19 +16,21 @@ * specific language governing permissions and limitations * under the License. */ + plugins { id 'java' } dependencies { - compileOnly depLibs.lombok compileOnly depLibs.spotbugsAnnotations implementation depLibs.commonsLang3 implementation depLibs.guava implementation depLibs.slf4j testImplementation depLibs.junit - + compileOnly depLibs.lombok annotationProcessor depLibs.lombok + testCompileOnly depLibs.lombok + testAnnotationProcessor depLibs.lombok } task generateJniHeaders(type:Exec) { @@ -37,8 +39,8 @@ task generateJniHeaders(type:Exec) { } dependsOn classes def classpath = sourceSets.main.output.classesDirs.join(":") - commandLine("javah", "-d", javahOutputDir, "-classpath", - classpath, "org.apache.bookkeeper.common.util.affinity.impl.CpuAffinityJni") + commandLine("javac", "-h", javahOutputDir, "-classpath", + classpath, "./src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityJni.java") } jar { diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityJni.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityJni.java index 30ee325bca4..418d8a88175 100644 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityJni.java +++ b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityJni.java @@ -22,12 +22,9 @@ import java.io.IOException; -import lombok.experimental.UtilityClass; - /** * JNI wrapper of native functions for CPU affinity. */ -@UtilityClass public class CpuAffinityJni { static native boolean isRoot(); diff --git a/dependencies.gradle b/dependencies.gradle index c3159aef1b2..e7e559ee11a 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -48,6 +48,7 @@ depVersions = [ hdrhistogram: "2.1.4", httpclient: "4.5.13", jackson: "2.11.1", + javaAnnotations:"1.3.2", jcommander: "1.78", jctools: "2.1.2", jetty: "9.4.31.v20200723", @@ -111,6 +112,7 @@ depLibs = [ hamcrest: "org.hamcrest:hamcrest-all:${depVersions.hamcrest}", httpclient: "org.apache.httpcomponents:httpclient:${depVersions.httpclient}", jacksonAnnotations: "com.fasterxml.jackson.core:jackson-annotations:${depVersions.jackson}", + javaAnnotations: "javax.annotation:javax.annotation-api:${depVersions.javaAnnotations}", jacksonCore: "com.fasterxml.jackson.core:jackson-core:${depVersions.jackson}", jacksonDatabind: "com.fasterxml.jackson.core:jackson-databind:${depVersions.jackson}", jcommander: "com.beust:jcommander:${depVersions.jcommander}", diff --git a/stream/distributedlog/core/build.gradle b/stream/distributedlog/core/build.gradle index e4a65eacbef..ca739376dec 100644 --- a/stream/distributedlog/core/build.gradle +++ b/stream/distributedlog/core/build.gradle @@ -49,6 +49,7 @@ dependencies { testImplementation project(path: ':stream:distributedlog:common', configuration: 'testArtifacts') testImplementation depLibs.junit testImplementation depLibs.mockito + implementation depLibs.javaAnnotations annotationProcessor depLibs.lombok testAnnotationProcessor depLibs.lombok diff --git a/stream/proto/build.gradle b/stream/proto/build.gradle index cd90529e2b7..cc27288f437 100644 --- a/stream/proto/build.gradle +++ b/stream/proto/build.gradle @@ -25,6 +25,7 @@ dependencies { implementation depLibs.lombok implementation depLibs.grpc implementation depLibs.commonsLang3 + implementation depLibs.javaAnnotations annotationProcessor depLibs.lombok } diff --git a/stream/server/build.gradle b/stream/server/build.gradle index 08f2403ac4e..7eb3510848e 100644 --- a/stream/server/build.gradle +++ b/stream/server/build.gradle @@ -55,6 +55,7 @@ dependencies { runtimeOnly depLibs.vertxCore runtimeOnly depLibs.vertxWeb testImplementation depLibs.mockito + implementation depLibs.javaAnnotations annotationProcessor depLibs.lombok } diff --git a/stream/tests-common/build.gradle b/stream/tests-common/build.gradle index f3c33c78c56..5259cd6e0e9 100644 --- a/stream/tests-common/build.gradle +++ b/stream/tests-common/build.gradle @@ -26,6 +26,7 @@ dependencies { implementation depLibs.slf4j annotationProcessor depLibs.lombok + implementation depLibs.javaAnnotations } protobuf {