Skip to content

Commit

Permalink
[duckdb] Hello World (#1425)
Browse files Browse the repository at this point in the history
Created a new integrations/venice-duckdb module, which is currently pretty
barebones (just has the dependency and a unit test).

Miscellaneous build changes:

- The proto-related tasks are now only going to run in modules that have
  proto schemas defined.

- Also refactored the CI tasks a bit:

  - There is a new Integrations bucket in the unit tests, which includes
    all of the unit tests coming from integrations/ (previously, samza and
    pulsar unit tests ran in the Clients bucket).

  - The Alpini unit tests are moved to the Router bucket.

  - Reduced repetitions and boilerplate throughout the CI files.
  • Loading branch information
FelixGV authored Jan 10, 2025
1 parent 4672269 commit cf1accf
Show file tree
Hide file tree
Showing 6 changed files with 122 additions and 92 deletions.
63 changes: 2 additions & 61 deletions .github/workflows/VeniceCI-CompatibilityTests.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# GitHub Actions workflow for running compatibility tests: Avro, Alpini unit, Alpini functional tests, and Pulsar Venice integration tests
# GitHub Actions workflow for running compatibility tests: Avro and Pulsar Venice integration tests

name: TestsDeCompatibilite

Expand Down Expand Up @@ -50,65 +50,6 @@ jobs:
path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
retention-days: 30

AlpiniUnitTests:
strategy:
fail-fast: false
matrix:
jdk: [8, 11, 17]
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@v4
with:
# Checkout as many commits as needed for the diff
fetch-depth: 2
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up JDK
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: 'gradle'
# - name: Allow Deprecated TLS versions for Alpini tests
# run: |
# echo "java.security file before modifications: "
# cat "$JAVA_HOME/conf/security/java.security"

# # This is possibly flaky but
# sed -i 's/TLSv1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1
# sed -i 's/TLSv1.1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1.1

# echo "java.security file after modifications: "
# cat "$JAVA_HOME/conf/security/java.security"
- shell: bash
run: |
git remote set-head origin --auto
git remote add upstream https://github.com/linkedin/venice
git fetch upstream
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
with:
add-job-summary: never
- name: Run alpini unit tests
run: ./gradlew --continue --no-daemon -DmaxParallelForks=1 alpiniUnitTest
- name: Package Build Artifacts
if: (success() || failure())
shell: bash
run: |
mkdir ${{ github.job }}-artifacts
find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
- name: Upload Build Artifacts
if: (success() || failure())
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-jdk${{ matrix.jdk }}
path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
retention-days: 30

PulsarVeniceIntegrationTests:
strategy:
fail-fast: false
Expand Down Expand Up @@ -178,7 +119,7 @@ jobs:
strategy:
fail-fast: false
runs-on: ubuntu-latest
needs: [AvroCompatibilityTests, AlpiniUnitTests, PulsarVeniceIntegrationTests]
needs: [AvroCompatibilityTests, PulsarVeniceIntegrationTests]
timeout-minutes: 120
steps:
- name: AllIsWell
Expand Down
58 changes: 41 additions & 17 deletions .github/workflows/VeniceCI-StaticAnalysisAndUnitTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,46 +67,66 @@ jobs:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: clients
arg: :clients:venice-admin-tool:jacocoTestCoverageVerification :clients:venice-admin-tool:diffCoverage
:clients:venice-producer:jacocoTestCoverageVerification :clients:venice-producer:diffCoverage
:integrations:venice-pulsar:jacocoTestCoverageVerification :integrations:venice-pulsar:diffCoverage
:clients:venice-client:jacocoTestCoverageVerification :clients:venice-client:diffCoverage
:clients:venice-push-job:jacocoTestCoverageVerification :clients:venice-push-job:diffCoverage
:integrations:venice-samza:jacocoTestCoverageVerification :integrations:venice-samza:diffCoverage
:clients:venice-thin-client:jacocoTestCoverageVerification :clients:venice-thin-client:diffCoverage --continue
arg:
# Cannot use :clients:recursiveDiffCoverage because that would include DVC, and we want this one to run in server...
:clients:venice-admin-tool:diffCoverage
:clients:venice-producer:diffCoverage
:clients:venice-client:diffCoverage
:clients:venice-push-job:diffCoverage
:clients:venice-thin-client:diffCoverage
--continue

Integrations:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: integrations
arg:
:integrations:recursiveDiffCoverage
--continue

Internal:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: internal
arg: :internal:venice-client-common:jacocoTestCoverageVerification :internal:venice-client-common:diffCoverage
:internal:venice-common:jacocoTestCoverageVerification :internal:venice-common:diffCoverage
:internal:venice-jdk-compatibility-test:jacocoTestCoverageVerification :internal:venice-jdk-compatibility-test:diffCoverage
:internal:venice-test-common:jacocoTestCoverageVerification :internal:venice-test-common:diffCoverage --continue

arg:
# Cannot use :internal:recursiveDiffCoverage because that would include the avro compat test, and we want this one to run in the Compatibility group (TODO: move it out of internal?)...
:internal:venice-client-common:diffCoverage
:internal:venice-common:diffCoverage
:internal:venice-jdk-compatibility-test:diffCoverage
:internal:venice-test-common:diffCoverage
--continue

Controller:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: controller
arg: :services:venice-controller:jacocoTestCoverageVerification :services:venice-controller:diffCoverage --continue
arg:
:services:venice-controller:diffCoverage
--continue

Server:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: server
arg: :clients:da-vinci-client:jacocoTestCoverageVerification :clients:da-vinci-client:diffCoverage
:services:venice-server:jacocoTestCoverageVerification :services:venice-server:diffCoverage --continue
arg:
:clients:da-vinci-client:diffCoverage
:services:venice-server:diffCoverage
--continue

Router:
uses: ./.github/workflows/UnitTests-core.yml
with:
artifact_suffix: router
arg: :services:venice-router:jacocoTestCoverageVerification :services:venice-router:diffCoverage --continue
arg:
:services:venice-router:diffCoverage
alpiniUnitTest
--continue

StaticAnalysisAndUnitTestsCompletionCheck:
strategy:
fail-fast: false
runs-on: ubuntu-latest
needs: [ValidateGradleWrapper, StaticAnalysis, Clients, Internal, Controller, Server, Router]
needs: [ValidateGradleWrapper, StaticAnalysis, Clients, Integrations, Internal, Controller, Server, Router]
timeout-minutes: 120
if: success() || failure() # Always run this job, regardless of previous job status
steps:
Expand All @@ -128,6 +148,10 @@ jobs:
echo "Internal module unit tests failed."
exit 1
fi
if [ "${{ needs.Integrations.result }}" != "success" ]; then
echo "Integrations module unit tests failed."
exit 1
fi
if [ "${{ needs.Controller.result }}" != "success" ]; then
echo "Controller module unit tests failed."
exit 1
Expand Down
38 changes: 24 additions & 14 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ ext.libraries = [
commonsLang: 'commons-lang:commons-lang:2.6',
conscrypt: 'org.conscrypt:conscrypt-openjdk-uber:2.5.2',
d2: "com.linkedin.pegasus:d2:${pegasusVersion}",
duckdbJdbc: "org.duckdb:duckdb_jdbc:1.1.3",
failsafe: 'net.jodah:failsafe:2.4.0',
fastUtil: 'it.unimi.dsi:fastutil:8.3.0',
grpcNettyShaded: "io.grpc:grpc-netty-shaded:${grpcVersion}",
Expand Down Expand Up @@ -116,6 +117,11 @@ ext.libraries = [
mapreduceClientJobClient: "org.apache.hadoop:hadoop-mapreduce-client-jobclient:${hadoopVersion}",
mockito: 'org.mockito:mockito-core:4.11.0',
netty: 'io.netty:netty-all:4.1.74.Final',
opentelemetryApi: "io.opentelemetry:opentelemetry-api:${openTelemetryVersion}",
opentelemetrySdk: "io.opentelemetry:opentelemetry-sdk:${openTelemetryVersion}",
opentelemetryExporterLogging: "io.opentelemetry:opentelemetry-exporter-logging:${openTelemetryVersion}",
opentelemetryExporterOtlp: "io.opentelemetry:opentelemetry-exporter-otlp:${openTelemetryVersion}",
opentelemetryExporterCommon: "io.opentelemetry:opentelemetry-exporter-common:${openTelemetryVersion}",
oss: 'org.sonatype.oss:oss-parent:7',
pulsarClient: "${pulsarGroup}:pulsar-client:${pulsarVersion}",
pulsarIoCore: "${pulsarGroup}:pulsar-io-core:${pulsarVersion}",
Expand All @@ -142,12 +148,7 @@ ext.libraries = [
xerces: 'xerces:xercesImpl:2.9.1',
zkclient: 'com.101tec:zkclient:0.7', // For Kafka AdminUtils
zookeeper: 'org.apache.zookeeper:zookeeper:3.6.3',
zstd: 'com.github.luben:zstd-jni:1.5.2-3',
opentelemetryApi: "io.opentelemetry:opentelemetry-api:${openTelemetryVersion}",
opentelemetrySdk: "io.opentelemetry:opentelemetry-sdk:${openTelemetryVersion}",
opentelemetryExporterLogging: "io.opentelemetry:opentelemetry-exporter-logging:${openTelemetryVersion}",
opentelemetryExporterOtlp: "io.opentelemetry:opentelemetry-exporter-otlp:${openTelemetryVersion}",
opentelemetryExporterCommon: "io.opentelemetry:opentelemetry-exporter-common:${openTelemetryVersion}"
zstd: 'com.github.luben:zstd-jni:1.5.2-3'
]

group = 'com.linkedin.venice'
Expand Down Expand Up @@ -188,9 +189,14 @@ subprojects {

def isLeafSubModule = project.childProjects.isEmpty()

// We consider a sub-module to be a "proto module" if it has any proto schemas defined under the right path.
def protoDir = new File(project.projectDir, "src/main/proto");
def isProtoModule = protoDir != null && protoDir.list() != null && protoDir.list().size() != 0

apply {
plugin 'idea'
plugin 'java-library'
plugin 'com.form.diff-coverage'
plugin 'com.github.spotbugs'
plugin 'org.gradle.test-retry'
plugin 'org.checkerframework'
Expand All @@ -199,8 +205,9 @@ subprojects {
if (isLeafSubModule) {
apply {
plugin 'jacoco'
plugin 'com.form.diff-coverage'
plugin 'com.google.protobuf'
if (isProtoModule) {
plugin 'com.google.protobuf'
}
}
}

Expand All @@ -218,7 +225,7 @@ subprojects {
//withJavadocJar()
}

if (isLeafSubModule) {
if (isLeafSubModule && isProtoModule) {
protobuf {
protoc {
artifact = 'com.google.protobuf:protoc:' + protobufVersion
Expand Down Expand Up @@ -498,10 +505,6 @@ subprojects {
xml.enabled = true
html.enabled = true
}

doLast {
parseJacocoXml("$buildDir/reports/jacoco/test/jacocoTestReport.xml")
}
}

afterEvaluate {
Expand Down Expand Up @@ -582,6 +585,13 @@ subprojects {
}
}

// 2nd round of subprojects configuration... the 1st round must be fully done for all submodules for this one to work.
subprojects {
task recursiveDiffCoverage {
dependsOn subprojects.diffCoverage
}
}

task aggregateJavadoc(type: Javadoc) {
source subprojects.collect { project ->
project.sourceSets.main.allJava
Expand Down Expand Up @@ -829,4 +839,4 @@ task listSubprojects {
println "${subproject.name}"
}
}
}
}
10 changes: 10 additions & 0 deletions integrations/venice-duckdb/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
dependencies {
implementation libraries.duckdbJdbc
}

checkerFramework {
extraJavacArgs = ['-Xmaxerrs', '256']
checkers = ['org.checkerframework.checker.nullness.NullnessChecker']
skipCheckerFramework = true
excludeTests = true
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package com.linkedin.venice.duckdb;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.testng.annotations.Test;


/**
* The aim of this class is just to test DuckDB itself, without any Venice-ism involved.
*/
public class HelloWorldTest {
/**
* Lightly adapted from: https://duckdb.org/docs/api/java.html#querying
*/
@Test
public void test() throws SQLException {
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
Statement stmt = connection.createStatement()) {
// create a table
stmt.execute("CREATE TABLE items (item VARCHAR, value DECIMAL(10, 2), count INTEGER)");
// insert two items into the table
stmt.execute("INSERT INTO items VALUES ('jeans', 20.0, 1), ('hammer', 42.2, 2)");

try (ResultSet rs = stmt.executeQuery("SELECT * FROM items")) {
assertTrue(rs.next(), "There should be a first row!");
assertEquals(rs.getString(1), "jeans");
assertEquals(rs.getInt(3), 1);

assertTrue(rs.next(), "There should be a second row!");
assertEquals(rs.getString(1), "hammer");
assertEquals(rs.getInt(3), 2);

assertFalse(rs.next(), "There should only be two rows!");
}
}
}
}
1 change: 1 addition & 0 deletions settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ include 'internal:alpini:router:alpini-router-impl'

// 3rd-party system integration modules
include 'integrations:venice-beam'
include 'integrations:venice-duckdb'
include 'integrations:venice-pulsar'
include 'integrations:venice-samza'

Expand Down

0 comments on commit cf1accf

Please sign in to comment.