Skip to content

Commit

Permalink
[Feature] add dockerfile (#7346)
Browse files Browse the repository at this point in the history
  • Loading branch information
@liunaijie
liunaijie authored Aug 23, 2024
1 parent db53b49 commit 8a28290
Show file tree
Hide file tree
Showing 5 changed files with 250 additions and 22 deletions.
43 changes: 25 additions & 18 deletions .github/workflows/publish-docker.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,17 @@ name: publish-docker

on:
push:
tags:
- '*'
branches:
- dev
- docker
paths-ignore:
- 'docs/**'
- '**/*.md'
- 'seatunnel-ui/**'

env:
HUB: ghcr.io/${{ github.repository }}
DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

jobs:
build:
Expand All @@ -36,11 +37,16 @@ jobs:
permissions:
contents: read
packages: write
timeout-minutes: 30
timeout-minutes: 60
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
with:
submodules: true
- name: free disk space
run: tools/github/free_disk_space.sh
- uses: actions/checkout@v4
- name: Cache local Maven repository
uses: actions/cache@v2
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
Expand All @@ -51,26 +57,27 @@ jobs:
with:
java-version: 8
distribution: 'adopt'

- name: Log in to the Container registry
uses: docker/login-action@v1.10.0
uses: docker/login-action@v3
with:
registry: ${{ env.HUB }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
username: ${{ env.DOCKER_USERNAME }}
password: ${{ env.DOCKER_PASSWORD }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v3
- name: Build and push docker images
env:
MAVEN_OPTS: -Xmx2G -Xms2G
MAVEN_OPTS: -Xmx4096m
run: |
./mvnw -B clean deploy \
./mvnw -B clean install \
-Dmaven.test.skip \
-Dmaven.javadoc.skip \
-Dlicense.skipAddThirdParty=true \
-D"docker.build.skip"=false \
-D"docker.verify.skip"=false \
-D"docker.push.skip"=false \
-Dmaven.deploy.skip \
-Ddocker.tag=${{ github.sha }} \
-Ddocker.hub=${{ env.HUB }} \
-Pdocker \
--no-snapshot-updates
--no-snapshot-updates \
-Pdocker,seatunnel
103 changes: 100 additions & 3 deletions docs/en/start-v2/docker/docker.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,104 @@
sidebar_position: 3
---

# Set Up With Docker
# Set Up with Docker in local mode

## Zeta Engine

### Download

```shell
docker pull apache/seatunnel:<version_tag>
```

How to submit job in local mode

```shell
docker run --rm -it apache/seatunnel bash ./bin/seatunnel.sh -e local -c <CONFIG_FILE>


# eg: a fake source to console sink
docker run --rm -it apache/seatunnel bash ./bin/seatunnel.sh -e local -c config/v2.batch.config.template

```

### Build Image By Yourself

```Dockerfile
FROM openjdk:8

ARG VERSION
# Build from Source Code And Copy it into image
COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/

# Download From Internet
# Please Note this file only include fake/console connector, You'll need to download the other connectors manually
# wget -P /opt https://dlcdn.apache.org/seatunnel/2.3.6/apache-seatunnel-${VERSION}-bin.tar.gz

RUN cd /opt && \
tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \
mv apache-seatunnel-${VERSION} seatunnel && \
rm apache-seatunnel-${VERSION}-bin.tar.gz

WORKDIR /opt/seatunnel
```

## Spark or Flink Engine

### Download And Install the connectors you needed

refer the step as Zeta Engine

### Mount Spark/Flink library

By default, Spark home is `/opt/spark`, Flink home is `/opt/flink`.
If you need run with spark/flink, you can mount the related library to `/opt/spark` or `/opt/flink`.

```shell
docker run \
-v <SPARK_BINARY_PATH>:/opt/spark \
-v <FLINK_BINARY_PATH>:/opt/flink \
...
```

Or you can change the `SPARK_HOME`, `FLINK_HOME` environment variable in Dockerfile and re-build your and mount the spark/flink to related path.

```Dockerfile
FROM apache/seatunnel

ENV SPARK_HOME=<YOUR_CUSTOMIZATION_PATH>

...

```

```shell
docker run \
-v <SPARK_BINARY_PATH>:<YOUR_CUSTOMIZATION_PATH> \
...
```

### Submit job

The command is different for different engines and different versions of the same engine, please choose the correct command.

- Spark

```shell
# spark2
docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-2-connector-v2.sh -c config/v2.batch.config.template

# spark3
docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-3-connector-v2.sh -c config/v2.batch.config.template
```

- Flink
before you submit job, you need start flink cluster first.

```shell
# flink version between `1.12.x` and `1.14.x`
docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-13-connector-v2.sh -c config/v2.streaming.conf.template'
# flink version between `1.15.x` and `1.16.x`
docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-15-connector-v2.sh -c config/v2.streaming.conf.template'
```

<!-- TODO -->
-->
93 changes: 93 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,11 @@
<codec.version>1.13</codec.version>
<exec-maven-plugin.version>3.0.0</exec-maven-plugin.version>
<docker.hub>apache</docker.hub>
<docker.repo>seatunnel</docker.repo>
<docker.tag>${project.version}</docker.tag>
<docker.build.skip>true</docker.build.skip>
<docker.verify.skip>true</docker.verify.skip>
<docker.push.skip>true</docker.push.skip>
<jcommander.version>1.81</jcommander.version>
<junit4.version>4.13.2</junit4.version>
<junit5.version>5.9.0</junit5.version>
Expand Down Expand Up @@ -760,6 +764,95 @@
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>${exec-maven-plugin.version}</version>
<executions>
<execution>
<id>docker-build</id>
<goals>
<goal>exec</goal>
</goals>
<phase>package</phase>
<configuration>
<skip>${docker.build.skip}</skip>
<environmentVariables>
<DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
</environmentVariables>
<executable>docker</executable>
<workingDirectory>${project.basedir}</workingDirectory>
<arguments>
<argument>buildx</argument>
<argument>build</argument>
<argument>--load</argument>
<argument>--no-cache</argument>
<argument>-t</argument>
<argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
<argument>${project.basedir}</argument>
<argument>--build-arg</argument>
<argument>VERSION=${project.version}</argument>
<argument>--file=src/main/docker/Dockerfile</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>docker-verify</id>
<goals>
<goal>exec</goal>
</goals>
<phase>verify</phase>
<configuration>
<skip>${docker.verify.skip}</skip>
<environmentVariables>
<DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
</environmentVariables>
<executable>docker</executable>
<workingDirectory>${project.basedir}</workingDirectory>
<arguments>
<argument>run</argument>
<argument>--rm</argument>
<argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
<argument>bash</argument>
<argument>./bin/seatunnel.sh</argument>
<argument>-e</argument>
<argument>local</argument>
<argument>-c</argument>
<argument>config/v2.batch.config.template</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>docker-push</id>
<goals>
<goal>exec</goal>
</goals>
<phase>install</phase>
<configuration>
<skip>${docker.push.skip}</skip>
<environmentVariables>
<DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
</environmentVariables>
<executable>docker</executable>
<workingDirectory>${project.basedir}</workingDirectory>
<arguments>
<argument>buildx</argument>
<argument>build</argument>
<argument>--platform</argument>
<argument>linux/amd64,linux/arm64</argument>
<argument>--no-cache</argument>
<argument>--push</argument>
<argument>-t</argument>
<argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
<argument>${project.basedir}</argument>
<argument>--build-arg</argument>
<argument>VERSION=${revision}</argument>
<argument>--file=src/main/docker/Dockerfile</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>

Expand Down
21 changes: 20 additions & 1 deletion seatunnel-dist/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>

</executions>
</plugin>
</plugins>
Expand All @@ -83,6 +82,9 @@
</property>
</activation>
<properties>
<docker.build.skip>false</docker.build.skip>
<docker.verify.skip>false</docker.verify.skip>
<docker.push.skip>false</docker.push.skip>
<mysql.version>8.0.27</mysql.version>
<postgresql.version>42.4.3</postgresql.version>
<postgis.jdbc.version>2.5.1</postgis.jdbc.version>
Expand Down Expand Up @@ -965,5 +967,22 @@
</plugins>
</build>
</profile>
<profile>
<id>docker</id>
<activation>
<property>
<name>release</name>
<value>false</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
12 changes: 12 additions & 0 deletions seatunnel-dist/src/main/docker/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
FROM openjdk:8

ARG VERSION

COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/

RUN cd /opt && \
tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \
mv apache-seatunnel-${VERSION} seatunnel && \
rm apache-seatunnel-${VERSION}-bin.tar.gz

WORKDIR /opt/seatunnel

0 comments on commit 8a28290

Please sign in to comment.