Skip to content

Commit 2b12ba4

Browse files
JingsongLiJingsongLi
committed
[vortex] Introduce vortex file format integration
1 parent 2021788 commit 2b12ba4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+6401
-2
lines changed

‎.github/workflows/utitcase-vortex.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
name: UTCase Vortex
20+
21+
on:
22+
push:
23+
paths:
24+
- 'paimon-vortex/**'
25+
pull_request:
26+
paths:
27+
- 'paimon-vortex/**'
28+
29+
env:
30+
JDK_VERSION: 8
31+
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
32+
33+
concurrency:
34+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
35+
cancel-in-progress: true
36+
37+
jobs:
38+
vortex_test:
39+
runs-on: ubuntu-latest
40+
41+
steps:
42+
- name: Checkout code
43+
uses: actions/checkout@v6
44+
45+
- name: Set up JDK ${{ env.JDK_VERSION }}
46+
uses: actions/setup-java@v5
47+
with:
48+
java-version: ${{ env.JDK_VERSION }}
49+
distribution: 'temurin'
50+
51+
- name: Install Rust toolchain
52+
uses: dtolnay/rust-toolchain@stable
53+
54+
- name: Clone and build Vortex native library
55+
run: |
56+
git clone --depth 1 https://github.com/spiraldb/vortex.git ${RUNNER_TEMP}/vortex
57+
cd ${RUNNER_TEMP}/vortex
58+
cargo build --package vortex-jni --release
59+
60+
- name: Copy native library to resources
61+
run: |
62+
RESOURCE_DIR=paimon-vortex/paimon-vortex-jni/src/main/resources/native/linux-amd64
63+
mkdir -p ${RESOURCE_DIR}
64+
cp ${RUNNER_TEMP}/vortex/target/release/libvortex_jni.so ${RESOURCE_DIR}/
65+
66+
- name: Build and test Vortex modules
67+
timeout-minutes: 30
68+
run: |
69+
mvn -T 2C -B -ntp clean install -DskipTests
70+
mvn -B -ntp verify -pl paimon-vortex/paimon-vortex-jni,paimon-vortex/paimon-vortex-format -Dcheckstyle.skip=true -Dspotless.check.skip=true
71+
env:
72+
MAVEN_OPTS: -Xmx4096m

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,7 @@ paimon-python/dev/log
4343
### Misc ###
4444
*.swp
4545
.cache
46+
47+
### Vortex lib ###
48+
49+
*libvortex_jni*

‎paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848

4949
import javax.annotation.Nullable;
5050

51+
import java.io.ByteArrayOutputStream;
5152
import java.io.IOException;
5253
import java.io.OutputStream;
5354
import java.time.Instant;
@@ -276,11 +277,15 @@ public static ArrowCStruct serializeToCStruct(
276277
return ArrowCStruct.of(array, schema);
277278
}
278279

280+
public static byte[] serializeToIpc(VectorSchemaRoot vsr) {
281+
ByteArrayOutputStream out = new ByteArrayOutputStream();
282+
serializeToIpc(vsr, out);
283+
return out.toByteArray();
284+
}
285+
279286
public static void serializeToIpc(VectorSchemaRoot vsr, OutputStream out) {
280287
try (ArrowStreamWriter writer = new ArrowStreamWriter(vsr, null, out)) {
281-
writer.start();
282288
writer.writeBatch();
283-
writer.end();
284289
} catch (IOException e) {
285290
throw new RuntimeException("Failed to serialize VectorSchemaRoot to IPC", e);
286291
}

‎paimon-vortex/paimon-vortex-format/pom.xml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
25+
<parent>
26+
<artifactId>paimon-vortex</artifactId>
27+
<groupId>org.apache.paimon</groupId>
28+
<version>1.5-SNAPSHOT</version>
29+
</parent>
30+
31+
<artifactId>paimon-vortex-format</artifactId>
32+
<name>Paimon : Vortex : Format</name>
33+
34+
<dependencies>
35+
<dependency>
36+
<groupId>org.apache.paimon</groupId>
37+
<artifactId>paimon-vortex-jni</artifactId>
38+
<version>${project.version}</version>
39+
</dependency>
40+
41+
<dependency>
42+
<groupId>org.apache.paimon</groupId>
43+
<artifactId>paimon-arrow</artifactId>
44+
<version>${project.version}</version>
45+
</dependency>
46+
47+
<dependency>
48+
<groupId>org.apache.paimon</groupId>
49+
<artifactId>paimon-common</artifactId>
50+
<version>${project.version}</version>
51+
<scope>provided</scope>
52+
</dependency>
53+
54+
<dependency>
55+
<groupId>org.apache.paimon</groupId>
56+
<artifactId>paimon-core</artifactId>
57+
<version>${project.version}</version>
58+
<scope>provided</scope>
59+
</dependency>
60+
61+
<!-- test dependencies -->
62+
63+
<dependency>
64+
<groupId>org.apache.paimon</groupId>
65+
<artifactId>paimon-common</artifactId>
66+
<version>${project.version}</version>
67+
<type>test-jar</type>
68+
<scope>test</scope>
69+
</dependency>
70+
71+
<dependency>
72+
<groupId>org.apache.paimon</groupId>
73+
<artifactId>paimon-test-utils</artifactId>
74+
<version>${project.version}</version>
75+
<scope>test</scope>
76+
</dependency>
77+
78+
<dependency>
79+
<groupId>org.apache.paimon</groupId>
80+
<artifactId>paimon-core</artifactId>
81+
<version>${project.version}</version>
82+
<type>test-jar</type>
83+
<scope>test</scope>
84+
</dependency>
85+
86+
<dependency>
87+
<groupId>org.apache.paimon</groupId>
88+
<artifactId>paimon-format</artifactId>
89+
<version>${project.version}</version>
90+
<scope>test</scope>
91+
</dependency>
92+
</dependencies>
93+
</project>

‎paimon-vortex/paimon-vortex-format/src/main/java/org/apache/arrow/memory/BoundsChecking.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.arrow.memory;
20+
21+
/** Overwrite BoundsChecking in arrow, disable bounds checkpoint always. */
22+
public class BoundsChecking {
23+
public static final boolean BOUNDS_CHECKING_ENABLED = false;
24+
}

0 commit comments

Comments
 (0)