feat(format): support Vortex file format

.github/.rat-excludes

@@ -12,6 +12,7 @@ iwyu_tool\.py
 arrow\.diff
 jieba\.diff
 orc\.diff
+vortex\.diff
 licenses/*
 .codespell_ignore
 .gitignore

.github/workflows/build_and_test.yaml

@@ -114,6 +114,9 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y gdb libcurl4-openssl-dev libssl-dev
+      - name: Install flatc and libclang (Vortex)
+        shell: bash
+        run: ci/scripts/setup_flatc.sh
       - name: Build Paimon
         shell: bash
         env:

.github/workflows/gcc8_test.yaml

@@ -67,6 +67,9 @@ jobs:
         uses: ./.github/actions/setup-ccache
         with:
           cache-key-prefix: ccache-gcc8-test
+      - name: Install flatc and libclang (Vortex)
+        shell: bash
+        run: ci/scripts/setup_flatc.sh
       - name: Build Paimon
         shell: bash
         env:

.github/workflows/release_candidate.yaml

@@ -135,6 +135,10 @@ jobs:
         shell: bash
         run: ci/scripts/setup_rust.sh
 
+      - name: Install flatc and libclang (Vortex)
+        shell: bash
+        run: ci/scripts/setup_flatc.sh
+
       - name: Install HTTP and TLS development dependencies
         run: |
           sudo apt-get update

CMakeLists.txt

@@ -62,6 +62,7 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_MOSAIC "Whether to enable mosaic file format (Rust FFI)" OFF)
 option(PAIMON_ENABLE_LANCE "Whether to enable lance file format (Rust FFI)" OFF)
+option(PAIMON_ENABLE_VORTEX "Whether to enable vortex file format (Rust FFI)" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
 option(PAIMON_ENABLE_OSS "Whether to enable OSS SDK v2 file system" OFF)
 option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
@@ -97,6 +98,9 @@ endif()
 if(PAIMON_ENABLE_LANCE)
     add_definitions(-DPAIMON_ENABLE_LANCE)
 endif()
+if(PAIMON_ENABLE_VORTEX)
+    add_definitions(-DPAIMON_ENABLE_VORTEX)
+endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
@@ -419,6 +423,10 @@ if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS)
         paimon_link_libraries_whole_archive(PAIMON_LANCE_FILE_FORMAT_STATIC_LINK_LIBS
                                             paimon_lance_file_format_static)
     endif()
+    if(PAIMON_ENABLE_VORTEX)
+        paimon_link_libraries_whole_archive(PAIMON_VORTEX_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_vortex_file_format_static)
+    endif()
 
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS
@@ -486,6 +494,13 @@ if(PAIMON_BUILD_TESTS)
         paimon_link_libraries_whole_archive(PAIMON_LANCE_FILE_FORMAT_STATIC_LINK_LIBS
                                             paimon_lance_file_format_static)
     endif()
+    if(PAIMON_ENABLE_VORTEX)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_vortex_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+        paimon_link_libraries_whole_archive(PAIMON_VORTEX_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_vortex_file_format_static)
+    endif()
 
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
@@ -582,6 +597,9 @@ endif()
 if(PAIMON_ENABLE_LANCE)
     add_subdirectory(src/paimon/format/lance)
 endif()
+if(PAIMON_ENABLE_VORTEX)
+    add_subdirectory(src/paimon/format/vortex)
+endif()
 if(PAIMON_ENABLE_LUMINA)
     add_subdirectory(src/paimon/global_index/lumina)
 endif()

build_support/lsan-suppressions.txt

@@ -17,3 +17,8 @@
 
 # False positive from atexit() registration in libc
 leak:*__new_exitfn*
+
+# Vortex links an un-instrumented Rust staticlib whose runtime keeps allocations alive at process
+# exit; these are not paimon leaks. Rust crate frames are prefixed vortex_<crate>, which does not
+# match paimon::vortex:: C++ frames, so genuine paimon leaks are still reported.
+leak:vortex_

build_support/tsan-suppressions.txt

@@ -18,3 +18,26 @@
 # Prebuilt shared libraries are not TSAN-instrumented. Suppress reports from the whole library.
 race:liblumina.so
 race:libjindosdk_c.so.6
+
+# Vortex links an un-instrumented Rust staticlib. The handoff between its blocking-pool threads
+# (which invoke the paimon read/write callbacks) and the calling thread happens inside Rust, so
+# TSAN cannot see the happens-before edge and reports false data races on the shared buffers.
+# These patterns match only the Rust frames, so races confined to paimon C++ are still reported.
+race:vortex_buffer
+race:vortex_io
+race:CallbackReadAt
+# Vortex also deserializes its footer/dtype and drives segment reads from internal runtime threads;
+# that concurrency lives inside the same un-instrumented Rust staticlib (dtype flatbuffers decode,
+# shared-future polling, segment cache and mask handling), so TSAN reports false races whose only
+# non-paimon frames are these Rust modules. Patterns match Rust frames only.
+race:vortex_array
+race:vortex_error
+race:vortex_file
+race:vortex_layout
+race:vortex_mask
+# The Arrow C-bridge hands buffer ownership to/from the Rust side (arrow_buffer Allocation and
+# bytes::Bytes), and Vortex formats its error messages on its runtime threads; those
+# synchronizations also live in the un-instrumented Rust staticlib, so suppress those Rust frames.
+race:arrow_buffer
+race:bytes::bytes::Bytes
+race:alloc::string::String

ci/scripts/build_paimon.sh

@@ -151,6 +151,7 @@ CMAKE_ARGS=(
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_MOSAIC=ON"
     "-DPAIMON_ENABLE_LANCE=${ENABLE_LANCE}"
+    "-DPAIMON_ENABLE_VORTEX=ON"
     "-DPAIMON_ENABLE_JINDO=ON"
     "-DPAIMON_ENABLE_OSS=ON"
     "-DPAIMON_ENABLE_S3=ON"

ci/scripts/setup_flatc.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Build and install flatc (the FlatBuffers compiler) at the version Vortex requires, and make sure
+# libclang is present.
+#
+# Vortex compiles its FlatBuffers schemas (.fbs) into Rust at build time, and
+# cmake_modules/ThirdpartyToolchain.cmake requires a flatc whose version matches Vortex's
+# flatbuffers crate (find_program(... flatc REQUIRED)). Distribution packages are too old, so the
+# pinned version is built from source. Separately, Vortex's native dependencies run bindgen in
+# their build scripts, which needs libclang at build time; some CI images do not ship it.
+# flatc and libclang are build-time tools only; neither is needed at runtime.
+#
+# The dev container (see .devcontainer/) already has these preinstalled; this script is for the
+# GitHub Actions runners and is called before ci/scripts/build_paimon.sh.
+#
+# Idempotent: a no-op when a matching flatc is already on PATH and libclang is already present.
+
+set -eux
+
+# Must match the flatbuffers crate version pinned by Vortex (see its Cargo.lock).
+FLATBUFFERS_VERSION=${FLATBUFFERS_VERSION:-25.12.19}
+FLATC_INSTALL_DIR=${FLATC_INSTALL_DIR:-"${HOME}/.local/bin"}
+
+# Vortex's native dependencies invoke bindgen in their build scripts, which loads libclang at
+# build time. Images that only carry GCC (e.g. the gcc8 test container) may lack it.
+if ! ldconfig -p 2>/dev/null | grep -q 'libclang'; then
+    if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1; then
+        apt_prefix="sudo"
+    else
+        apt_prefix=""
+    fi
+    ${apt_prefix} apt-get update -y
+    ${apt_prefix} apt-get install -y libclang-dev
+fi
+
+# Skip when a matching flatc is already available.
+if command -v flatc >/dev/null 2>&1 &&
+    [[ "$(flatc --version 2>/dev/null | awk '{print $NF}')" == "${FLATBUFFERS_VERSION}" ]]; then
+    flatc --version
+    exit 0
+fi
+
+workdir="$(mktemp -d)"
+trap 'rm -rf "${workdir}"' EXIT
+
+git clone --depth 1 --branch "v${FLATBUFFERS_VERSION}" \
+    https://github.com/google/flatbuffers.git "${workdir}/flatbuffers"
+cmake -S "${workdir}/flatbuffers" -B "${workdir}/build" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DFLATBUFFERS_BUILD_TESTS=OFF
+cmake --build "${workdir}/build" --target flatc -j "$(nproc)"
+
+mkdir -p "${FLATC_INSTALL_DIR}"
+install -m 0755 "${workdir}/build/flatc" "${FLATC_INSTALL_DIR}/flatc"
+
+# Make flatc discoverable by later steps: on PATH for find_program, and via FLATC for build_vortex.
+export PATH="${FLATC_INSTALL_DIR}:${PATH}"
+echo "${FLATC_INSTALL_DIR}" >>"${GITHUB_PATH:-/dev/null}" || true
+echo "FLATC=${FLATC_INSTALL_DIR}/flatc" >>"${GITHUB_ENV:-/dev/null}" || true
+
+"${FLATC_INSTALL_DIR}/flatc" --version

ci/scripts/setup_rust.sh

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Install the Rust toolchain used by the Lance, Mosaic, and tantivy-fts FFI builds, plus
+# Install the Rust toolchain used by the Lance, Mosaic, Vortex and tantivy-fts FFI builds, plus
 # cbindgen required by Lance and tantivy-fts.
 #
 # The dev container (see .devcontainer/) already has these preinstalled;
@@ -76,6 +76,11 @@ if ! command -v protoc >/dev/null 2>&1 || \
     echo "${protoc_home}/bin" >> "${GITHUB_PATH:-/dev/null}" || true
 fi
 
+# Vortex's FFI build pins rust-version >= 1.95 and is invoked with RUSTUP_TOOLCHAIN=stable (see
+# cmake_modules/ThirdpartyToolchain.cmake). Install the stable channel alongside the pinned default
+# so Vortex builds with it while Lance, Mosaic and tantivy-fts keep using the default toolchain.
+rustup toolchain install stable --profile minimal
+
 # cbindgen is used by the crate's build.rs to emit the C header that the
 # C++ side includes. Corrosion will also run cbindgen at CMake configure
 # time; both paths need it available.

cmake_modules/ThirdpartyToolchain.cmake

@@ -91,6 +91,18 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_VORTEX_URL})
+    set(VORTEX_SOURCE_URL "$ENV{PAIMON_VORTEX_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_VORTEX_PKG_NAME}")
+        set_urls(VORTEX_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_VORTEX_PKG_NAME}")
+    else()
+        set_urls(VORTEX_SOURCE_URL
+                 "${THIRDPARTY_MIRROR_URL}https://github.com/vortex-data/vortex/archive/${PAIMON_VORTEX_BUILD_VERSION}.tar.gz"
+        )
+    endif()
+endif()
+
 if(DEFINED ENV{PAIMON_MOSAIC_URL})
     set(MOSAIC_SOURCE_URL "$ENV{PAIMON_MOSAIC_URL}")
 else()
@@ -1403,6 +1415,103 @@ macro(build_mosaic)
     install(FILES "${MOSAIC_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 endmacro()
 
+macro(build_vortex)
+    message(STATUS "Building Vortex Rust FFI (static) from source")
+    find_program(PAIMON_CARGO_EXECUTABLE cargo REQUIRED)
+    # Vortex compiles FlatBuffers schemas at build time; flatc must match the flatbuffers crate
+    # version Vortex uses. Allow a FLATC env override, else require flatc on PATH.
+    if(DEFINED ENV{FLATC})
+        set(VORTEX_FLATC "$ENV{FLATC}")
+    else()
+        find_program(VORTEX_FLATC flatc REQUIRED)
+    endif()
+
+    set(VORTEX_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/vortex_ep-install")
+    set(VORTEX_INCLUDE_DIR "${VORTEX_PREFIX}/include")
+    set(VORTEX_LIB_DIR "${VORTEX_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+    set(VORTEX_STATIC_LIB
+        "${VORTEX_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}vortex_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    )
+    set(VORTEX_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/vortex_ep-cargo")
+    set(VORTEX_CARGO_STATIC_LIB
+        "${VORTEX_CARGO_TARGET_DIR}/release/${CMAKE_STATIC_LIBRARY_PREFIX}vortex_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    )
+    # Vortex's C API can only read a whole file already in memory or a path it resolves itself,
+    # neither of which can go through paimon's filesystem. `callback_io.rs` adds an entry point
+    # backed by host callbacks; it is maintained in this repository and injected into the Vortex
+    # source tree, and the patch only declares the module.
+    set(VORTEX_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/vortex.diff")
+    set(VORTEX_CALLBACK_IO_SRC
+        "${CMAKE_SOURCE_DIR}/crates/vortex_callback_io/callback_io.rs")
+
+    file(MAKE_DIRECTORY "${VORTEX_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${VORTEX_LIB_DIR}")
+
+    # A local checkout (PAIMON_VORTEX_SOURCE_DIR) avoids re-downloading the full cargo workspace;
+    # otherwise download the pinned source archive (with SHA256 when versions.txt provides one).
+    if(DEFINED ENV{PAIMON_VORTEX_SOURCE_DIR})
+        set(VORTEX_EP_DOWNLOAD SOURCE_DIR "$ENV{PAIMON_VORTEX_SOURCE_DIR}"
+                               DOWNLOAD_COMMAND "")
+    elseif(PAIMON_VORTEX_BUILD_SHA256_CHECKSUM)
+        set(VORTEX_EP_DOWNLOAD URL ${VORTEX_SOURCE_URL} URL_HASH
+                               "SHA256=${PAIMON_VORTEX_BUILD_SHA256_CHECKSUM}")
+    else()
+        set(VORTEX_EP_DOWNLOAD URL ${VORTEX_SOURCE_URL})
+    endif()
+
+    externalproject_add(vortex_ep
+                        ${VORTEX_EP_DOWNLOAD} ${THIRDPARTY_LOG_OPTIONS}
+                        BUILD_IN_SOURCE TRUE
+                        CONFIGURE_COMMAND ""
+                        LOG_PATCH ON
+                        PATCH_COMMAND ${CMAKE_COMMAND} -E chdir <SOURCE_DIR> bash -c
+                                      "[ -f .patched ] && echo '<SOURCE_DIR> patch already applied, ignore...' || patch -s -N -p1 -i '${VORTEX_PATCH_FILE}' && touch .patched"
+                        # Rebuild every time so edits to callback_io.rs are picked up; cargo
+                        # returns immediately when nothing changed, and copy_if_different keeps
+                        # the archive's timestamp stable so dependents do not relink.
+                        BUILD_ALWAYS TRUE
+                        BUILD_COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                                      "${VORTEX_CALLBACK_IO_SRC}"
+                                      "<SOURCE_DIR>/vortex-ffi/src/callback_io.rs"
+                        COMMAND ${CMAKE_COMMAND} -E env "FLATC=${VORTEX_FLATC}"
+                                "RUSTUP_TOOLCHAIN=stable" "CC=${CMAKE_C_COMPILER}"
+                                "CXX=${CMAKE_CXX_COMPILER}"
+                                "CARGO_TARGET_DIR=${VORTEX_CARGO_TARGET_DIR}"
+                                "RUSTFLAGS=-Crelocation-model=pic"
+                                ${PAIMON_CARGO_EXECUTABLE} rustc --locked --package
+                                vortex-ffi --lib --crate-type=staticlib --release
+                        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                                "${VORTEX_CARGO_STATIC_LIB}" "${VORTEX_STATIC_LIB}"
+                        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                                "<SOURCE_DIR>/vortex-ffi/cinclude/vortex.h"
+                                "${VORTEX_INCLUDE_DIR}/vortex.h"
+                        INSTALL_COMMAND ""
+                        BUILD_BYPRODUCTS "${VORTEX_STATIC_LIB}")
+
+    if(NOT TARGET Threads::Threads)
+        set(THREADS_PREFER_PTHREAD_FLAG TRUE)
+        find_package(Threads REQUIRED)
+    endif()
+    add_library(paimon_vortex_ffi STATIC IMPORTED GLOBAL)
+    set_target_properties(paimon_vortex_ffi
+                          PROPERTIES IMPORTED_LOCATION "${VORTEX_STATIC_LIB}"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${VORTEX_INCLUDE_DIR}")
+    # System libraries required to link the Rust static archive on Linux (per Vortex's own
+    # vortex-ffi/cmake/SystemDependencies.cmake).
+    target_link_libraries(paimon_vortex_ffi
+                          INTERFACE gcc_s
+                                    util
+                                    rt
+                                    Threads::Threads
+                                    m
+                                    ${CMAKE_DL_LIBS}
+                                    c)
+    add_dependencies(paimon_vortex_ffi vortex_ep)
+
+    install(FILES "${VORTEX_STATIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endmacro()
+
 macro(build_jindosdk_nextarch)
     message(STATUS "Building jindosdk-nextarch from local source")
 
@@ -2128,6 +2237,9 @@ resolve_dependency(glog)
 if(PAIMON_ENABLE_MOSAIC)
     build_mosaic()
 endif()
+if(PAIMON_ENABLE_VORTEX)
+    build_vortex()
+endif()
 if(PAIMON_ENABLE_AVRO)
     resolve_dependency(Avro)
 endif()

cmake_modules/vortex.diff

@@ -0,0 +1,9 @@
+diff --git a/vortex-ffi/src/lib.rs b/vortex-ffi/src/lib.rs
+--- a/vortex-ffi/src/lib.rs
++++ b/vortex-ffi/src/lib.rs
+@@ -8,4 +8,5 @@
+ mod array;
++mod callback_io;
+ mod array_iterator;
+ mod binary;
+ mod data_source;