From 09bb73ba0dc97bbfbc438b09df9c2007db1a4880 Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Sun, 20 Sep 2026 10:38:30 +0000 Subject: [PATCH 1/9] feat(format): support Vortex file format --- .github/.rat-excludes | 1 + .github/workflows/build_and_test.yaml | 3 + .github/workflows/gcc8_test.yaml | 3 + .github/workflows/release_candidate.yaml | 4 + CMakeLists.txt | 18 + ci/scripts/build_paimon.sh | 1 + ci/scripts/setup_flatc.sh | 61 ++ ci/scripts/setup_rust.sh | 7 +- cmake_modules/ThirdpartyToolchain.cmake | 112 +++ cmake_modules/vortex.diff | 9 + crates/vortex_callback_io/README.md | 83 ++ crates/vortex_callback_io/callback_io.rs | 783 ++++++++++++++++++ docs/source/user_guide/format_table.rst | 14 +- .../reader/data_file_reader_factory.cpp | 13 +- src/paimon/core/schema/schema_validation.cpp | 68 ++ src/paimon/core/schema/schema_validation.h | 4 + src/paimon/core/table/format/format_table.cpp | 2 +- .../core/table/format/format_table_test.cpp | 2 +- src/paimon/format/vortex/CMakeLists.txt | 62 ++ src/paimon/format/vortex/vortex_ffi.h | 119 +++ src/paimon/format/vortex/vortex_ffi_util.cpp | 35 + src/paimon/format/vortex/vortex_ffi_util.h | 44 + .../vortex/vortex_file_batch_reader.cpp | 468 +++++++++++ .../format/vortex/vortex_file_batch_reader.h | 133 +++ .../format/vortex/vortex_file_format.cpp | 56 ++ src/paimon/format/vortex/vortex_file_format.h | 49 ++ .../vortex/vortex_file_format_factory.cpp | 36 + .../vortex/vortex_file_format_factory.h | 41 + .../format/vortex/vortex_file_format_test.cpp | 349 ++++++++ .../format/vortex/vortex_format_writer.cpp | 155 ++++ .../format/vortex/vortex_format_writer.h | 79 ++ .../format/vortex/vortex_io_callbacks.cpp | 205 +++++ .../format/vortex/vortex_io_callbacks.h | 155 ++++ .../format/vortex/vortex_reader_builder.h | 60 ++ .../format/vortex/vortex_stats_extractor.cpp | 74 ++ .../format/vortex/vortex_stats_extractor.h | 59 ++ .../format/vortex/vortex_writer_builder.cpp | 39 + .../format/vortex/vortex_writer_builder.h | 56 ++ test/inte/scan_and_read_inte_test.cpp | 43 + .../append_java_compat/README.md | 24 + ...e1867-52a2-4264-ae8e-a2d9fe20bf1e-0.vortex | Bin 0 -> 3492 bytes ...est-ac2dfb39-417f-42a4-887a-7618b40580a5-0 | Bin 0 -> 2070 bytes ...ist-85e17711-240b-41ef-86a7-c6aec4c25d68-0 | Bin 0 -> 1006 bytes ...ist-85e17711-240b-41ef-86a7-c6aec4c25d68-1 | Bin 0 -> 1110 bytes .../append_java_compat/schema/schema-0 | 21 + .../append_java_compat/snapshot/EARLIEST | 1 + .../append_java_compat/snapshot/LATEST | 1 + .../append_java_compat/snapshot/snapshot-1 | 17 + third_party/versions.txt | 9 + 49 files changed, 3563 insertions(+), 15 deletions(-) create mode 100755 ci/scripts/setup_flatc.sh create mode 100644 cmake_modules/vortex.diff create mode 100644 crates/vortex_callback_io/README.md create mode 100644 crates/vortex_callback_io/callback_io.rs create mode 100644 src/paimon/format/vortex/CMakeLists.txt create mode 100644 src/paimon/format/vortex/vortex_ffi.h create mode 100644 src/paimon/format/vortex/vortex_ffi_util.cpp create mode 100644 src/paimon/format/vortex/vortex_ffi_util.h create mode 100644 src/paimon/format/vortex/vortex_file_batch_reader.cpp create mode 100644 src/paimon/format/vortex/vortex_file_batch_reader.h create mode 100644 src/paimon/format/vortex/vortex_file_format.cpp create mode 100644 src/paimon/format/vortex/vortex_file_format.h create mode 100644 src/paimon/format/vortex/vortex_file_format_factory.cpp create mode 100644 src/paimon/format/vortex/vortex_file_format_factory.h create mode 100644 src/paimon/format/vortex/vortex_file_format_test.cpp create mode 100644 src/paimon/format/vortex/vortex_format_writer.cpp create mode 100644 src/paimon/format/vortex/vortex_format_writer.h create mode 100644 src/paimon/format/vortex/vortex_io_callbacks.cpp create mode 100644 src/paimon/format/vortex/vortex_io_callbacks.h create mode 100644 src/paimon/format/vortex/vortex_reader_builder.h create mode 100644 src/paimon/format/vortex/vortex_stats_extractor.cpp create mode 100644 src/paimon/format/vortex/vortex_stats_extractor.h create mode 100644 src/paimon/format/vortex/vortex_writer_builder.cpp create mode 100644 src/paimon/format/vortex/vortex_writer_builder.h create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/README.md create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/bucket-0/data-430e1867-52a2-4264-ae8e-a2d9fe20bf1e-0.vortex create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-ac2dfb39-417f-42a4-887a-7618b40580a5-0 create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-0 create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-1 create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/schema/schema-0 create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/EARLIEST create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/LATEST create mode 100644 test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/snapshot-1 diff --git a/.github/.rat-excludes b/.github/.rat-excludes index c410eb9c..c1c5ddef 100644 --- a/.github/.rat-excludes +++ b/.github/.rat-excludes @@ -12,6 +12,7 @@ iwyu_tool\.py arrow\.diff jieba\.diff orc\.diff +vortex\.diff licenses/* .codespell_ignore .gitignore diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index cd9174c0..2db6ef04 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -114,6 +114,9 @@ jobs: run: | sudo apt-get update sudo apt-get install -y gdb libcurl4-openssl-dev libssl-dev + - name: Install flatc (Vortex) + shell: bash + run: ci/scripts/setup_flatc.sh - name: Build Paimon shell: bash env: diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index 46b0574e..6b3c99e2 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -67,6 +67,9 @@ jobs: uses: ./.github/actions/setup-ccache with: cache-key-prefix: ccache-gcc8-test + - name: Install flatc (Vortex) + shell: bash + run: ci/scripts/setup_flatc.sh - name: Build Paimon shell: bash env: diff --git a/.github/workflows/release_candidate.yaml b/.github/workflows/release_candidate.yaml index 402fbd88..ce6b74c5 100644 --- a/.github/workflows/release_candidate.yaml +++ b/.github/workflows/release_candidate.yaml @@ -135,6 +135,10 @@ jobs: shell: bash run: ci/scripts/setup_rust.sh + - name: Install flatc (Vortex) + shell: bash + run: ci/scripts/setup_flatc.sh + - name: Install HTTP and TLS development dependencies run: | sudo apt-get update diff --git a/CMakeLists.txt b/CMakeLists.txt index 85fc0d8c..4b36a54f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,7 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON) option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON) option(PAIMON_ENABLE_MOSAIC "Whether to enable mosaic file format (Rust FFI)" OFF) option(PAIMON_ENABLE_LANCE "Whether to enable lance file format (Rust FFI)" OFF) +option(PAIMON_ENABLE_VORTEX "Whether to enable vortex file format (Rust FFI)" OFF) option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF) option(PAIMON_ENABLE_OSS "Whether to enable OSS SDK v2 file system" OFF) option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF) @@ -97,6 +98,9 @@ endif() if(PAIMON_ENABLE_LANCE) add_definitions(-DPAIMON_ENABLE_LANCE) endif() +if(PAIMON_ENABLE_VORTEX) + add_definitions(-DPAIMON_ENABLE_VORTEX) +endif() if(PAIMON_ENABLE_JINDO) add_definitions(-DPAIMON_ENABLE_JINDO) endif() @@ -419,6 +423,10 @@ if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS) paimon_link_libraries_whole_archive(PAIMON_LANCE_FILE_FORMAT_STATIC_LINK_LIBS paimon_lance_file_format_static) endif() + if(PAIMON_ENABLE_VORTEX) + paimon_link_libraries_whole_archive(PAIMON_VORTEX_FILE_FORMAT_STATIC_LINK_LIBS + paimon_vortex_file_format_static) + endif() if(PAIMON_ENABLE_ORC) paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS @@ -486,6 +494,13 @@ if(PAIMON_BUILD_TESTS) paimon_link_libraries_whole_archive(PAIMON_LANCE_FILE_FORMAT_STATIC_LINK_LIBS paimon_lance_file_format_static) endif() + if(PAIMON_ENABLE_VORTEX) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_vortex_file_format_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) + paimon_link_libraries_whole_archive(PAIMON_VORTEX_FILE_FORMAT_STATIC_LINK_LIBS + paimon_vortex_file_format_static) + endif() if(PAIMON_ENABLE_ORC) paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS @@ -582,6 +597,9 @@ endif() if(PAIMON_ENABLE_LANCE) add_subdirectory(src/paimon/format/lance) endif() +if(PAIMON_ENABLE_VORTEX) + add_subdirectory(src/paimon/format/vortex) +endif() if(PAIMON_ENABLE_LUMINA) add_subdirectory(src/paimon/global_index/lumina) endif() diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index b012e7f4..b91a3336 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -151,6 +151,7 @@ CMAKE_ARGS=( "-DPAIMON_BUILD_TESTS=ON" "-DPAIMON_ENABLE_MOSAIC=ON" "-DPAIMON_ENABLE_LANCE=${ENABLE_LANCE}" + "-DPAIMON_ENABLE_VORTEX=ON" "-DPAIMON_ENABLE_JINDO=ON" "-DPAIMON_ENABLE_OSS=ON" "-DPAIMON_ENABLE_S3=ON" diff --git a/ci/scripts/setup_flatc.sh b/ci/scripts/setup_flatc.sh new file mode 100755 index 00000000..f853fa6c --- /dev/null +++ b/ci/scripts/setup_flatc.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Build and install flatc (the FlatBuffers compiler) at the version Vortex requires. +# +# Vortex compiles its FlatBuffers schemas (.fbs) into Rust at build time, and +# cmake_modules/ThirdpartyToolchain.cmake requires a flatc whose version matches Vortex's +# flatbuffers crate (find_program(... flatc REQUIRED)). Distribution packages are too old, so the +# pinned version is built from source. flatc is a build-time tool only; it is not needed at runtime. +# +# The dev container (see .devcontainer/) already has flatc preinstalled; this script is for the +# GitHub Actions runners and is called before ci/scripts/build_paimon.sh. +# +# Idempotent: a no-op when a matching flatc is already on PATH. + +set -eux + +# Must match the flatbuffers crate version pinned by Vortex (see its Cargo.lock). +FLATBUFFERS_VERSION=${FLATBUFFERS_VERSION:-25.12.19} +FLATC_INSTALL_DIR=${FLATC_INSTALL_DIR:-"${HOME}/.local/bin"} + +# Skip when a matching flatc is already available. +if command -v flatc >/dev/null 2>&1 && + [[ "$(flatc --version 2>/dev/null | awk '{print $NF}')" == "${FLATBUFFERS_VERSION}" ]]; then + flatc --version + exit 0 +fi + +workdir="$(mktemp -d)" +trap 'rm -rf "${workdir}"' EXIT + +git clone --depth 1 --branch "v${FLATBUFFERS_VERSION}" \ + https://github.com/google/flatbuffers.git "${workdir}/flatbuffers" +cmake -S "${workdir}/flatbuffers" -B "${workdir}/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DFLATBUFFERS_BUILD_TESTS=OFF +cmake --build "${workdir}/build" --target flatc -j "$(nproc)" + +mkdir -p "${FLATC_INSTALL_DIR}" +install -m 0755 "${workdir}/build/flatc" "${FLATC_INSTALL_DIR}/flatc" + +# Make flatc discoverable by later steps: on PATH for find_program, and via FLATC for build_vortex. +export PATH="${FLATC_INSTALL_DIR}:${PATH}" +echo "${FLATC_INSTALL_DIR}" >>"${GITHUB_PATH:-/dev/null}" || true +echo "FLATC=${FLATC_INSTALL_DIR}/flatc" >>"${GITHUB_ENV:-/dev/null}" || true + +"${FLATC_INSTALL_DIR}/flatc" --version diff --git a/ci/scripts/setup_rust.sh b/ci/scripts/setup_rust.sh index 5576d523..613c648c 100755 --- a/ci/scripts/setup_rust.sh +++ b/ci/scripts/setup_rust.sh @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Install the Rust toolchain used by the Lance, Mosaic, and tantivy-fts FFI builds, plus +# Install the Rust toolchain used by the Lance, Mosaic, Vortex and tantivy-fts FFI builds, plus # cbindgen required by Lance and tantivy-fts. # # The dev container (see .devcontainer/) already has these preinstalled; @@ -76,6 +76,11 @@ if ! command -v protoc >/dev/null 2>&1 || \ echo "${protoc_home}/bin" >> "${GITHUB_PATH:-/dev/null}" || true fi +# Vortex's FFI build pins rust-version >= 1.95 and is invoked with RUSTUP_TOOLCHAIN=stable (see +# cmake_modules/ThirdpartyToolchain.cmake). Install the stable channel alongside the pinned default +# so Vortex builds with it while Lance, Mosaic and tantivy-fts keep using the default toolchain. +rustup toolchain install stable --profile minimal + # cbindgen is used by the crate's build.rs to emit the C header that the # C++ side includes. Corrosion will also run cbindgen at CMake configure # time; both paths need it available. diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 0c64819c..032baf1e 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -91,6 +91,18 @@ else() endif() endif() +if(DEFINED ENV{PAIMON_VORTEX_URL}) + set(VORTEX_SOURCE_URL "$ENV{PAIMON_VORTEX_URL}") +else() + if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_VORTEX_PKG_NAME}") + set_urls(VORTEX_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_VORTEX_PKG_NAME}") + else() + set_urls(VORTEX_SOURCE_URL + "${THIRDPARTY_MIRROR_URL}https://github.com/vortex-data/vortex/archive/${PAIMON_VORTEX_BUILD_VERSION}.tar.gz" + ) + endif() +endif() + if(DEFINED ENV{PAIMON_MOSAIC_URL}) set(MOSAIC_SOURCE_URL "$ENV{PAIMON_MOSAIC_URL}") else() @@ -1403,6 +1415,103 @@ macro(build_mosaic) install(FILES "${MOSAIC_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR}) endmacro() +macro(build_vortex) + message(STATUS "Building Vortex Rust FFI (static) from source") + find_program(PAIMON_CARGO_EXECUTABLE cargo REQUIRED) + # Vortex compiles FlatBuffers schemas at build time; flatc must match the flatbuffers crate + # version Vortex uses. Allow a FLATC env override, else require flatc on PATH. + if(DEFINED ENV{FLATC}) + set(VORTEX_FLATC "$ENV{FLATC}") + else() + find_program(VORTEX_FLATC flatc REQUIRED) + endif() + + set(VORTEX_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/vortex_ep-install") + set(VORTEX_INCLUDE_DIR "${VORTEX_PREFIX}/include") + set(VORTEX_LIB_DIR "${VORTEX_PREFIX}/${CMAKE_INSTALL_LIBDIR}") + set(VORTEX_STATIC_LIB + "${VORTEX_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}vortex_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(VORTEX_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/vortex_ep-cargo") + set(VORTEX_CARGO_STATIC_LIB + "${VORTEX_CARGO_TARGET_DIR}/release/${CMAKE_STATIC_LIBRARY_PREFIX}vortex_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + # Vortex's C API can only read a whole file already in memory or a path it resolves itself, + # neither of which can go through paimon's filesystem. `callback_io.rs` adds an entry point + # backed by host callbacks; it is maintained in this repository and injected into the Vortex + # source tree, and the patch only declares the module. + set(VORTEX_PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/vortex.diff") + set(VORTEX_CALLBACK_IO_SRC + "${CMAKE_SOURCE_DIR}/crates/vortex_callback_io/callback_io.rs") + + file(MAKE_DIRECTORY "${VORTEX_INCLUDE_DIR}") + file(MAKE_DIRECTORY "${VORTEX_LIB_DIR}") + + # A local checkout (PAIMON_VORTEX_SOURCE_DIR) avoids re-downloading the full cargo workspace; + # otherwise download the pinned source archive (with SHA256 when versions.txt provides one). + if(DEFINED ENV{PAIMON_VORTEX_SOURCE_DIR}) + set(VORTEX_EP_DOWNLOAD SOURCE_DIR "$ENV{PAIMON_VORTEX_SOURCE_DIR}" + DOWNLOAD_COMMAND "") + elseif(PAIMON_VORTEX_BUILD_SHA256_CHECKSUM) + set(VORTEX_EP_DOWNLOAD URL ${VORTEX_SOURCE_URL} URL_HASH + "SHA256=${PAIMON_VORTEX_BUILD_SHA256_CHECKSUM}") + else() + set(VORTEX_EP_DOWNLOAD URL ${VORTEX_SOURCE_URL}) + endif() + + externalproject_add(vortex_ep + ${VORTEX_EP_DOWNLOAD} ${THIRDPARTY_LOG_OPTIONS} + BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + LOG_PATCH ON + PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c + "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${VORTEX_PATCH_FILE}' && touch .patched" + # Rebuild every time so edits to callback_io.rs are picked up; cargo + # returns immediately when nothing changed, and copy_if_different keeps + # the archive's timestamp stable so dependents do not relink. + BUILD_ALWAYS TRUE + BUILD_COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${VORTEX_CALLBACK_IO_SRC}" + "/vortex-ffi/src/callback_io.rs" + COMMAND ${CMAKE_COMMAND} -E env "FLATC=${VORTEX_FLATC}" + "RUSTUP_TOOLCHAIN=stable" "CC=${CMAKE_C_COMPILER}" + "CXX=${CMAKE_CXX_COMPILER}" + "CARGO_TARGET_DIR=${VORTEX_CARGO_TARGET_DIR}" + "RUSTFLAGS=-Crelocation-model=pic" + ${PAIMON_CARGO_EXECUTABLE} rustc --locked --package + vortex-ffi --lib --crate-type=staticlib --release + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${VORTEX_CARGO_STATIC_LIB}" "${VORTEX_STATIC_LIB}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "/vortex-ffi/cinclude/vortex.h" + "${VORTEX_INCLUDE_DIR}/vortex.h" + INSTALL_COMMAND "" + BUILD_BYPRODUCTS "${VORTEX_STATIC_LIB}") + + if(NOT TARGET Threads::Threads) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) + find_package(Threads REQUIRED) + endif() + add_library(paimon_vortex_ffi STATIC IMPORTED GLOBAL) + set_target_properties(paimon_vortex_ffi + PROPERTIES IMPORTED_LOCATION "${VORTEX_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${VORTEX_INCLUDE_DIR}") + # System libraries required to link the Rust static archive on Linux (per Vortex's own + # vortex-ffi/cmake/SystemDependencies.cmake). + target_link_libraries(paimon_vortex_ffi + INTERFACE gcc_s + util + rt + Threads::Threads + m + ${CMAKE_DL_LIBS} + c) + add_dependencies(paimon_vortex_ffi vortex_ep) + + install(FILES "${VORTEX_STATIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endmacro() + macro(build_jindosdk_nextarch) message(STATUS "Building jindosdk-nextarch from local source") @@ -2128,6 +2237,9 @@ resolve_dependency(glog) if(PAIMON_ENABLE_MOSAIC) build_mosaic() endif() +if(PAIMON_ENABLE_VORTEX) + build_vortex() +endif() if(PAIMON_ENABLE_AVRO) resolve_dependency(Avro) endif() diff --git a/cmake_modules/vortex.diff b/cmake_modules/vortex.diff new file mode 100644 index 00000000..bbd277d3 --- /dev/null +++ b/cmake_modules/vortex.diff @@ -0,0 +1,9 @@ +diff --git a/vortex-ffi/src/lib.rs b/vortex-ffi/src/lib.rs +--- a/vortex-ffi/src/lib.rs ++++ b/vortex-ffi/src/lib.rs +@@ -11,4 +11,5 @@ + mod array; ++mod callback_io; + mod data_source; + mod dtype; + mod error; diff --git a/crates/vortex_callback_io/README.md b/crates/vortex_callback_io/README.md new file mode 100644 index 00000000..2e92064b --- /dev/null +++ b/crates/vortex_callback_io/README.md @@ -0,0 +1,83 @@ + + +# vortex_callback_io + +Callback-based I/O for the Vortex C API, so Vortex reads and writes through paimon's +`InputStream` / `OutputStream` instead of materializing whole files in memory or staging them in +local temporary files. + +## This is not a standalone crate + +There is deliberately no `Cargo.toml` here. `callback_io.rs` is compiled **as a module of the +upstream `vortex-ffi` crate**, because the handles it must construct (`vx_data_source` and friends) +are created through `box_wrapper`-generated constructors that are `pub(crate)`; an external crate +cannot build them. Being inside `vortex-ffi` also means all symbols land in the single +`libvortex_ffi.a` that paimon already links. + +The write side still defines its own handle (`vx_callback_sink`) instead of reusing `vx_array_sink`, +whose fields are private to `crate::sink` and therefore unreachable even from a sibling module. + +The build wiring lives in `build_vortex` in +[`cmake_modules/ThirdpartyToolchain.cmake`](../../cmake_modules/ThirdpartyToolchain.cmake): + +1. `callback_io.rs` is copied into `/vortex-ffi/src/` before cargo runs. +2. [`cmake_modules/vortex.diff`](../../cmake_modules/vortex.diff) adds the single line + `mod callback_io;` to `vortex-ffi/src/lib.rs`. That one line is the entire patch, which keeps + Vortex version bumps from conflicting. + +Keeping the code here rather than inside the diff means it is reviewed, formatted, and +license-checked like the rest of the repository. + +## Working on it + +The module is built and tested through cargo in the Vortex source tree. With an existing configured +build directory (`build-clang` below), the source directory and cargo cache are already set up: + +```bash +# Rebuild the static library through the normal CMake path. +cmake --build build-clang --target vortex_ep + +# Run this module's Rust tests directly. +cd "$(sed -n 's/^source_dir=//p' \ + build-clang/vortex_ep-prefix/src/vortex_ep-stamp/vortex_ep-source_dirinfo.txt)" +FLATC= CARGO_TARGET_DIR=/build-clang/vortex_ep-cargo \ + cargo test --locked --release -p vortex-ffi callback_io +``` + +Two constraints apply when editing this file: + +- **No new cargo dependencies.** The build runs `cargo rustc --locked`, so touching `Cargo.lock` + breaks it. Only what `vortex-ffi` already depends on is available. +- **Upstream lints apply.** `vortex-ffi` sets `#![deny(missing_docs)]` and the Vortex workspace + denies several rustc lints (`unsafe_op_in_unsafe_fn`, `unused_qualifications`, + `let_underscore_drop`, ...), so every public item needs documentation and every `unsafe` + operation needs its own `unsafe` block. + +## Threading contract + +`vx_input_callbacks::read_at_fn` must be safe to call concurrently for the same context: Vortex +issues parallel positional reads, each on a blocking-pool thread. paimon's positional +`InputStream::Read(buffer, size, offset)` satisfies this (the local implementation uses `pread`), +which is the same assumption the mosaic format's callbacks already rely on. + +`vx_output_callbacks::write_fn` has the opposite contract: writes for one sink are sequential and +never concurrent, so no locking is required. They do run on a Vortex runtime thread rather than the +caller's, which is why the paimon-side context guards its error slot and byte counter with a mutex. +The host keeps ownership of its stream: `shutdown` only flushes, and closing is left to the caller. diff --git a/crates/vortex_callback_io/callback_io.rs b/crates/vortex_callback_io/callback_io.rs new file mode 100644 index 00000000..099b66e7 --- /dev/null +++ b/crates/vortex_callback_io/callback_io.rs @@ -0,0 +1,783 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Callback-based I/O for the Vortex C API. +//! +//! The stock C API can only read a whole file already materialized in memory +//! (`vx_data_source_new_buffer`) or a path that Vortex itself resolves +//! (`vx_data_source_new`), and it can only write to a local filesystem path +//! (`vx_array_sink_open_file`, which uses `async_fs::File::create`). paimon owns its own pluggable +//! filesystem, so none of these fit: reading forces the entire file into memory, and writing forces +//! a local temporary file that then has to be copied back. +//! +//! This module adds entry points that take host callbacks instead, so Vortex issues positional +//! reads and sequential writes straight against paimon's `InputStream` / `OutputStream`. Vortex's +//! core I/O traits are the plug points; the shape of the bridge mirrors `vortex-jni`'s +//! `JavaReadable` and `JavaWrite`. +//! +//! The write side needs its own handle type rather than reusing `vx_array_sink`, whose fields are +//! private to `crate::sink`. +//! +//! This file is maintained in the paimon-cpp tree (`crates/vortex_callback_io/`) and copied into +//! `vortex-ffi/src/` at build time; see `cmake_modules/vortex.diff`. +//! +//! The C declarations that mirror the types and entry points below live in paimon-cpp's +//! `src/paimon/format/vortex/vortex_ffi.h` (cbindgen does not see this module, so they are written +//! by hand). The two sides are only checked by human review, and a mismatch in field order or a +//! function signature is silent memory corruption, not a compile error. Any change to a +//! `#[repr(C)]` struct, a callback `type`, or a `#[no_mangle]` entry point here MUST be mirrored in +//! that header in the same change. + +use std::ffi::c_void; +use std::io; +use std::sync::Arc; + +use futures::FutureExt; +use futures::SinkExt; +use futures::TryStreamExt; +use futures::channel::mpsc; +use futures::channel::mpsc::Sender; +use futures::future::BoxFuture; +use vortex::array::ArrayRef; +use vortex::array::buffer::BufferHandle; +use vortex::array::stream::ArrayStreamAdapter; +use vortex::buffer::Alignment; +use vortex::buffer::ByteBufferMut; +use vortex::dtype::DType; +use vortex::error::VortexResult; +use vortex::error::vortex_bail; +use vortex::error::vortex_ensure; +use vortex::error::vortex_err; +use vortex::file::OpenOptionsSessionExt; +use vortex::file::WriteOptionsSessionExt; +use vortex::file::WriteStrategyBuilder; +use vortex::file::WriteSummary; +use vortex::io::CoalesceConfig; +use vortex::io::IoBuf; +use vortex::io::VortexReadAt; +use vortex::io::VortexWrite; +use vortex::io::runtime::BlockingRuntime; +use vortex::io::runtime::Handle; +use vortex::io::runtime::Task; +use vortex::io::session::RuntimeSessionExt; +use vortex::layout::scan::multi::MultiLayoutDataSource; + +use crate::RUNTIME; +use crate::array::vx_array; +use crate::data_source::vx_data_source; +use crate::dtype::vx_dtype; +use crate::error::try_or; +use crate::error::try_or_default; +use crate::error::vx_error; +use crate::session::vx_session; + +/// Fill `length` bytes starting at `offset` into `dst`. +/// +/// Returns 0 on success and non-zero on failure; a partial read must be reported as a failure. +/// The host is expected to keep its own error detail on the context, since only a status code +/// crosses the boundary. +pub type vx_read_at_fn = + unsafe extern "C" fn(ctx: *mut c_void, offset: u64, dst: *mut u8, length: usize) -> i32; + +/// Release the host context. Called exactly once, when the owning handle is freed. +pub type vx_release_fn = unsafe extern "C" fn(ctx: *mut c_void); + +/// Append `length` bytes from `src` to the host sink. +/// +/// Returns 0 on success and non-zero on failure; a partial write must be reported as a failure. +/// Unlike reads, writes are sequential and never concurrent for one sink. +pub type vx_write_fn = unsafe extern "C" fn(ctx: *mut c_void, src: *const u8, length: usize) -> i32; + +/// Flush whatever the host has buffered. Returns 0 on success and non-zero on failure. +pub type vx_flush_fn = unsafe extern "C" fn(ctx: *mut c_void) -> i32; + +/// Host callbacks backing a positional reader. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct vx_input_callbacks { + /// Opaque host context, passed back to every callback. + pub ctx: *mut c_void, + /// Positional read. Required. + /// + /// Must be thread-safe: Vortex issues concurrent positional reads, so this is called from + /// several blocking threads at once for the same context. + pub read_at_fn: Option, + /// Context destructor. Optional; when set, it is the last callback invoked. + pub release_fn: Option, +} + +/// Host callbacks backing a sequential writer. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct vx_output_callbacks { + /// Opaque host context, passed back to every callback. + pub ctx: *mut c_void, + /// Sequential write. Required. + pub write_fn: Option, + /// Flush. Optional; when null, flush requests are ignored. + pub flush_fn: Option, + /// Context destructor. Optional; when set, it is the last callback invoked. + pub release_fn: Option, +} + +/// The host context, owned by the reader. +/// +/// Held behind an `Arc` so in-flight reads keep the context alive, and so `release_fn` runs exactly +/// once when the last reference goes away. It carries only the context, not the read callback, so +/// that ownership can be taken over before anything else is validated. +struct CallbackTarget { + ctx: *mut c_void, + release_fn: Option, +} + +// SAFETY: the host contract for `vx_input_callbacks` requires `read_at_fn` to be thread-safe for +// concurrent calls on the same `ctx`, which is what makes it sound to move the context across +// threads and share it between concurrent reads. +unsafe impl Send for CallbackTarget {} +// SAFETY: see the `Send` impl above. +unsafe impl Sync for CallbackTarget {} + +impl Drop for CallbackTarget { + fn drop(&mut self) { + if let Some(release) = self.release_fn { + // SAFETY: `ctx` was supplied by the host together with `release_fn`, and an `Arc` + // guarantees this runs once, after every in-flight read has finished. + unsafe { release(self.ctx) }; + } + } +} + +/// A [`VortexReadAt`] that forwards positional reads to host callbacks. +/// +/// Reads run on the runtime's blocking pool, since the host callback is synchronous and must not +/// occupy an async executor thread. The file size is supplied at construction time, so `size()` +/// never crosses the FFI boundary. +struct CallbackReadAt { + target: Arc, + read_at_fn: vx_read_at_fn, + len: u64, + handle: Handle, + concurrency: usize, +} + +/// Number of concurrent host read callbacks Vortex may have in flight for one file. +/// +/// The host filesystem may be remote, where concurrency hides latency; it is also the bound on +/// blocking threads occupied by one reader. +const DEFAULT_CONCURRENCY: usize = 16; + +impl VortexReadAt for CallbackReadAt { + fn coalesce_config(&self) -> Option { + // Each read costs an FFI hop plus a blocking-pool hand-off, and the host filesystem may be + // remote, so favor fewer and larger reads. + Some(CoalesceConfig::object_storage()) + } + + fn concurrency(&self) -> usize { + self.concurrency + } + + fn size(&self) -> BoxFuture<'static, VortexResult> { + let len = self.len; + async move { Ok(len) }.boxed() + } + + fn read_at( + &self, + offset: u64, + length: usize, + alignment: Alignment, + ) -> BoxFuture<'static, VortexResult> { + let target = Arc::clone(&self.target); + let read_at_fn = self.read_at_fn; + let len = self.len; + let handle = self.handle.clone(); + + async move { + handle + .spawn_blocking(move || { + let end = offset + .checked_add(length as u64) + .ok_or_else(|| vortex_err!("read {offset}+{length} overflows u64"))?; + if end > len { + vortex_bail!("read {offset}..{end} out of bounds for file of length {len}"); + } + + let mut buffer = ByteBufferMut::with_capacity_aligned(length, alignment); + if length > 0 { + // SAFETY: the spare capacity covers `length` bytes of live allocation, and + // the host contract requires the callback to fill exactly that many bytes + // before returning success. + let status = unsafe { + read_at_fn( + target.ctx, + offset, + buffer.spare_capacity_mut().as_mut_ptr().cast::(), + length, + ) + }; + if status != 0 { + vortex_bail!( + "host read callback failed with status {status} for {offset}..{end}" + ); + } + } + // SAFETY: the callback reported success, which per its contract means all + // `length` bytes were written. + unsafe { buffer.set_len(length) }; + Ok(BufferHandle::new_host(buffer.freeze())) + }) + .await + } + .boxed() + } +} + +unsafe fn data_source_new_callback( + session: *const vx_session, + callbacks: vx_input_callbacks, + size: u64, +) -> VortexResult<*const vx_data_source> { + // Take ownership of the host context before anything else, so an early error below still + // releases it exactly once when this `Arc` is dropped. + let target = Arc::new(CallbackTarget { + ctx: callbacks.ctx, + release_fn: callbacks.release_fn, + }); + + vortex_ensure!(!session.is_null()); + let read_at_fn = callbacks + .read_at_fn + .ok_or_else(|| vortex_err!("vx_input_callbacks.read_at_fn is required"))?; + + let session = vx_session::as_ref(session).clone(); + let reader: Arc = Arc::new(CallbackReadAt { + target, + read_at_fn, + len: size, + handle: session.handle(), + concurrency: DEFAULT_CONCURRENCY, + }); + + let file = RUNTIME.block_on(async { session.open_options().open(reader).await })?; + let data_source = MultiLayoutDataSource::new_with_first( + file.layout_reader()?, + Vec::new(), + vec![Some(size)], + &session, + ); + Ok(vx_data_source::new(data_source)) +} + +/// Create a data source that reads through host callbacks. +/// +/// `size` is the total length of the file in bytes; the host knows it up front, so it is passed +/// here instead of being fetched through another callback. +/// +/// The callbacks (and the context they carry) are owned by the returned data source and released +/// when it is freed, including when this call fails. +/// +/// On error, returns NULL and sets "err". +/// +/// # Safety +/// +/// `session` must be a valid `vx_session`. `callbacks.read_at_fn` must be non-null, thread-safe, +/// and valid for as long as the data source lives. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_data_source_new_callback( + session: *const vx_session, + callbacks: vx_input_callbacks, + size: u64, + err: *mut *mut vx_error, +) -> *const vx_data_source { + try_or(err, std::ptr::null(), || unsafe { + data_source_new_callback(session, callbacks, size) + }) +} + +/// A [`VortexWrite`] that forwards sequential writes to host callbacks. +/// +/// Writes run inline on the runtime thread driving the write task, the same way `vortex-jni` +/// forwards to Java sinks: the host callback is synchronous and the layout writer hands over one +/// buffer at a time, so there is nothing to overlap. +struct CallbackWrite { + target: Arc, + write_fn: vx_write_fn, + flush_fn: Option, +} + +impl CallbackWrite { + fn write_slice(&self, bytes: &[u8]) -> io::Result<()> { + if bytes.is_empty() { + return Ok(()); + } + // SAFETY: `bytes` points at a live slice for the duration of the call, and the host + // contract forbids retaining it afterwards. + let status = unsafe { (self.write_fn)(self.target.ctx, bytes.as_ptr(), bytes.len()) }; + if status != 0 { + return Err(io::Error::other(format!( + "host write callback failed with status {status}" + ))); + } + Ok(()) + } + + fn flush_host(&self) -> io::Result<()> { + let Some(flush) = self.flush_fn else { + return Ok(()); + }; + // SAFETY: the context is kept alive by `target`. + let status = unsafe { flush(self.target.ctx) }; + if status != 0 { + return Err(io::Error::other(format!( + "host flush callback failed with status {status}" + ))); + } + Ok(()) + } +} + +impl VortexWrite for CallbackWrite { + async fn write_all(&mut self, buffer: B) -> io::Result { + self.write_slice(buffer.as_slice())?; + Ok(buffer) + } + + async fn flush(&mut self) -> io::Result<()> { + self.flush_host() + } + + async fn shutdown(&mut self) -> io::Result<()> { + // The host owns its stream and closes it itself, so this only flushes. + self.flush_host() + } +} + +/// A sink that writes a Vortex file through host callbacks. +/// +/// Mirrors `vx_array_sink`: pushed arrays go through a channel feeding a write task on the session's +/// runtime, and errors surface when the sink is closed. This is a separate type because +/// `vx_array_sink`'s fields are private to its own module. +pub struct vx_callback_sink { + sink: Sender>, + writer: Task>, + dtype: DType, +} + +unsafe fn callback_sink_open( + session: *const vx_session, + callbacks: vx_output_callbacks, + dtype: *const vx_dtype, +) -> VortexResult<*mut vx_callback_sink> { + // Take ownership of the host context before anything else, so an early error below still + // releases it exactly once when this `Arc` is dropped. + let target = Arc::new(CallbackTarget { + ctx: callbacks.ctx, + release_fn: callbacks.release_fn, + }); + + vortex_ensure!(!session.is_null()); + vortex_ensure!(!dtype.is_null()); + let write_fn = callbacks + .write_fn + .ok_or_else(|| vortex_err!("vx_output_callbacks.write_fn is required"))?; + + let session = vx_session::as_ref(session).clone(); + let dtype = vx_dtype::as_ref(dtype).clone(); + let write = CallbackWrite { + target, + write_fn, + flush_fn: callbacks.flush_fn, + }; + + // The channel size matches the stock file sink. + let (sink, rx) = mpsc::channel(32); + let array_stream = ArrayStreamAdapter::new(dtype.clone(), rx.into_stream()); + + let strategy = WriteStrategyBuilder::default().build(); + let writer_session = session.clone(); + let writer = session.handle().spawn(async move { + writer_session + .write_options() + .with_strategy(strategy) + .write(write, array_stream) + .await + }); + + Ok(Box::into_raw(Box::new(vx_callback_sink { + sink, + writer, + dtype, + }))) +} + +/// Open a sink that writes a Vortex file through `callbacks`. +/// +/// The callbacks (and the context they carry) are owned by the returned sink and released when it is +/// closed or aborted, including when this call fails. Write errors are reported when the sink is +/// closed, since the bytes are produced by a background task. +/// +/// On error, returns NULL and sets "err". +/// +/// # Safety +/// +/// `session` and `dtype` must be valid. `callbacks.write_fn` must be non-null and valid for as long +/// as the sink lives. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_callback_sink_open( + session: *const vx_session, + callbacks: vx_output_callbacks, + dtype: *const vx_dtype, + err: *mut *mut vx_error, +) -> *mut vx_callback_sink { + try_or(err, std::ptr::null_mut(), || unsafe { + callback_sink_open(session, callbacks, dtype) + }) +} + +/// Push an array into a callback sink. Does not take ownership of `array`. +/// +/// Errors if the array's DType does not match the sink's. +/// +/// # Safety +/// +/// `sink` must come from `vx_callback_sink_open` and must not have been closed or aborted. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_callback_sink_push( + sink: *mut vx_callback_sink, + array: *const vx_array, + error_out: *mut *mut vx_error, +) { + try_or_default(error_out, || { + vortex_ensure!(!sink.is_null()); + vortex_ensure!(!array.is_null()); + + let array = vx_array::as_ref(array); + let sink = unsafe { &mut *sink }; + + vortex_ensure!( + *array.dtype() == sink.dtype, + "array dtype {} does not match sink dtype {}", + array.dtype(), + sink.dtype + ); + RUNTIME + .block_on(sink.sink.send(Ok(array.clone()))) + .map_err(|e| vortex_err!("Send error: {e}")) + }) +} + +/// Close a callback sink, flushing everything pushed so far and writing the file footer. +/// +/// Consumes `sink` even when it reports an error; do not use it afterwards. +/// +/// # Safety +/// +/// `sink` must come from `vx_callback_sink_open` and must not have been closed or aborted. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_callback_sink_close( + sink: *mut vx_callback_sink, + error_out: *mut *mut vx_error, +) { + try_or_default(error_out, || { + vortex_ensure!(!sink.is_null()); + let vx_callback_sink { + sink, + writer, + dtype: _, + } = *unsafe { Box::from_raw(sink) }; + // Dropping the sender signals end of input to the write task. + drop(sink); + + RUNTIME.block_on(async { + let _summary = writer.await?; + VortexResult::Ok(()) + })?; + + Ok(()) + }) +} + +/// Abort a callback sink. No footer is written, so whatever the host received is not a valid Vortex +/// file. Do not use `sink` afterwards. +/// +/// # Safety +/// +/// `sink` must come from `vx_callback_sink_open` and must not have been closed or aborted. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_callback_sink_abort(sink: *mut vx_callback_sink) { + if sink.is_null() { + return; + } + drop(unsafe { Box::from_raw(sink) }); +} + +#[cfg(test)] +#[cfg(unix)] +mod tests { + use std::fs::File; + use std::os::unix::fs::FileExt; + use std::ptr; + use std::sync::Mutex; + use std::sync::atomic::AtomicBool; + use std::sync::atomic::Ordering; + + use tempfile::NamedTempFile; + use vortex::array::IntoArray; + use vortex::array::arrays::PrimitiveArray; + use vortex::array::validity::Validity; + use vortex::buffer::buffer; + use vortex::dtype::DType; + use vortex::dtype::PType; + + use super::*; + use crate::array::vx_array; + use crate::array::vx_array_free; + use crate::data_source::vx_data_source_free; + use crate::data_source::vx_data_source_get_row_count; + use crate::dtype::vx_dtype; + use crate::dtype::vx_dtype_free; + use crate::error::vx_error_free; + use crate::scan::vx_estimate; + use crate::scan::vx_estimate_type; + use crate::session::vx_session_free; + use crate::session::vx_session_new; + use crate::sink::vx_array_sink_close; + use crate::sink::vx_array_sink_open_file; + use crate::sink::vx_array_sink_push; + use crate::string::vx_view; + + /// Host context standing in for paimon's `InputStream`: positional reads against a file. + struct FileContext { + file: File, + } + + unsafe extern "C" fn file_read_at( + ctx: *mut c_void, + offset: u64, + dst: *mut u8, + length: usize, + ) -> i32 { + // SAFETY: `ctx` is the `FileContext` handed to `vx_data_source_new_callback`, and `dst` + // covers `length` bytes, per the callback contract. + let context = unsafe { &*ctx.cast::() }; + let buffer = unsafe { std::slice::from_raw_parts_mut(dst, length) }; + match context.file.read_exact_at(buffer, offset) { + Ok(()) => 0, + Err(_) => -1, + } + } + + unsafe extern "C" fn file_release(ctx: *mut c_void) { + // SAFETY: the context was leaked from a `Box` when the callbacks were built, and this runs + // exactly once. + drop(unsafe { Box::from_raw(ctx.cast::()) }); + } + + /// Writing a file with the stock sink and reading it back through the callback data source + /// must yield the same dtype and row count: the callback plug point is wired up correctly. + #[test] + #[cfg_attr(miri, ignore)] + fn test_callback_data_source_round_trip() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path().to_str().unwrap().to_string(); + let dtype = DType::Primitive(PType::I32, false.into()); + + unsafe { + let session = vx_session_new(); + let vx_dtype_ptr = vx_dtype::new(dtype.clone()); + let mut error = ptr::null_mut(); + + let sink = vx_array_sink_open_file( + session, + vx_view::from_str(&path), + vx_dtype_ptr, + &raw mut error, + ); + assert!(error.is_null()); + let array = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); + let vx_array_ptr = vx_array::new(array.into_array()); + vx_array_sink_push(sink, vx_array_ptr, &raw mut error); + assert!(error.is_null()); + vx_array_sink_close(sink, &raw mut error); + assert!(error.is_null()); + vx_array_free(vx_array_ptr); + vx_dtype_free(vx_dtype_ptr); + + let file = File::open(&path).unwrap(); + let size = file.metadata().unwrap().len(); + let context = Box::into_raw(Box::new(FileContext { file })); + let callbacks = vx_input_callbacks { + ctx: context.cast::(), + read_at_fn: Some(file_read_at), + release_fn: Some(file_release), + }; + + let data_source = vx_data_source_new_callback(session, callbacks, size, &raw mut error); + assert!(error.is_null()); + assert!(!data_source.is_null()); + + let mut row_count = vx_estimate::default(); + vx_data_source_get_row_count(data_source, &raw mut row_count); + assert_eq!(row_count.r#type, vx_estimate_type::VX_ESTIMATE_EXACT); + assert_eq!(row_count.estimate, 3); + + vx_data_source_free(data_source); + vx_session_free(session); + } + } + + /// A rejected request must still release the host context: the entry point takes ownership + /// before it validates anything, so the host never has to guess whether to clean up. + #[test] + #[cfg_attr(miri, ignore)] + fn test_rejected_callbacks_release_context() { + static RELEASED: AtomicBool = AtomicBool::new(false); + + unsafe extern "C" fn mark_released(_ctx: *mut c_void) { + RELEASED.store(true, Ordering::SeqCst); + } + + unsafe { + let session = vx_session_new(); + let mut error = ptr::null_mut(); + // No read callback, so the call must fail. + let callbacks = vx_input_callbacks { + ctx: ptr::null_mut(), + read_at_fn: None, + release_fn: Some(mark_released), + }; + + let data_source = vx_data_source_new_callback(session, callbacks, 0, &raw mut error); + assert!(data_source.is_null()); + assert!(!error.is_null()); + vx_error_free(error); + assert!( + RELEASED.load(Ordering::SeqCst), + "release_fn must run when the data source fails to open" + ); + + vx_session_free(session); + } + } + + /// Writing through the callback sink and reading the bytes back through the callback data source + /// must round-trip, with no filesystem involved in either direction. + #[test] + #[cfg_attr(miri, ignore)] + fn test_callback_sink_round_trip() { + /// Host context standing in for a paimon stream pair: an in-memory byte buffer. + struct BufferContext { + data: Mutex>, + } + + unsafe extern "C" fn buffer_write(ctx: *mut c_void, src: *const u8, length: usize) -> i32 { + // SAFETY: `ctx` is the `Arc` handed to the sink, and `src` covers + // `length` bytes, per the callback contract. + let holder = unsafe { &*ctx.cast::>() }; + let bytes = unsafe { std::slice::from_raw_parts(src, length) }; + holder.data.lock().unwrap().extend_from_slice(bytes); + 0 + } + + unsafe extern "C" fn buffer_read_at( + ctx: *mut c_void, + offset: u64, + dst: *mut u8, + length: usize, + ) -> i32 { + // SAFETY: as above; `dst` covers `length` writable bytes. + let holder = unsafe { &*ctx.cast::>() }; + let data = holder.data.lock().unwrap(); + let start = offset as usize; + let Some(end) = start.checked_add(length) else { + return -1; + }; + if end > data.len() { + return -1; + } + let dst = unsafe { std::slice::from_raw_parts_mut(dst, length) }; + dst.copy_from_slice(&data[start..end]); + 0 + } + + unsafe extern "C" fn buffer_release(ctx: *mut c_void) { + // SAFETY: each handle was given its own leaked `Arc` clone, released exactly once. + drop(unsafe { Box::from_raw(ctx.cast::>()) }); + } + + let shared = Arc::new(BufferContext { + data: Mutex::new(Vec::new()), + }); + let dtype = DType::Primitive(PType::I32, false.into()); + + unsafe { + let session = vx_session_new(); + let vx_dtype_ptr = vx_dtype::new(dtype.clone()); + let mut error = ptr::null_mut(); + + let out_ctx = Box::into_raw(Box::new(Arc::clone(&shared))); + let sink = vx_callback_sink_open( + session, + vx_output_callbacks { + ctx: out_ctx.cast::(), + write_fn: Some(buffer_write), + flush_fn: None, + release_fn: Some(buffer_release), + }, + vx_dtype_ptr, + &raw mut error, + ); + assert!(error.is_null()); + assert!(!sink.is_null()); + + let array = + PrimitiveArray::new(buffer![10i32, 20i32, 30i32, 40i32], Validity::NonNullable); + let vx_array_ptr = vx_array::new(array.into_array()); + vx_callback_sink_push(sink, vx_array_ptr, &raw mut error); + assert!(error.is_null()); + vx_callback_sink_close(sink, &raw mut error); + assert!(error.is_null()); + vx_array_free(vx_array_ptr); + + let size = shared.data.lock().unwrap().len() as u64; + assert!(size > 0, "the sink must have written through the callback"); + + let in_ctx = Box::into_raw(Box::new(Arc::clone(&shared))); + let data_source = vx_data_source_new_callback( + session, + vx_input_callbacks { + ctx: in_ctx.cast::(), + read_at_fn: Some(buffer_read_at), + release_fn: Some(buffer_release), + }, + size, + &raw mut error, + ); + assert!(error.is_null()); + assert!(!data_source.is_null()); + + let mut row_count = vx_estimate::default(); + vx_data_source_get_row_count(data_source, &raw mut row_count); + assert_eq!(row_count.r#type, vx_estimate_type::VX_ESTIMATE_EXACT); + assert_eq!(row_count.estimate, 4); + + vx_data_source_free(data_source); + vx_dtype_free(vx_dtype_ptr); + vx_session_free(session); + } + } +} diff --git a/docs/source/user_guide/format_table.rst b/docs/source/user_guide/format_table.rst index 0e066ad1..492b5f0d 100644 --- a/docs/source/user_guide/format_table.rst +++ b/docs/source/user_guide/format_table.rst @@ -289,13 +289,13 @@ The layout, the option precedence and the read, write and overwrite semantics de follow Java Paimon's. What this implementation covers is nonetheless a subset of what Java's format table does; it does not yet support: -* the ``csv``, ``json``, ``text``, ``lance`` and ``mosaic`` file formats, leaving ``parquet`` - and ``orc``. - The first three are line-delimited text in Java, which shares one line-reading layer between - them; this library has no text file format at all, so the first of them to be added has to bring - that layer with it. ``lance`` and ``mosaic`` each have their own reader and writer, built under - ``PAIMON_ENABLE_LANCE`` and ``PAIMON_ENABLE_MOSAIC`` respectively, but a format table does not - reach them yet; +* the ``csv``, ``json``, ``text``, ``lance``, ``mosaic`` and ``vortex`` file formats, leaving + ``parquet`` and ``orc``. The first three are line-delimited text in Java, which shares one + line-reading layer between them; this library has no text file format at all, so the first of + them to be added has to bring that layer with it. ``lance``, ``mosaic`` and ``vortex`` are + neither: each has a reader and a writer of its own, which this library builds under + ``PAIMON_ENABLE_LANCE``, ``PAIMON_ENABLE_MOSAIC`` and ``PAIMON_ENABLE_VORTEX`` respectively, but + a format table does not reach them yet; * cutting one large data file into byte ranges so that several readers share it. Java does this for an uncompressed ``csv`` or ``json`` file written with the default line delimiter, and for no other format - not for ``text`` or ``mosaic`` either; ``parquet`` and ``orc`` each record where diff --git a/src/paimon/common/reader/data_file_reader_factory.cpp b/src/paimon/common/reader/data_file_reader_factory.cpp index 1321cb39..94561075 100644 --- a/src/paimon/common/reader/data_file_reader_factory.cpp +++ b/src/paimon/common/reader/data_file_reader_factory.cpp @@ -34,13 +34,16 @@ namespace paimon { namespace { /// Formats the prefetching reader cannot drive: `blob` is read whole rather than in batches and -/// `avro` is row-oriented, so neither has the batch boundaries it reads ahead to. `mosaic` has -/// never been tried under one. Lance's Rust reader drives its own batch readahead. -// TODO(xinyu.lxy): test mosaic under prefetch. A format table cannot hold mosaic files, so this -// only concerns the managed table path. +/// `avro` is row-oriented, so neither has the batch boundaries it reads ahead to. `mosaic` and +/// `vortex` (Rust FFI) have never been tried under one; Lance's Rust reader drives its own batch +/// readahead. +// TODO(xinyu.lxy): test mosaic/vortex under prefetch. A format table cannot hold mosaic or vortex +// files (FormatTable::ParseFormat accepts only parquet/orc), so this only concerns the managed +// table path. bool FormatSupportsPrefetch(const std::string& format_identifier) { return format_identifier != "blob" && format_identifier != "avro" && - format_identifier != "lance" && format_identifier != "mosaic"; + format_identifier != "lance" && format_identifier != "mosaic" && + format_identifier != "vortex"; } } // namespace diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 2a9c3322..b3284fb3 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -341,6 +341,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { PAIMON_RETURN_NOT_OK(ValidateBlobFields(schema, options)); PAIMON_RETURN_NOT_OK(ValidateMosaicDataFields(schema, options)); PAIMON_RETURN_NOT_OK(ValidateLanceDataFields(schema, options)); + PAIMON_RETURN_NOT_OK(ValidateVortexDataFields(schema, options)); PAIMON_RETURN_NOT_OK(ValidateMapStorageLayout(schema, options)); PAIMON_RETURN_NOT_OK(ValidateVectorFields(schema, options)); return Status::OK(); @@ -886,6 +887,73 @@ Status SchemaValidation::ValidateLanceDataFields(const TableSchema& schema, return Status::OK(); } +Status SchemaValidation::ValidateVortexDataField(const std::shared_ptr& field) { + if (VariantTypeUtils::IsVariantField(field)) { + return Status::Invalid("Vortex file format does not support type VARIANT"); + } + if (BlobUtils::IsBlobField(field)) { + return Status::Invalid("Vortex file format does not support type BLOB"); + } + + const std::shared_ptr& type = field->type(); + switch (type->id()) { + case arrow::Type::BOOL: + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::INT64: + case arrow::Type::FLOAT: + case arrow::Type::DOUBLE: + case arrow::Type::DATE32: + case arrow::Type::STRING: + case arrow::Type::BINARY: + case arrow::Type::TIME32: + case arrow::Type::TIMESTAMP: + case arrow::Type::DECIMAL128: + return Status::OK(); + case arrow::Type::LIST: + case arrow::Type::FIXED_SIZE_LIST: + // Array / Vector: recurse into the element type. + return ValidateVortexDataField(type->field(0)); + case arrow::Type::STRUCT: { + // Row: recurse into every field. + const auto& struct_type = checked_cast(*type); + for (int32_t i = 0; i < struct_type.num_fields(); ++i) { + PAIMON_RETURN_NOT_OK(ValidateVortexDataField(struct_type.field(i))); + } + return Status::OK(); + } + case arrow::Type::MAP: + // MAP (and MULTISET, which is stored as a MAP) is not supported by Vortex. + return Status::Invalid("Vortex file format does not support type MAP"); + default: + break; + } + return Status::Invalid( + fmt::format("Vortex file format does not support type {}", type->ToString())); +} + +Status SchemaValidation::ValidateVortexDataFields(const TableSchema& schema, + const CoreOptions& options) { + if (StringUtils::ToLowerCase(options.GetFileFormat()->Identifier()) != "vortex") { + return Status::OK(); + } + + const std::vector inline_blob_fields = options.GetBlobInlineFields(); + const std::set inline_blob_field_set(inline_blob_fields.begin(), + inline_blob_fields.end()); + // Mirror the Mosaic path: only validate fields stored in the normal data file. A non-inline + // BLOB lives in a separate blob file, so it is skipped here; an inline BLOB is rejected. + for (const DataField& field : schema.Fields()) { + if (BlobUtils::IsBlobField(field.ArrowField()) && + inline_blob_field_set.count(field.Name()) == 0) { + continue; + } + PAIMON_RETURN_NOT_OK(ValidateVortexDataField(field.ArrowField())); + } + return Status::OK(); +} + Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, const CoreOptions& options) { // Extract all field names that have map.storage-layout configured from options diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 8c7d84c5..5794365d 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -129,6 +129,10 @@ class SchemaValidation { static Status ValidateLanceDataField(const std::shared_ptr& field); + static Status ValidateVortexDataFields(const TableSchema& schema, const CoreOptions& options); + + static Status ValidateVortexDataField(const std::shared_ptr& field); + static Status ValidateMapStorageLayout(const TableSchema& schema, const CoreOptions& options); static Status ValidateVectorFields(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/table/format/format_table.cpp b/src/paimon/core/table/format/format_table.cpp index a039baa4..0b9aeb73 100644 --- a/src/paimon/core/table/format/format_table.cpp +++ b/src/paimon/core/table/format/format_table.cpp @@ -67,7 +67,7 @@ Result FormatTable::ParseFormat(const std::string& file_for return Format::ORC; } if (normalized == "csv" || normalized == "text" || normalized == "json" || - normalized == "mosaic") { + normalized == "mosaic" || normalized == "vortex") { return Status::NotImplemented( fmt::format("format table file format '{}' is not supported by paimon-cpp yet. " "Supported formats: parquet, orc", diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index 6071a749..9b527390 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -613,7 +613,7 @@ TEST(FormatTableTest, TestParseFormat) { // Format table formats with no reader here yet answer `NotImplemented`, which is a different // answer from a name that is no format at all. - for (const char* format : {"csv", "text", "json", "mosaic"}) { + for (const char* format : {"csv", "text", "json", "mosaic", "vortex"}) { Result unimplemented = FormatTable::ParseFormat(format); ASSERT_FALSE(unimplemented.ok()) << format; ASSERT_TRUE(unimplemented.status().IsNotImplemented()) << format; diff --git a/src/paimon/format/vortex/CMakeLists.txt b/src/paimon/format/vortex/CMakeLists.txt new file mode 100644 index 00000000..a03a5842 --- /dev/null +++ b/src/paimon/format/vortex/CMakeLists.txt @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Vortex file format (Rust FFI, statically linked via paimon_vortex_ffi). + +if(PAIMON_ENABLE_VORTEX) + set(PAIMON_VORTEX_FILE_FORMAT + vortex_file_batch_reader.cpp + vortex_file_format.cpp + vortex_file_format_factory.cpp + vortex_format_writer.cpp + vortex_ffi_util.cpp + vortex_io_callbacks.cpp + vortex_stats_extractor.cpp + vortex_writer_builder.cpp) + + add_paimon_lib(paimon_vortex_file_format + SOURCES + ${PAIMON_VORTEX_FILE_FORMAT} + DEPENDENCIES + paimon_shared + paimon_vortex_ffi + STATIC_LINK_LIBS + arrow + fmt + paimon_vortex_ffi + Threads::Threads + SHARED_LINK_LIBS + paimon_shared + paimon_vortex_ffi + SHARED_LINK_FLAGS + ${PAIMON_VERSION_SCRIPT_FLAGS}) + + target_link_libraries(paimon_vortex_file_format_objlib PUBLIC paimon_vortex_ffi) + + if(PAIMON_BUILD_TESTS) + add_paimon_test(vortex_format_test + SOURCES + vortex_file_format_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + ${PAIMON_VORTEX_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} + paimon_vortex_ffi + ${GTEST_LINK_TOOLCHAIN}) + endif() +endif() diff --git a/src/paimon/format/vortex/vortex_ffi.h b/src/paimon/format/vortex/vortex_ffi.h new file mode 100644 index 00000000..9eba29ef --- /dev/null +++ b/src/paimon/format/vortex/vortex_ffi.h @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +// vortex.h can define the Arrow C-data-interface structs (ArrowSchema / +// ArrowArray / ArrowArrayStream) itself, which would collide with +// arrow/c/abi.h. vortex.h guards its own copies with USE_OWN_ARROW, so include +// Arrow's abi.h first (canonical definitions) and define USE_OWN_ARROW to make +// vortex.h reuse them. The vx_* API references `struct Arrow*` directly, so the +// skipped FFI_* typedefs are not needed. +#include "arrow/c/abi.h" + +#ifndef USE_OWN_ARROW +#define USE_OWN_ARROW +#endif + +// vortex.h's API refers to the Arrow C-data-interface structs via these typedef +// names, which it skips defining under USE_OWN_ARROW. Provide them from Arrow's +// definitions, exactly as vortex.h's own header comment instructs. +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; + +extern "C" { +#include "vortex.h" // NOLINT(build/include_subdir) +} + +// Callback-based data source, added to vortex-ffi by this repository (see +// crates/vortex_callback_io/callback_io.rs). Vortex's own C API can only read a whole file already +// in memory or a path it resolves itself, neither of which goes through paimon's FileSystem. These +// declarations are kept here because the upstream cbindgen-generated vortex.h does not know about +// them; they must stay in sync with the Rust definitions. +extern "C" { + +/// Fill `length` bytes starting at `offset` into `dst`. Returns 0 on success and non-zero on +/// failure; a partial read must be reported as a failure. Only the status code crosses the +/// boundary, so the callback is expected to keep its error detail on `ctx`. +/// +/// Must be thread-safe: Vortex issues concurrent positional reads for the same context. +typedef int32_t (*vx_read_at_fn)(void* ctx, uint64_t offset, uint8_t* dst, size_t length); + +/// Release `ctx`. Called exactly once, when the owning data source is freed (including when it +/// fails to open). +typedef void (*vx_release_fn)(void* ctx); + +/// Host callbacks backing a positional reader. +typedef struct vx_input_callbacks { + /// Opaque host context, passed back to every callback. + void* ctx; + /// Positional read. Required. + vx_read_at_fn read_at_fn; + /// Context destructor. Optional; may be null. + vx_release_fn release_fn; +} vx_input_callbacks; + +/// Create a data source that reads through `callbacks`. `size` is the total file length in bytes. +/// Returns null and sets `err` on failure. +const vx_data_source* vx_data_source_new_callback(const vx_session* session, + vx_input_callbacks callbacks, uint64_t size, + vx_error** err); + +/// Append `length` bytes from `src` to the host sink. Returns 0 on success and non-zero on failure; +/// a partial write must be reported as a failure. Writes for one sink are sequential, never +/// concurrent. +typedef int32_t (*vx_write_fn)(void* ctx, const uint8_t* src, size_t length); + +/// Flush whatever the host has buffered. Returns 0 on success and non-zero on failure. +typedef int32_t (*vx_flush_fn)(void* ctx); + +/// Host callbacks backing a sequential writer. +typedef struct vx_output_callbacks { + /// Opaque host context, passed back to every callback. + void* ctx; + /// Sequential write. Required. + vx_write_fn write_fn; + /// Flush. Optional; may be null, in which case flush requests are ignored. + vx_flush_fn flush_fn; + /// Context destructor. Optional; may be null. + vx_release_fn release_fn; +} vx_output_callbacks; + +/// A sink writing a Vortex file through host callbacks. Mirrors `vx_array_sink`, which can only +/// target a local filesystem path. +typedef struct vx_callback_sink vx_callback_sink; + +/// Open a sink writing through `callbacks`. Returns null and sets `err` on failure. Write errors +/// are reported by `vx_callback_sink_close`, since the bytes are produced by a background task. +vx_callback_sink* vx_callback_sink_open(const vx_session* session, vx_output_callbacks callbacks, + const vx_dtype* dtype, vx_error** err); + +/// Push an array into the sink. Does not take ownership of `array`. +void vx_callback_sink_push(vx_callback_sink* sink, const vx_array* array, vx_error** error_out); + +/// Flush everything pushed so far and write the file footer. Consumes `sink` even on error. +void vx_callback_sink_close(vx_callback_sink* sink, vx_error** error_out); + +/// Discard the sink without writing a footer, leaving whatever the host received invalid. +void vx_callback_sink_abort(vx_callback_sink* sink); +} diff --git a/src/paimon/format/vortex/vortex_ffi_util.cpp b/src/paimon/format/vortex/vortex_ffi_util.cpp new file mode 100644 index 00000000..83e7c296 --- /dev/null +++ b/src/paimon/format/vortex/vortex_ffi_util.cpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/vortex/vortex_ffi_util.h" + +namespace paimon::vortex { + +Status VortexFfiError(const std::string& operation, vx_error* error) { + if (error == nullptr) { + return Status::OK(); + } + vx_view message = vx_error_message(error); + std::string text = + message.ptr == nullptr ? std::string() : std::string(message.ptr, message.len); + vx_error_free(error); + return Status::Invalid(operation, ": ", text); +} + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_ffi_util.h b/src/paimon/format/vortex/vortex_ffi_util.h new file mode 100644 index 00000000..6172b317 --- /dev/null +++ b/src/paimon/format/vortex/vortex_ffi_util.h @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/format/vortex/vortex_ffi.h" +#include "paimon/status.h" + +namespace paimon::vortex { + +/// Build a paimon `Status` from a Vortex error and free it. Returns OK when `error` is null, so it +/// can be called unconditionally after an FFI call that reports errors through a `vx_error**` +/// out-parameter. +Status VortexFfiError(const std::string& operation, vx_error* error); + +// Owning RAII aliases for the Vortex C handles; each frees its handle on scope exit. The element +// type matches the constness returned by the corresponding constructor function. +using VxSessionPtr = std::unique_ptr; +using VxDataSourcePtr = std::unique_ptr; +using VxScanPtr = std::unique_ptr; +using VxPartitionPtr = std::unique_ptr; +using VxDtypePtr = std::unique_ptr; +using VxArrayPtr = std::unique_ptr; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.cpp b/src/paimon/format/vortex/vortex_file_batch_reader.cpp new file mode 100644 index 00000000..5079143d --- /dev/null +++ b/src/paimon/format/vortex/vortex_file_batch_reader.cpp @@ -0,0 +1,468 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/vortex/vortex_file_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/utils/arrow/arrow_utils.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/math.h" +#include "paimon/format/vortex/vortex_ffi_util.h" +#include "paimon/fs/file_system.h" + +namespace paimon::vortex { + +namespace { + +// Vortex's Arrow export represents strings and binaries with the view types (StringView / +// BinaryView). paimon schemas use the standard string/binary types, so map the view types back, +// recursing through nested types. Types that contain no views are returned unchanged. +std::shared_ptr NormalizeViewType(const std::shared_ptr& type) { + switch (type->id()) { + case arrow::Type::STRING_VIEW: + return arrow::utf8(); + case arrow::Type::BINARY_VIEW: + return arrow::binary(); + case arrow::Type::STRUCT: { + arrow::FieldVector fields; + fields.reserve(type->num_fields()); + for (const std::shared_ptr& field : type->fields()) { + fields.push_back(field->WithType(NormalizeViewType(field->type()))); + } + return arrow::struct_(fields); + } + case arrow::Type::LIST: + return arrow::list(type->field(0)->WithType(NormalizeViewType(type->field(0)->type()))); + case arrow::Type::LARGE_LIST: + return arrow::large_list( + type->field(0)->WithType(NormalizeViewType(type->field(0)->type()))); + case arrow::Type::FIXED_SIZE_LIST: + return arrow::fixed_size_list( + type->field(0)->WithType(NormalizeViewType(type->field(0)->type())), + checked_cast(*type).list_size()); + default: + return type; + } +} + +std::shared_ptr NormalizeViewSchema(const std::shared_ptr& schema) { + arrow::FieldVector fields; + fields.reserve(schema->num_fields()); + for (const std::shared_ptr& field : schema->fields()) { + fields.push_back(field->WithType(NormalizeViewType(field->type()))); + } + return arrow::schema(fields, schema->metadata()); +} + +// Rebuilds `array` with every StringView/BinaryView column (which Vortex's Arrow export produces) +// converted to the standard string/binary type. Arrow 17 has no cast kernel between the view and +// standard types, so the leaves are rebuilt with builders and the parents reconstructed, recursing +// through structs and lists. Arrays with no view-typed data are returned unchanged. `array` must be +// offset-normalized (offset 0) so the rebuilt parents and children stay consistent. +Result> NormalizeViewArray(const std::shared_ptr& array, + arrow::MemoryPool* pool) { + switch (array->type_id()) { + case arrow::Type::STRING_VIEW: { + const auto& view = checked_cast(*array); + arrow::StringBuilder builder(pool); + for (int64_t i = 0; i < view.length(); ++i) { + if (view.IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(view.GetView(i))); + } + } + std::shared_ptr out; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&out)); + return out; + } + case arrow::Type::BINARY_VIEW: { + const auto& view = checked_cast(*array); + arrow::BinaryBuilder builder(pool); + for (int64_t i = 0; i < view.length(); ++i) { + if (view.IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(view.GetView(i))); + } + } + std::shared_ptr out; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&out)); + return out; + } + case arrow::Type::STRUCT: { + const auto& struct_array = checked_cast(*array); + arrow::ArrayVector children; + arrow::FieldVector fields; + children.reserve(struct_array.num_fields()); + fields.reserve(struct_array.num_fields()); + bool changed = false; + for (int32_t i = 0; i < struct_array.num_fields(); ++i) { + const std::shared_ptr& child = struct_array.field(i); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_child, + NormalizeViewArray(child, pool)); + changed = changed || normalized_child.get() != child.get(); + fields.push_back(array->type()->field(i)->WithType(normalized_child->type())); + children.push_back(std::move(normalized_child)); + } + if (!changed) { + return array; + } + return std::make_shared( + arrow::struct_(fields), struct_array.length(), children, struct_array.null_bitmap(), + struct_array.null_count(), struct_array.offset()); + } + case arrow::Type::LIST: { + const auto& list = checked_cast(*array); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr values, + NormalizeViewArray(list.values(), pool)); + if (values.get() == list.values().get()) { + return array; + } + return std::make_shared( + arrow::list(array->type()->field(0)->WithType(values->type())), list.length(), + list.value_offsets(), values, list.null_bitmap(), list.null_count(), list.offset()); + } + default: + return array; + } +} + +// Projects a struct array to the columns named by `read_schema`, selected by field name and ordered +// as in `read_schema`, so NextBatch returns exactly the read schema (the FileBatchReader contract; +// paimon's FieldMappingReader maps fields but does not re-project). Returns `array` unchanged when +// it already matches, or when `read_schema` is null (SetReadSchema not yet called). +Result> ProjectToReadSchema( + const std::shared_ptr& array, const std::shared_ptr& read_schema) { + if (read_schema == nullptr) { + return array; + } + const std::shared_ptr target_type = arrow::struct_(read_schema->fields()); + if (array->type()->Equals(target_type)) { + return array; + } + const auto& struct_array = checked_cast(*array); + const auto& struct_type = checked_cast(*array->type()); + arrow::ArrayVector children; + children.reserve(read_schema->num_fields()); + for (const std::shared_ptr& read_field : read_schema->fields()) { + const int32_t index = struct_type.GetFieldIndex(read_field->name()); + if (index < 0) { + return Status::Invalid(fmt::format( + "Vortex read field '{}' is not present in the file schema", read_field->name())); + } + children.push_back(struct_array.field(index)); + } + return std::make_shared(target_type, struct_array.length(), children, + struct_array.null_bitmap(), + struct_array.null_count(), struct_array.offset()); +} + +} // namespace + +VortexFileBatchReader::VortexFileBatchReader( + const std::shared_ptr& input, int32_t batch_size, + std::shared_ptr input_context, VxSessionPtr session, + VxDataSourcePtr data_source, VxScanPtr scan, const std::shared_ptr& file_schema, + const std::shared_ptr& struct_type, uint64_t total_rows, + const std::shared_ptr& pool, const std::shared_ptr& arrow_pool) + : input_(input), + batch_size_(batch_size), + input_context_(std::move(input_context)), + session_(std::move(session)), + data_source_(std::move(data_source)), + scan_(std::move(scan)), + file_schema_(NormalizeViewSchema(file_schema)), + struct_type_(struct_type), + total_rows_(total_rows), + pool_(pool), + arrow_pool_(arrow_pool), + metrics_(std::make_shared()) {} + +Result> VortexFileBatchReader::Create( + const std::shared_ptr& input, int32_t batch_size, + const std::shared_ptr& pool, const std::shared_ptr& arrow_pool) { + if (input == nullptr || pool == nullptr || batch_size <= 0) { + return Status::Invalid( + "Vortex reader requires non-null input and memory pool, and positive batch size"); + } + // Vortex reads the file through positional callbacks into `input`, so nothing is staged here. + PAIMON_ASSIGN_OR_RAISE(int64_t signed_length, input->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(signed_length, "Vortex input length")); + auto length = static_cast(signed_length); + auto input_context = std::make_shared(input); + + VxSessionPtr session(vx_session_new(), vx_session_free); + if (session == nullptr) { + return Status::IOError("failed to create Vortex session"); + } + vx_error* error = nullptr; + VxDataSourcePtr data_source( + vx_data_source_new_callback(session.get(), VortexInputContext::MakeCallbacks(input_context), + length, &error), + vx_data_source_free); + if (data_source == nullptr) { + return VortexCallbackError("open Vortex callback data source", error, + input_context->GetCallbackStatus()); + } + + // The data source dtype is available without consuming a scan. + VxDtypePtr dtype(vx_data_source_dtype(data_source.get()), vx_dtype_free); + if (dtype == nullptr) { + return Status::IOError("failed to read Vortex data source dtype"); + } + ::ArrowSchema ffi_schema = {}; + error = nullptr; + if (vx_dtype_to_arrow_schema(session.get(), dtype.get(), &ffi_schema, &error) != 0) { + return VortexFfiError("convert Vortex dtype to Arrow schema", error); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, + arrow::ImportSchema(&ffi_schema)); + // `file_schema` here still carries Vortex's view types. `struct_type` is built from it raw on + // purpose, because NextBatch imports each scanned batch as this exact type; the constructor + // normalizes the separate copy it keeps as file_schema_. + std::shared_ptr struct_type = arrow::struct_(file_schema->fields()); + + vx_estimate row_count{}; + vx_data_source_get_row_count(data_source.get(), &row_count); + if (row_count.type == VX_ESTIMATE_UNKNOWN) { + return Status::Invalid("Vortex data source did not report a row count"); + } + const uint64_t total_rows = row_count.estimate; + + // The scan is created lazily on first read (and re-created by SetReadSchema), since a Vortex + // scan may be consumed only once. + return std::unique_ptr(new VortexFileBatchReader( + input, batch_size, std::move(input_context), std::move(session), std::move(data_source), + VxScanPtr(nullptr, vx_scan_free), file_schema, struct_type, total_rows, pool, arrow_pool)); +} + +VortexFileBatchReader::~VortexFileBatchReader() { + CloseInternal(); +} + +Result VortexFileBatchReader::OpenNextPartitionStream() { + if (scan_ == nullptr) { + vx_error* error = nullptr; + // NULL options: scan all rows and columns (no projection/predicate pushdown). + VxScanPtr scan(vx_data_source_scan(data_source_.get(), /*options=*/nullptr, + /*estimate=*/nullptr, &error), + vx_scan_free); + if (scan == nullptr) { + return VortexCallbackError("create Vortex scan", error, + input_context_->GetCallbackStatus()); + } + scan_ = std::move(scan); + } + vx_error* error = nullptr; + VxPartitionPtr partition(vx_scan_next_partition(scan_.get(), &error), vx_partition_free); + if (error != nullptr) { + return VortexCallbackError("advance Vortex scan partition", error, + input_context_->GetCallbackStatus()); + } + if (partition == nullptr) { + return false; // Scan exhausted. + } + // vx_partition_scan_arrow consumes the partition (even on error), so release ownership here. + ::ArrowArrayStream stream{}; + error = nullptr; + if (vx_partition_scan_arrow(session_.get(), partition.release(), &stream, &error) != 0) { + return VortexCallbackError("scan Vortex partition to Arrow", error, + input_context_->GetCallbackStatus()); + } + current_stream_ = stream; + stream_active_ = true; + return true; +} + +Result> VortexFileBatchReader::ReadNextArray() { + while (true) { + if (!stream_active_) { + PAIMON_ASSIGN_OR_RAISE(bool opened, OpenNextPartitionStream()); + if (!opened) { + return std::shared_ptr(); // End of scan. + } + } + ::ArrowArray ffi_array = {}; + const int32_t rc = current_stream_.get_next(¤t_stream_, &ffi_array); + if (rc != 0) { + const char* message = current_stream_.get_last_error(¤t_stream_); + // A read failure surfaces here as an opaque stream error, so prefer the status the IO + // callback stashed. + Status callback_status = input_context_->GetCallbackStatus(); + ReleaseStream(); + if (!callback_status.ok()) { + return callback_status.WithMessage("read Vortex batch", ": ", + callback_status.message()); + } + return Status::IOError("Vortex Arrow stream error: ", + message == nullptr ? "unknown" : message); + } + if (ffi_array.release == nullptr) { + ReleaseStream(); // Partition exhausted; try the next one. + continue; + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + arrow::ImportArray(&ffi_array, struct_type_)); + if (array->length() == 0) { + continue; // Skip empty batches. + } + return array; + } +} + +Result VortexFileBatchReader::NextBatch() { + if (closed_) { + return Status::Invalid("Vortex reader is closed"); + } + if (current_batch_ == nullptr || current_batch_offset_ == current_batch_->length()) { + PAIMON_ASSIGN_OR_RAISE(current_batch_, ReadNextArray()); + current_batch_offset_ = 0; + } + if (current_batch_ == nullptr) { + previous_batch_row_count_ = 0; + return BatchReader::MakeEofBatch(); + } + + const int64_t row_count = + std::min(batch_size_, current_batch_->length() - current_batch_offset_); + std::shared_ptr sliced_array = + current_batch_->Slice(current_batch_offset_, row_count); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_array, + ArrowUtils::NormalizeArrayOffsets(sliced_array, arrow_pool_.get())); + // Vortex exports strings/binaries as Arrow view types; convert them to the standard types + // (Arrow 17 has no cast kernel for this) so the batch matches the file schema paimon expects. + // Done after offset normalization so the array is offset-0. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output_array, + NormalizeViewArray(normalized_array, arrow_pool_.get())); + // Honor the read schema: NextBatch must return exactly its columns (selected by name), which + // paimon's read path relies on (FieldMappingReader maps fields but does not re-project). + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr projected_array, + ProjectToReadSchema(output_array, read_schema_)); + + previous_first_row_ = rows_emitted_; + previous_batch_row_count_ = static_cast(row_count); + rows_emitted_ += static_cast(row_count); + current_batch_offset_ += row_count; + + auto ffi_array = std::make_unique<::ArrowArray>(); + auto ffi_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*projected_array, ffi_array.get(), ffi_schema.get())); + PAIMON_RETURN_NOT_OK(AddArrowArrayLifetime(ffi_array.get(), ffi_schema.get(), arrow_pool_)); + return std::make_pair(std::move(ffi_array), std::move(ffi_schema)); +} + +Result> VortexFileBatchReader::GetFileSchema() const { + auto schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema_, schema.get())); + return schema; +} + +Status VortexFileBatchReader::SetReadSchema( + ::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) { + if (read_schema == nullptr) { + return Status::Invalid("Vortex read schema is nullptr"); + } + (void)selection_bitmap; + // Projection and predicates are not pushed into Vortex: the whole file is scanned and + // NextBatch projects the result to `read_schema`. Reading restarts from the first row, so drop + // the single-use scan and any open stream; the next read re-creates the scan. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr imported_read_schema, + arrow::ImportSchema(read_schema)); + read_schema_ = std::move(imported_read_schema); + predicate_filter_ = std::dynamic_pointer_cast(predicate); + ReleaseStream(); + current_batch_.reset(); + current_batch_offset_ = 0; + scan_.reset(); + rows_emitted_ = 0; + previous_first_row_ = std::numeric_limits::max(); + previous_batch_row_count_ = 0; + return Status::OK(); +} + +Result VortexFileBatchReader::GetPreviousBatchFileRowId(uint64_t batch_row_id) const { + if (previous_batch_row_count_ == 0) { + return Status::Invalid(previous_first_row_ == std::numeric_limits::max() + ? "no Vortex batch has been read yet" + : "last Vortex batch was EOF"); + } + if (batch_row_id >= previous_batch_row_count_) { + return Status::Invalid(fmt::format("batch row id {} is out of range {}", batch_row_id, + previous_batch_row_count_)); + } + return previous_first_row_ + batch_row_id; +} + +Result VortexFileBatchReader::GetNumberOfRows() const { + return total_rows_; +} + +std::shared_ptr VortexFileBatchReader::GetReaderMetrics() const { + return metrics_; +} + +void VortexFileBatchReader::Close() { + CloseInternal(); +} + +void VortexFileBatchReader::ReleaseStream() { + if (stream_active_) { + if (current_stream_.release != nullptr) { + current_stream_.release(¤t_stream_); + } + current_stream_ = ::ArrowArrayStream{}; + stream_active_ = false; + } +} + +void VortexFileBatchReader::CloseInternal() { + if (closed_) { + return; + } + // Release in dependency order: the Arrow stream (needs the session alive) first, then the scan + // (borrows the data source), the data source (reads through the input context), the session, + // then our reference to the input context. A read still running on a Vortex thread keeps the + // context alive through the reference its callbacks hold. + ReleaseStream(); + current_batch_.reset(); + scan_.reset(); + data_source_.reset(); + session_.reset(); + input_context_.reset(); + if (input_ != nullptr) { + (void)input_->Close(); + } + input_.reset(); + closed_ = true; +} + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.h b/src/paimon/format/vortex/vortex_file_batch_reader.h new file mode 100644 index 00000000..21b44fa9 --- /dev/null +++ b/src/paimon/format/vortex/vortex_file_batch_reader.h @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "arrow/type_fwd.h" +#include "paimon/format/vortex/vortex_ffi.h" +#include "paimon/format/vortex/vortex_ffi_util.h" +#include "paimon/format/vortex/vortex_io_callbacks.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class DataType; +class MemoryPool; +class Schema; +} // namespace arrow + +namespace paimon { +class InputStream; +class MemoryPool; +class Metrics; +class PredicateFilter; +} // namespace paimon + +namespace paimon::vortex { + +/// Reads a Vortex file via the vortex-ffi scan API. +/// +/// IO goes through `vx_data_source_new_callback`, the callback-based data source this repository +/// adds to vortex-ffi: Vortex issues positional reads back into `input_context_`, which forwards +/// them to the paimon `InputStream`. Nothing is staged in memory, range reads stay lazy, and any +/// paimon `FileSystem` works. Each batch the scan produces is converted through the Arrow C Data +/// Interface. +class VortexFileBatchReader : public FileBatchReader { + public: + static Result> Create( + const std::shared_ptr& input, int32_t batch_size, + const std::shared_ptr& pool, + const std::shared_ptr& arrow_pool); + + ~VortexFileBatchReader() override; + + Result NextBatch() override; + Result> GetFileSchema() const override; + Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) override; + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override; + Result GetNumberOfRows() const override; + std::shared_ptr GetReaderMetrics() const override; + void Close() override; + bool SupportPreciseBitmapSelection() const override { + return false; + } + + private: + VortexFileBatchReader(const std::shared_ptr& input, int32_t batch_size, + std::shared_ptr input_context, VxSessionPtr session, + VxDataSourcePtr data_source, VxScanPtr scan, + const std::shared_ptr& file_schema, + const std::shared_ptr& struct_type, uint64_t total_rows, + const std::shared_ptr& pool, + const std::shared_ptr& arrow_pool); + + /// Pull the next Arrow array from the current partition stream, opening the next partition's + /// stream when the current one is exhausted. Returns nullptr at end of scan. + Result> ReadNextArray(); + /// Open the next partition and fill `current_stream_` via `vx_partition_scan_arrow`. + /// Returns false when there are no more partitions. + Result OpenNextPartitionStream(); + void ReleaseStream(); + void CloseInternal(); + + // Teardown order matters, so these are declared to be destroyed in a safe reverse order and + // also reset explicitly in CloseInternal(): scan_ borrows data_source_, data_source_ reads + // through input_context_, and the Arrow stream must be released while session_ is alive. + std::shared_ptr input_; + int32_t batch_size_; + std::shared_ptr input_context_; + VxSessionPtr session_; + VxDataSourcePtr data_source_; + VxScanPtr scan_; + // The file schema with Vortex's string/binary view types normalized to the standard types; this + // is what GetFileSchema() reports to paimon. + std::shared_ptr file_schema_; + // The RAW view-typed struct that Vortex's Arrow export produces. NextBatch imports each scanned + // batch as this type (so it must match the C export's buffer layout), then NormalizeViewArray + // rewrites it to file_schema_'s types. Keeping this view-typed is load-bearing: normalizing it + // would make ImportArray misread the buffers. + std::shared_ptr struct_type_; + // The schema NextBatch must output, set by SetReadSchema (the projected read schema). Null + // until SetReadSchema is called, in which case the full file schema is returned. + std::shared_ptr read_schema_; + uint64_t total_rows_ = 0; + + ::ArrowArrayStream current_stream_{}; + bool stream_active_ = false; + std::shared_ptr current_batch_; + int64_t current_batch_offset_ = 0; + + uint64_t rows_emitted_ = 0; + uint64_t previous_first_row_ = std::numeric_limits::max(); + uint64_t previous_batch_row_count_ = 0; + + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; + std::shared_ptr predicate_filter_; // from SetReadSchema; not pushed down + std::shared_ptr metrics_; + bool closed_ = false; +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_file_format.cpp b/src/paimon/format/vortex/vortex_file_format.cpp new file mode 100644 index 00000000..9d85b2fb --- /dev/null +++ b/src/paimon/format/vortex/vortex_file_format.cpp @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/vortex/vortex_file_format.h" + +#include "arrow/c/bridge.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/format/vortex/vortex_reader_builder.h" +#include "paimon/format/vortex/vortex_stats_extractor.h" +#include "paimon/format/vortex/vortex_writer_builder.h" + +namespace paimon::vortex { + +Result> VortexFileFormat::CreateReaderBuilder( + int32_t batch_size) const { + return std::make_unique(batch_size); +} + +Result> VortexFileFormat::CreateWriterBuilder( + ::ArrowSchema* schema, int32_t batch_size) const { + (void)batch_size; + if (schema == nullptr) { + return Status::Invalid("Vortex writer schema is nullptr"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, + arrow::ImportSchema(schema)); + return std::make_unique(typed_schema, options_); +} + +Result> VortexFileFormat::CreateStatsExtractor( + ::ArrowSchema* schema) const { + if (schema == nullptr) { + return Status::Invalid("Vortex stats schema is nullptr"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr typed_schema, + arrow::ImportSchema(schema)); + return std::make_unique(typed_schema); +} + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_file_format.h b/src/paimon/format/vortex/vortex_file_format.h new file mode 100644 index 00000000..422b8f7b --- /dev/null +++ b/src/paimon/format/vortex/vortex_file_format.h @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/format/file_format.h" + +namespace paimon::vortex { + +class VortexFileFormat : public FileFormat { + public: + explicit VortexFileFormat(const std::map& options) + : identifier_("vortex"), options_(options) {} + + const std::string& Identifier() const override { + return identifier_; + } + Result> CreateReaderBuilder(int32_t batch_size) const override; + Result> CreateWriterBuilder(::ArrowSchema* schema, + int32_t batch_size) const override; + Result> CreateStatsExtractor( + ::ArrowSchema* schema) const override; + + private: + std::string identifier_; + std::map options_; +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_file_format_factory.cpp b/src/paimon/format/vortex/vortex_file_format_factory.cpp new file mode 100644 index 00000000..6bc3861a --- /dev/null +++ b/src/paimon/format/vortex/vortex_file_format_factory.cpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/vortex/vortex_file_format_factory.h" + +#include "paimon/factories/factory.h" +#include "paimon/format/vortex/vortex_file_format.h" + +namespace paimon::vortex { + +const char VortexFileFormatFactory::IDENTIFIER[] = "vortex"; + +Result> VortexFileFormatFactory::Create( + const std::map& options) const { + return std::make_unique(options); +} + +REGISTER_PAIMON_FACTORY(VortexFileFormatFactory); + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_file_format_factory.h b/src/paimon/format/vortex/vortex_file_format_factory.h new file mode 100644 index 00000000..7b25ad2d --- /dev/null +++ b/src/paimon/format/vortex/vortex_file_format_factory.h @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/format/file_format_factory.h" + +namespace paimon::vortex { + +class VortexFileFormatFactory : public FileFormatFactory { + public: + static const char IDENTIFIER[]; + + const char* Identifier() const override { + return IDENTIFIER; + } + Result> Create( + const std::map& options) const override; +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_file_format_test.cpp b/src/paimon/format/vortex/vortex_file_format_test.cpp new file mode 100644 index 00000000..b07e0402 --- /dev/null +++ b/src/paimon/format/vortex/vortex_file_format_test.cpp @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Round-trip unit tests for the Vortex file format: write, then read back via vortex-ffi. + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/array/concatenate.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/factories/io_hook.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/format/column_stats.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/format/format_stats_extractor.h" +#include "paimon/format/format_writer.h" +#include "paimon/format/reader_builder.h" +#include "paimon/format/writer_builder.h" +#include "paimon/fs/file_system.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/testing/utils/io_exception_helper.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::vortex::test { + +class VortexFileFormatTest : public ::testing::Test { + public: + void SetUp() override { + ASSERT_OK_AND_ASSIGN(format_, + FileFormatFactory::Get("vortex", {{"file.format", "vortex"}})); + file_system_ = std::make_shared(); + directory_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_NE(directory_, nullptr); + pool_ = GetDefaultPool(); + arrow_pool_ = GetArrowPool(pool_); + } + + // Writes `array` (a struct array matching `schema`) to a Vortex file at `path`, pushing it in + // `batch_size` slices exactly like the production write path does. + Status WriteFile(const std::string& path, const std::shared_ptr& schema, + const std::shared_ptr& array, int32_t batch_size) const { + ::ArrowSchema ffi_schema = {}; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer_builder, + format_->CreateWriterBuilder(&ffi_schema, batch_size)); + writer_builder->WithMemoryPool(pool_); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output, + file_system_->Create(path, /*overwrite=*/false)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + writer_builder->Build(output, "zstd")); + for (int64_t offset = 0; offset < array->length(); offset += batch_size) { + std::shared_ptr slice = array->Slice(offset, batch_size); + ::ArrowArray ffi_array = {}; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*slice, &ffi_array)); + PAIMON_RETURN_NOT_OK(writer->AddBatch(&ffi_array)); + } + PAIMON_RETURN_NOT_OK(writer->Finish()); + return output->Close(); + } + + // Reads the whole Vortex file at `path` back as a single concatenated struct array, checking + // the reported row count and that no batch exceeds the configured batch size. + Result> ReadFile(const std::string& path, + const std::shared_ptr& schema, + int32_t batch_size, + uint64_t expected_row_count) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, + format_->CreateReaderBuilder(batch_size)); + reader_builder->WithMemoryPool(pool_); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, file_system_->Open(path)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + reader_builder->Build(input)); + ::ArrowSchema ffi_schema = {}; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema)); + PAIMON_RETURN_NOT_OK(reader->SetReadSchema(&ffi_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + PAIMON_ASSIGN_OR_RAISE(uint64_t total_rows, reader->GetNumberOfRows()); + if (total_rows != expected_row_count) { + return Status::Invalid("unexpected Vortex row count"); + } + + std::vector> batches; + uint64_t rows_seen = 0; + while (true) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + PAIMON_RETURN_NOT_OK(paimon::test::ReadResultCollector::CheckBatchOffset(batch)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr arrow_batch, + arrow::ImportArray(batch.first.get(), batch.second.get())); + if (arrow_batch->length() > batch_size) { + return Status::Invalid("Vortex read batch exceeds configured batch size"); + } + rows_seen += static_cast(arrow_batch->length()); + batches.push_back(std::move(arrow_batch)); + } + if (rows_seen != expected_row_count) { + return Status::Invalid("Vortex batches do not contain all rows"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr result, + arrow::Concatenate(batches, arrow_pool_.get())); + return result; + } + + void AssertReadWithBatchSizes(const std::string& path, + const std::shared_ptr& schema, + const std::shared_ptr& expected, + std::initializer_list batch_sizes) const { + for (int32_t batch_size : batch_sizes) { + SCOPED_TRACE("batch_size=" + std::to_string(batch_size)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr actual, + ReadFile(path, schema, batch_size, + /*expected_row_count=*/static_cast(expected->length()))); + ASSERT_TRUE(actual->Equals(expected)) << actual->ToString() << "\nvs\n" + << expected->ToString(); + } + } + + protected: + std::shared_ptr format_; + std::shared_ptr file_system_; + std::unique_ptr directory_; + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; +}; + +TEST_F(VortexFileFormatTest, WriteThenRead) { + std::string path = PathUtil::JoinPath(directory_->Str(), "data.vortex"); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false), + arrow::field("name", arrow::utf8())}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(fields), R"([[1,"one"],[2,null],[3,"three"],[4,"four"],[5,"five"]])") + .ValueOrDie(); + + ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2)); + AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5, 8}); +} + +TEST_F(VortexFileFormatTest, WriteThenReadSupportedTypes) { + std::string path = PathUtil::JoinPath(directory_->Str(), "supported-types.vortex"); + arrow::FieldVector fields = { + arrow::field("f0", arrow::boolean()), + arrow::field("f1", arrow::int8()), + arrow::field("f2", arrow::int16()), + arrow::field("f3", arrow::int32()), + arrow::field("f4", arrow::int64()), + arrow::field("f5", arrow::float32()), + arrow::field("f6", arrow::float64()), + arrow::field("f7", arrow::utf8()), + arrow::field("f8", arrow::binary()), + arrow::field("f9", arrow::date32()), + arrow::field("f10", arrow::timestamp(arrow::TimeUnit::NANO)), + arrow::field("f11", arrow::decimal128(2, 2)), + arrow::field("f12", arrow::decimal128(30, 2)), + arrow::field("f13", arrow::list(arrow::float32())), + }; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [true,-128,-32768,-2147483648,-4294967298,0.5,1.141592659,"vortex","binary", + -1,"1970-01-01 00:00:00.000000001","-0.99","-123456789987654321.45",[1.5,null]], + [false,127,32767,2147483647,4294967296,2.0,3.141592657,"","", + 12345,"2030-12-31 23:59:59.999999999","0.78","123456789987654321.45",[]], + [null,null,null,null,null,null,null,null,null,null,null,null,null,null] + ])") + .ValueOrDie(); + + ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2)); + AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5}); +} + +TEST_F(VortexFileFormatTest, ExtractStatisticsReportsRowCount) { + std::string path = PathUtil::JoinPath(directory_->Str(), "statistics.vortex"); + arrow::FieldVector fields = {arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8())}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr data = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(fields), R"([[1,"one"],[2,"two"],[3,null],[4,"four"],[5,"five"]])") + .ValueOrDie(); + ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2)); + + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr extractor, + format_->CreateStatsExtractor(&ffi_schema)); + ASSERT_OK_AND_ASSIGN(auto result, extractor->ExtractWithFileInfo(file_system_, path, pool_)); + ASSERT_EQ(result.second.GetRowCount(), 5); + // Vortex exposes no per-column statistics; only the row count is verified. + ASSERT_TRUE(result.first.empty()); +} + +TEST_F(VortexFileFormatTest, ProjectedReadReturnsOnlyRequestedColumns) { + std::string path = PathUtil::JoinPath(directory_->Str(), "projection.vortex"); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false), + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::float64())}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr written = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([[1,"one",1.5],[2,null,2.5],[3,"three",3.5]])") + .ValueOrDie(); + ASSERT_OK(WriteFile(path, schema, written, /*batch_size=*/2)); + + // Read a reordered subset {name, id}; the reader must return exactly those columns, proving it + // honors SetReadSchema's projection rather than always emitting the full file schema. + std::shared_ptr projected = arrow::schema({fields[1], fields[0]}); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[1], fields[0]}), + R"([["one",1],[null,2],["three",3]])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadFile(path, projected, /*batch_size=*/2, /*expected_row_count=*/3)); + ASSERT_TRUE(actual->Equals(expected)) << actual->ToString() << "\nvs\n" << expected->ToString(); +} + +// The IO callback bridge must surface a paimon IO failure as that paimon error, not Vortex's +// opaque "stream error": the callback can only hand a status code back across the FFI boundary, so +// the reader/writer stashes the real error and prefers it. `IOHook` injects a failure at the Nth +// LocalFile IO; the loop walks the injection point across every IO of the whole read (or write) +// workflow, so each failure position is checked, and stops once the point falls past the last IO +// and the workflow runs clean. `CHECK_HOOK_STATUS` continues the loop when the surfaced error is +// the injected one (proving it propagated), and fails the test on any other error. +TEST_F(VortexFileFormatTest, ReadSurfacesInjectedIOError) { + std::string path = PathUtil::JoinPath(directory_->Str(), "read-io-error.vortex"); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false), + arrow::field("name", arrow::utf8())}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr written = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([[1,"one"],[2,null],[3,"three"]])") + .ValueOrDie(); + // Write the fixture with the hook disarmed so only the read path is exercised below. + ASSERT_OK(WriteFile(path, schema, written, /*batch_size=*/2)); + + IOHook* io_hook = IOHook::GetInstance(); + bool run_complete = false; + for (size_t i = 0; i < 200; i++) { + ScopeGuard guard([io_hook]() { io_hook->Clear(); }); + io_hook->Reset(static_cast(i), IOHook::Mode::RETURN_ERROR); + + Result> input = file_system_->Open(path); + CHECK_HOOK_STATUS(input.status(), i); + Result> reader_builder = + format_->CreateReaderBuilder(/*batch_size=*/2); + CHECK_HOOK_STATUS(reader_builder.status(), i); + reader_builder.value()->WithMemoryPool(pool_); + + std::shared_ptr input_stream = std::move(input).value(); + Result> reader = + reader_builder.value()->Build(input_stream); + CHECK_HOOK_STATUS(reader.status(), i); + + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + CHECK_HOOK_STATUS(reader.value()->SetReadSchema(&ffi_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + i); + + bool eof = false; + while (!eof) { + Result batch = reader.value()->NextBatch(); + CHECK_HOOK_STATUS(batch.status(), i); + eof = BatchReader::IsEofBatch(batch.value()); + } + run_complete = true; + break; + } + ASSERT_TRUE(run_complete); +} + +TEST_F(VortexFileFormatTest, WriteSurfacesInjectedIOError) { + std::string path = PathUtil::JoinPath(directory_->Str(), "write-io-error.vortex"); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false)}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr data = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([[1],[2],[3]])") + .ValueOrDie(); + + IOHook* io_hook = IOHook::GetInstance(); + bool run_complete = false; + for (size_t i = 0; i < 200; i++) { + ScopeGuard guard([io_hook]() { io_hook->Clear(); }); + io_hook->Reset(static_cast(i), IOHook::Mode::RETURN_ERROR); + + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + Result> writer_builder = + format_->CreateWriterBuilder(&ffi_schema, /*batch_size=*/2); + CHECK_HOOK_STATUS(writer_builder.status(), i); + writer_builder.value()->WithMemoryPool(pool_); + + Result> output = + file_system_->Create(path, /*overwrite=*/true); + CHECK_HOOK_STATUS(output.status(), i); + std::shared_ptr output_stream = std::move(output).value(); + + Result> writer = + writer_builder.value()->Build(output_stream, "zstd"); + CHECK_HOOK_STATUS(writer.status(), i); + + ::ArrowArray ffi_array = {}; + ASSERT_TRUE(arrow::ExportArray(*data, &ffi_array).ok()); + CHECK_HOOK_STATUS(writer.value()->AddBatch(&ffi_array), i); + CHECK_HOOK_STATUS(writer.value()->Finish(), i); + CHECK_HOOK_STATUS(output_stream->Close(), i); + run_complete = true; + break; + } + ASSERT_TRUE(run_complete); +} + +} // namespace paimon::vortex::test diff --git a/src/paimon/format/vortex/vortex_format_writer.cpp b/src/paimon/format/vortex/vortex_format_writer.cpp new file mode 100644 index 00000000..5e56ae38 --- /dev/null +++ b/src/paimon/format/vortex/vortex_format_writer.cpp @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/vortex/vortex_format_writer.h" + +#include + +#include "arrow/c/bridge.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/format/vortex/vortex_ffi_util.h" +#include "paimon/fs/file_system.h" + +namespace paimon::vortex { + +VortexFormatWriter::VortexFormatWriter(std::shared_ptr output, + std::shared_ptr schema, VxSessionPtr session, + std::shared_ptr output_context, + vx_callback_sink* sink) + : output_(std::move(output)), + schema_(std::move(schema)), + session_(std::move(session)), + output_context_(std::move(output_context)), + sink_(sink), + metrics_(std::make_shared()) {} + +Result> VortexFormatWriter::Create( + const std::shared_ptr& output, const std::shared_ptr& schema) { + if (output == nullptr || schema == nullptr) { + return Status::Invalid("Vortex writer requires non-null output and schema"); + } + VxSessionPtr session(vx_session_new(), vx_session_free); + if (session == nullptr) { + return Status::IOError("failed to create Vortex session"); + } + ::ArrowSchema ffi_schema = {}; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema)); + vx_error* error = nullptr; + // vx_dtype_from_arrow_schema consumes ffi_schema on both success and failure. + VxDtypePtr dtype(vx_dtype_from_arrow_schema(session.get(), &ffi_schema, &error), vx_dtype_free); + if (dtype == nullptr) { + return VortexFfiError("convert Arrow schema to Vortex dtype", error); + } + error = nullptr; + auto output_context = std::make_shared(output); + vx_callback_sink* sink = vx_callback_sink_open( + session.get(), VortexOutputContext::MakeCallbacks(output_context), dtype.get(), &error); + if (sink == nullptr) { + return VortexCallbackError("open Vortex callback sink", error, + output_context->GetCallbackStatus()); + } + return std::unique_ptr(new VortexFormatWriter( + output, schema, std::move(session), std::move(output_context), sink)); +} + +VortexFormatWriter::~VortexFormatWriter() { + if (sink_ != nullptr) { + // Not finished: abort so Vortex drops the sink without writing a footer. Whatever reached + // the output stream is an incomplete file, which the caller discards along with it. + vx_callback_sink_abort(sink_); + sink_ = nullptr; + } +} + +Status VortexFormatWriter::AddBatch(::ArrowArray* batch) { + if (batch == nullptr) { + return Status::Invalid("Vortex writer batch is nullptr"); + } + if (finished_ || sink_ == nullptr) { + return Status::Invalid("cannot add a batch after Vortex writer is finished"); + } + ::ArrowSchema ffi_schema = {}; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, &ffi_schema)); + vx_error* error = nullptr; + // vx_array_from_arrow consumes both `batch` and `ffi_schema` on success and on failure. + VxArrayPtr array( + vx_array_from_arrow(session_.get(), batch, &ffi_schema, /*nullable=*/false, &error), + vx_array_free); + if (array == nullptr) { + return VortexFfiError("convert Arrow batch to Vortex array", error); + } + error = nullptr; + vx_callback_sink_push(sink_, array.get(), &error); + if (error != nullptr) { + return VortexCallbackError("push batch to Vortex sink", error, + output_context_->GetCallbackStatus()); + } + return Status::OK(); +} + +Status VortexFormatWriter::Flush() { + if (finished_) { + return Status::OK(); + } + // Vortex buffers internally, so this only flushes what its sink has already handed over. + return output_->Flush(); +} + +Status VortexFormatWriter::Finish() { + if (finished_) { + return Status::OK(); + } + if (sink_ == nullptr) { + return Status::Invalid("Vortex writer sink is not open"); + } + vx_error* error = nullptr; + vx_callback_sink_close(sink_, &error); + // close consumes the sink even when it reports an error + sink_ = nullptr; + if (error != nullptr) { + return VortexCallbackError("close Vortex writer", error, + output_context_->GetCallbackStatus()); + } + // A write callback that failed leaves the file incomplete even if Vortex itself reported + // success, so surface it here as well. + PAIMON_RETURN_NOT_OK(output_context_->GetCallbackStatus()); + finished_ = true; + return Status::OK(); +} + +Result VortexFormatWriter::ReachTargetSize(bool suggested_check, int64_t target_size) const { + if (!suggested_check || output_context_ == nullptr) { + return false; + } + // Bytes already handed to the output stream. Vortex still holds buffered data and writes the + // footer on close, so this is a lower bound and rolling only ever happens later than ideal. + return output_context_->BytesWritten() >= target_size; +} + +std::shared_ptr VortexFormatWriter::GetWriterMetrics() const { + return metrics_; +} + +Status VortexFormatWriter::AddMetadata(const std::map& metadata) { + (void)metadata; + return Status::NotImplemented("Vortex writer metadata is not supported"); +} + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_format_writer.h b/src/paimon/format/vortex/vortex_format_writer.h new file mode 100644 index 00000000..4df57405 --- /dev/null +++ b/src/paimon/format/vortex/vortex_format_writer.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/format/format_writer.h" +#include "paimon/format/vortex/vortex_ffi.h" +#include "paimon/format/vortex/vortex_ffi_util.h" +#include "paimon/format/vortex/vortex_io_callbacks.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow +namespace paimon { +class Metrics; +class OutputStream; +} // namespace paimon + +namespace paimon::vortex { + +/// Writes a Vortex file. +/// +/// Bytes go straight to the paimon `OutputStream` through `vx_callback_sink_open`, the +/// callback-based sink this repository adds to vortex-ffi: Vortex writes into `output_context_`, +/// which forwards to the stream. Vortex's own sink (`vx_array_sink_open_file`) can only create a +/// local file, which would mean staging the file on local disk and copying it back on finish. +class VortexFormatWriter : public FormatWriter { + public: + static Result> Create( + const std::shared_ptr& output, const std::shared_ptr& schema); + + ~VortexFormatWriter() override; + + Status AddBatch(::ArrowArray* batch) override; + Status Flush() override; + Status Finish() override; + Result ReachTargetSize(bool suggested_check, int64_t target_size) const override; + std::shared_ptr GetWriterMetrics() const override; + Status AddMetadata(const std::map& metadata) override; + + private: + VortexFormatWriter(std::shared_ptr output, std::shared_ptr schema, + VxSessionPtr session, std::shared_ptr output_context, + vx_callback_sink* sink); + + std::shared_ptr output_; + std::shared_ptr schema_; + VxSessionPtr session_; + std::shared_ptr output_context_; + // Raw pointer on purpose: close (in Finish) and abort (in the destructor) are mutually + // exclusive terminations that each consume the sink even on error, which a single-deleter + // RAII alias cannot express. Nulled right after either, so it is never used or freed twice. + vx_callback_sink* sink_; + std::shared_ptr metrics_; + bool finished_ = false; +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_io_callbacks.cpp b/src/paimon/format/vortex/vortex_io_callbacks.cpp new file mode 100644 index 00000000..e80d2bbd --- /dev/null +++ b/src/paimon/format/vortex/vortex_io_callbacks.cpp @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/vortex/vortex_io_callbacks.h" + +#include + +#include "paimon/common/utils/math.h" +#include "paimon/format/vortex/vortex_ffi_util.h" +#include "paimon/fs/file_system.h" +#include "paimon/result.h" + +namespace paimon::vortex { + +namespace { + +/// The callbacks carry a heap-allocated `shared_ptr` copy, since only a raw pointer fits through +/// the FFI boundary. Recover the context from it, or null when the pointer is not usable. +template +Context* ContextFrom(void* ctx) { + auto* holder = static_cast*>(ctx); + return holder == nullptr ? nullptr : holder->get(); +} + +} // namespace + +VortexInputContext::VortexInputContext(std::shared_ptr input) + : input_(std::move(input)) {} + +vx_input_callbacks VortexInputContext::MakeCallbacks( + const std::shared_ptr& context) { + // Only a raw pointer fits through the FFI boundary, so a shared reference is handed over as + // one: the holder is released here and taken back by `Release`, which frees it. + auto holder = std::make_unique>(context); + vx_input_callbacks callbacks = {}; + callbacks.ctx = holder.release(); + callbacks.read_at_fn = VortexInputContext::ReadAt; + callbacks.release_fn = VortexInputContext::Release; + return callbacks; +} + +void VortexInputContext::Release(void* ctx) noexcept { + // Take the shared reference back so it is dropped exactly once. + const std::unique_ptr> holder( + static_cast*>(ctx)); +} + +void VortexInputContext::SetCallbackStatus(const Status& status) { + std::lock_guard lock(mutex_); + if (callback_status_.ok()) { + callback_status_ = status; + } +} + +Status VortexInputContext::GetCallbackStatus() const { + std::lock_guard lock(mutex_); + return callback_status_; +} + +int32_t VortexInputContext::ReadAt(void* ctx, uint64_t offset, uint8_t* dst, + size_t length) noexcept { + VortexInputContext* context = ContextFrom(ctx); + if (context == nullptr || dst == nullptr) { + if (context != nullptr) { + context->SetCallbackStatus(Status::Invalid("invalid Vortex read request")); + } + return -1; + } + Status status = ValidateValueInRange(offset, "Vortex read offset"); + if (status.ok()) { + status = ValidateValueInRange(length, "Vortex read length"); + } + if (!status.ok()) { + context->SetCallbackStatus(status); + return -1; + } + auto read_length = static_cast(length); + auto read_offset = static_cast(offset); + Result result = + context->input_->Read(reinterpret_cast(dst), read_length, read_offset); + if (!result.ok()) { + context->SetCallbackStatus(result.status()); + return -1; + } + int64_t bytes_read = std::move(result).value(); + if (bytes_read != read_length) { + context->SetCallbackStatus(Status::IOError("short read while reading Vortex file")); + return -1; + } + return 0; +} + +Status VortexCallbackError(const std::string& operation, vx_error* error, + const Status& callback_status) { + // Consume the Vortex error either way, then prefer the callback's error: it is the root cause, + // while Vortex only sees an opaque failure code. + Status ffi_status = VortexFfiError(operation, error); + if (!callback_status.ok()) { + return callback_status.WithMessage(operation, ": ", callback_status.message()); + } + if (!ffi_status.ok()) { + return ffi_status; + } + return Status::IOError(operation, ": unknown Vortex error"); +} + +VortexOutputContext::VortexOutputContext(std::shared_ptr output) + : output_(std::move(output)) {} + +vx_output_callbacks VortexOutputContext::MakeCallbacks( + const std::shared_ptr& context) { + // Ownership is handed over the same way as on the read side; see MakeCallbacks above. + auto holder = std::make_unique>(context); + vx_output_callbacks callbacks = {}; + callbacks.ctx = holder.release(); + callbacks.write_fn = VortexOutputContext::Write; + callbacks.flush_fn = VortexOutputContext::Flush; + callbacks.release_fn = VortexOutputContext::Release; + return callbacks; +} + +void VortexOutputContext::Release(void* ctx) noexcept { + const std::unique_ptr> holder( + static_cast*>(ctx)); +} + +void VortexOutputContext::SetCallbackStatus(const Status& status) { + std::lock_guard lock(mutex_); + if (callback_status_.ok()) { + callback_status_ = status; + } +} + +Status VortexOutputContext::GetCallbackStatus() const { + std::lock_guard lock(mutex_); + return callback_status_; +} + +int64_t VortexOutputContext::BytesWritten() const { + std::lock_guard lock(mutex_); + return bytes_written_; +} + +int32_t VortexOutputContext::Write(void* ctx, const uint8_t* src, size_t length) noexcept { + VortexOutputContext* context = ContextFrom(ctx); + if (context == nullptr || src == nullptr) { + if (context != nullptr) { + context->SetCallbackStatus(Status::Invalid("invalid Vortex write request")); + } + return -1; + } + Status status = ValidateValueInRange(length, "Vortex write length"); + if (!status.ok()) { + context->SetCallbackStatus(status); + return -1; + } + auto write_length = static_cast(length); + Result result = + context->output_->Write(reinterpret_cast(src), write_length); + if (!result.ok()) { + context->SetCallbackStatus(result.status()); + return -1; + } + int64_t bytes_written = std::move(result).value(); + if (bytes_written != write_length) { + context->SetCallbackStatus(Status::IOError("short write while writing Vortex file")); + return -1; + } + { + std::lock_guard lock(context->mutex_); + context->bytes_written_ += bytes_written; + } + return 0; +} + +int32_t VortexOutputContext::Flush(void* ctx) noexcept { + VortexOutputContext* context = ContextFrom(ctx); + if (context == nullptr) { + return -1; + } + Status status = context->output_->Flush(); + if (!status.ok()) { + context->SetCallbackStatus(status); + return -1; + } + return 0; +} + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_io_callbacks.h b/src/paimon/format/vortex/vortex_io_callbacks.h new file mode 100644 index 00000000..7e066e11 --- /dev/null +++ b/src/paimon/format/vortex/vortex_io_callbacks.h @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/format/vortex/vortex_ffi.h" +#include "paimon/status.h" + +namespace paimon { +class InputStream; +class OutputStream; +} // namespace paimon + +namespace paimon::vortex { + +/// Bridges Vortex's positional reads to a paimon `InputStream`. +/// +/// Vortex calls back into this context for every range it needs, so the file is never staged in +/// memory and any paimon `FileSystem` works. Callbacks cannot throw or return a `Status` across the +/// FFI boundary, so a failure is reported as a status code and the real error is stashed here for +/// the caller to pick up with `GetCallbackStatus()`. +class VortexInputContext { + public: + /// @param input The paimon stream every positional read is forwarded to. + explicit VortexInputContext(std::shared_ptr input); + + /// Build the FFI callbacks for `context`. + /// + /// The callbacks own a shared reference to `context`, released by `Release`, so a read still + /// running on a Vortex thread keeps the context (and its `InputStream`) alive even after the + /// reader has dropped its own reference. + /// + /// @param context The context to forward callbacks to; a shared reference is retained. + /// @return The callback struct to hand to `vx_data_source_new_callback`. + static vx_input_callbacks MakeCallbacks(const std::shared_ptr& context); + + /// `vx_read_at_fn`: fills `length` bytes at `offset` from the paimon stream. + /// + /// Called concurrently from several Vortex threads. This is safe because paimon's positional + /// `InputStream::Read` does not touch the stream position (the local implementation uses + /// `pread`), which is the same contract the mosaic format's callbacks rely on. + /// + /// @param ctx The `VortexInputContext` shared-reference holder passed as the callback context. + /// @param offset Byte offset in the file to read from. + /// @param dst Destination buffer, at least `length` bytes. + /// @param length Number of bytes to read; a short read is treated as a failure. + /// @return 0 on success, -1 on failure (the error is stashed for `GetCallbackStatus()`). + static int32_t ReadAt(void* ctx, uint64_t offset, uint8_t* dst, size_t length) noexcept; + + /// `vx_release_fn`: drops the shared reference held by the callbacks. + /// + /// @param ctx The `VortexInputContext` shared-reference holder to release. + static void Release(void* ctx) noexcept; + + /// @return The first error reported by a callback, or OK when none failed. + Status GetCallbackStatus() const; + + private: + void SetCallbackStatus(const Status& status); + + std::shared_ptr input_; + mutable std::mutex mutex_; + Status callback_status_; +}; + +/// Build a `Status` for a failed Vortex call, preferring the error a paimon IO callback stashed on +/// its context over Vortex's own message, since the callback error is the root cause. Must only be +/// called once the Vortex call is known to have failed. +/// +/// @param operation Human-readable name of the Vortex operation that failed, used as a prefix. +/// @param error The Vortex error to consume; freed by this call. May be null. +/// @param callback_status The status stashed by the IO callback; preferred when not OK. +/// @return A non-OK `Status` describing the failure. +Status VortexCallbackError(const std::string& operation, vx_error* error, + const Status& callback_status); + +/// Bridges Vortex's sequential writes to a paimon `OutputStream`. +/// +/// Vortex's own file sink can only create a local file, so without this the writer would have to +/// stage the file locally and copy it back on finish. Errors are stashed the same way as on the +/// read side, because the callbacks can only return a status code. +class VortexOutputContext { + public: + /// @param output The paimon stream every write is forwarded to. + explicit VortexOutputContext(std::shared_ptr output); + + /// Build the FFI callbacks for `context`, which own a shared reference released by `Release`. + /// + /// @param context The context to forward callbacks to; a shared reference is retained. + /// @return The callback struct to hand to `vx_callback_sink_open`. + static vx_output_callbacks MakeCallbacks(const std::shared_ptr& context); + + /// `vx_write_fn`: appends `length` bytes from `src` to the paimon stream. + /// + /// @param ctx The `VortexOutputContext` shared-reference holder passed as the callback context. + /// @param src Source buffer of `length` bytes. + /// @param length Number of bytes to write; a short write is treated as a failure. + /// @return 0 on success, -1 on failure (the error is stashed for `GetCallbackStatus()`). + static int32_t Write(void* ctx, const uint8_t* src, size_t length) noexcept; + + /// `vx_flush_fn`: flushes the paimon stream. + /// + /// @param ctx The `VortexOutputContext` shared-reference holder passed as the callback context. + /// @return 0 on success, -1 on failure (the error is stashed for `GetCallbackStatus()`). + static int32_t Flush(void* ctx) noexcept; + + /// `vx_release_fn`: drops the shared reference held by the callbacks. + /// + /// @param ctx The `VortexOutputContext` shared-reference holder to release. + static void Release(void* ctx) noexcept; + + /// @return The first error reported by a callback, or OK when none failed. + Status GetCallbackStatus() const; + + /// Bytes handed to the paimon stream so far. + /// + /// This is a lower bound on the final file size: Vortex buffers data internally and only writes + /// the footer on close, so a size-based rolling decision using this value errs on the side of + /// writing a larger file. + /// + /// @return The number of bytes written to the paimon stream so far. + int64_t BytesWritten() const; + + private: + void SetCallbackStatus(const Status& status); + + std::shared_ptr output_; + mutable std::mutex mutex_; + Status callback_status_; + int64_t bytes_written_ = 0; +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_reader_builder.h b/src/paimon/format/vortex/vortex_reader_builder.h new file mode 100644 index 00000000..87696d6f --- /dev/null +++ b/src/paimon/format/vortex/vortex_reader_builder.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/format/reader_builder.h" +#include "paimon/format/vortex/vortex_file_batch_reader.h" +#include "paimon/memory/memory_pool.h" + +namespace paimon::vortex { + +class VortexReaderBuilder : public ReaderBuilder { + public: + explicit VortexReaderBuilder(int32_t batch_size) + : batch_size_(batch_size), pool_(GetDefaultPool()), arrow_pool_(GetArrowPool(pool_)) {} + + ReaderBuilder* WithMemoryPool(const std::shared_ptr& pool) override { + pool_ = pool; + if (pool == nullptr) { + arrow_pool_.reset(); + } else { + arrow_pool_ = GetArrowPool(pool); + } + return this; + } + + Result> Build( + const std::shared_ptr& input) const override { + if (pool_ == nullptr) { + return Status::Invalid("Vortex reader memory pool is nullptr"); + } + return VortexFileBatchReader::Create(input, batch_size_, pool_, arrow_pool_); + } + + private: + int32_t batch_size_; + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_stats_extractor.cpp b/src/paimon/format/vortex/vortex_stats_extractor.cpp new file mode 100644 index 00000000..e7a3f32c --- /dev/null +++ b/src/paimon/format/vortex/vortex_stats_extractor.cpp @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/vortex/vortex_stats_extractor.h" + +#include + +#include "paimon/common/utils/math.h" +#include "paimon/format/vortex/vortex_ffi.h" +#include "paimon/format/vortex/vortex_ffi_util.h" +#include "paimon/format/vortex/vortex_io_callbacks.h" +#include "paimon/fs/file_system.h" + +namespace paimon::vortex { + +Result> +VortexStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& file_system, + const std::string& path, + const std::shared_ptr& pool) { + if (file_system == nullptr) { + return Status::Invalid("Vortex stats extractor requires a file system"); + } + if (schema_ == nullptr) { + return Status::Invalid("Vortex stats extractor has no schema"); + } + (void)pool; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input, file_system->Open(path)); + // Only the row count is needed, and Vortex keeps it in the file footer. Reading through + // callbacks means just those ranges are fetched instead of the whole file. + PAIMON_ASSIGN_OR_RAISE(int64_t signed_length, input->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(signed_length, "Vortex input length")); + auto length = static_cast(signed_length); + auto input_context = std::make_shared(input); + + VxSessionPtr session(vx_session_new(), vx_session_free); + if (session == nullptr) { + return Status::IOError("failed to create Vortex session"); + } + vx_error* error = nullptr; + VxDataSourcePtr data_source( + vx_data_source_new_callback(session.get(), VortexInputContext::MakeCallbacks(input_context), + length, &error), + vx_data_source_free); + if (data_source == nullptr) { + return VortexCallbackError("open Vortex file for stats", error, + input_context->GetCallbackStatus()); + } + vx_estimate row_count{}; + vx_data_source_get_row_count(data_source.get(), &row_count); + if (row_count.type == VX_ESTIMATE_UNKNOWN) { + return Status::Invalid("Vortex file did not report a row count"); + } + const int64_t rows = static_cast(row_count.estimate); + // Empty column stats: Vortex keeps its statistics internal and does not surface them to Paimon. + return std::make_pair(ColumnStatsVector(), FileInfo(rows)); +} + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_stats_extractor.h b/src/paimon/format/vortex/vortex_stats_extractor.h new file mode 100644 index 00000000..621b494b --- /dev/null +++ b/src/paimon/format/vortex/vortex_stats_extractor.h @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/format/format_stats_extractor.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon::vortex { + +/// Vortex exposes no per-column statistics to Paimon (the Java side has no StatsExtractor either). +/// Predicate pruning for Vortex happens inside Vortex via native expression pushdown, not +/// through Paimon-level row-group statistics. This extractor therefore returns empty column stats +/// and only reads back the file's row count for `FileInfo`. +class VortexStatsExtractor : public FormatStatsExtractor { + public: + explicit VortexStatsExtractor(const std::shared_ptr& schema) : schema_(schema) {} + + Result Extract(const std::shared_ptr& file_system, + const std::string& path, + const std::shared_ptr& pool) override { + using ExtractResult = std::pair; + PAIMON_ASSIGN_OR_RAISE(ExtractResult result, ExtractWithFileInfo(file_system, path, pool)); + return result.first; + } + + Result> ExtractWithFileInfo( + const std::shared_ptr& file_system, const std::string& path, + const std::shared_ptr& pool) override; + + private: + std::shared_ptr schema_; +}; + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_writer_builder.cpp b/src/paimon/format/vortex/vortex_writer_builder.cpp new file mode 100644 index 00000000..4400ae27 --- /dev/null +++ b/src/paimon/format/vortex/vortex_writer_builder.cpp @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/vortex/vortex_writer_builder.h" + +#include + +#include "paimon/format/vortex/vortex_format_writer.h" +#include "paimon/status.h" + +namespace paimon::vortex { + +Result> VortexWriterBuilder::Build( + const std::shared_ptr& output, const std::string& compression) { + if (pool_ == nullptr) { + return Status::Invalid("Vortex writer memory pool is nullptr"); + } + // The paimon compression hint is not mapped to the sink; Vortex picks its own encodings. + (void)compression; + return VortexFormatWriter::Create(output, schema_); +} + +} // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_writer_builder.h b/src/paimon/format/vortex/vortex_writer_builder.h new file mode 100644 index 00000000..82aa8d18 --- /dev/null +++ b/src/paimon/format/vortex/vortex_writer_builder.h @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/format/vortex/vortex_format_writer.h" +#include "paimon/format/writer_builder.h" +#include "paimon/memory/memory_pool.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon::vortex { + +class VortexWriterBuilder : public WriterBuilder { + public: + VortexWriterBuilder(const std::shared_ptr& schema, + const std::map& options) + : schema_(schema), options_(options), pool_(GetDefaultPool()) {} + + WriterBuilder* WithMemoryPool(const std::shared_ptr& pool) override { + pool_ = pool; + return this; + } + + Result> Build(const std::shared_ptr& output, + const std::string& compression) override; + + private: + std::shared_ptr schema_; + std::map options_; + std::shared_ptr pool_; +}; + +} // namespace paimon::vortex diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index a8e3a6ed..a3a71fb6 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -3690,6 +3690,49 @@ TEST_F(ScanAndReadInteTest, TestMosaicJavaAndPythonCompatibility) { } #endif +#ifdef PAIMON_ENABLE_VORTEX +TEST_F(ScanAndReadInteTest, TestVortexJavaCompatibility) { + // Fixture written by Paimon Java 2.0.0 with file.format=vortex; see + // test/test_data/vortex/append_java_compat.db/append_java_compat/README.md. This verifies that + // paimon-cpp reads back a Vortex table produced by the Java implementation. + arrow::FieldVector fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8()), + }; + std::shared_ptr data_type = arrow::struct_(fields); + std::shared_ptr expected_array = + arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ +[0, 1, "one"], +[0, 2, null], +[0, 3, "three"], +[0, 4, "four"], +[0, 5, "five"] +])") + .ValueOrDie(); + auto expected = std::make_shared(expected_array); + + const std::string table_name = "append_java_compat"; + const std::string table_path = GetDataDir() + "/vortex/" + table_name + ".db/" + table_name; + ScanContextBuilder scan_context_builder(table_path); + ReadContextBuilder read_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadResultCollector::CollectResult(std::move(batch_reader))); + ASSERT_TRUE(expected->Equals(actual)) + << "actual: " << (actual == nullptr ? "null" : actual->ToString()) + << "\nexpected: " << expected->ToString(); +} +#endif + TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) { auto read_data = [](int64_t snapshot_id, const std::string& result_json) { std::string table_path = GetDataDir() + "/avro/append_multiple.db/append_multiple"; diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/README.md b/test/test_data/vortex/append_java_compat.db/append_java_compat/README.md new file mode 100644 index 00000000..6bc4bb4f --- /dev/null +++ b/test/test_data/vortex/append_java_compat.db/append_java_compat/README.md @@ -0,0 +1,24 @@ +Table: append_java_compat +Writer: Paimon Java 2.0.0 (paimon-vortex-format + paimon-vortex-jni 2.0.0) + +Schema: +id INT NOT NULL +name STRING + +Options: +bucket = -1 +file.format = vortex + +Data is a single APPEND commit (snapshot 1) with five rows. The second row has a +NULL name: +(1, "one") +(2, NULL) +(3, "three") +(4, "four") +(5, "five") + +Generated by a standalone Java program using the Paimon batch write API +(BatchWriteBuilder / BatchTableWrite / BatchTableCommit) on JDK 17 with +--add-opens=java.base/java.nio=ALL-UNNAMED (required by Arrow Java memory). +Read back and verified by paimon-cpp in +ScanAndReadInteTest.TestVortexJavaCompatibility. diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/bucket-0/data-430e1867-52a2-4264-ae8e-a2d9fe20bf1e-0.vortex b/test/test_data/vortex/append_java_compat.db/append_java_compat/bucket-0/data-430e1867-52a2-4264-ae8e-a2d9fe20bf1e-0.vortex new file mode 100644 index 0000000000000000000000000000000000000000..4b062f482ee32c62e9014231c9ccc098d0f50332 GIT binary patch literal 3492 zcmd^CJ!@N67=C3-vZ6Y!;<`dLt`WtA4kly>9YTf>O7Rdv1`|A#IM>oqe985ddheCu zYRcfDlnkYWP&$QTO6e3jlnxm>7!Rd%C>b(z3dKY44=A+F?em-u>spfi2YSq-bI$v| z=lwh%*SA*JJ`%KPpoHH%eoGiH0C~W)8JU-&=2v7+W;C%Tm(VW*OF&K2X7yS<#7|MC zWL8gdIY*=J@_Nx8D~sv1=CqM5%TxbG!?1HDkhX;KB(&_&?nJTY68C;?*S{o93aR> zZ6r5EofGPzpNu)L|BNkXA7pu~?N;Suj8_5bHGWp1w*<}8$f9c%sp?@a$9&UA*XUav z3w4h%W1|0qHm-(MxdVYU;5y*!E}UYQGA{QR>ZWZXsH|<1GbbVP)E1#umfD2W7VDP9 z7*4LC(MeIeT-z8or|pB+tl_-egfMI4YWPCa&A!Nk$ND`2n!qK1d3sOR;f&NVcDW^e z4s_bi*6hX!vRADv*SXf$vc(a-ftAhX>Dv?Zer07DqiesPpm)#8(&s@Nw@1&Mx*(_8^5*uilQ(DdcwMVHrg^P2TfdT~n;JZaJCA^D z?f#GT=pd2YXU|P;riSys3Lg2~Gs}SEPvNxjK0+&y|I)0_m2i(`c$1Xpi7A0LY?kC#52vEKS zd1`liiM=^TGul5BUtYgc!*(f47#}P8?lG~RY=0Z_S^EcL?32&8fcCQXsoTZl{5AHN z+q}*#W8!+G@aPUc?e6CtoQ;b#L4zLNV0L#EMa0p+21$KcQU=6qe)PcW{{6B!7fhWKN zpb4x1O90=6KVf4J*adcgJHS<-4&;Ha|1mLTeCx+aQ+}}YOsswXZr{6$HNRONznN^T zJumXT5Q%tYdFPJtHvF*XtNWmp{~gI8)`Ab-`BC0cey@$;b-XC%VZCj6Zw_q6>XWVt z8%p}4>!NO`!X{)NTON0kjiVF!TYkb<0PRQ1=e}_KMjP>4dM0OKHwffl)Yg_4-&3?T zf%lsE&iEMkaiX)bXn7fZDll80RZ>#2d?Dy5WUMi85$@s3eQJ>F8 zbvAwM;BPGboq^7#YWQ1Tu)gtWqSldXy_tWpa+%s&IrFW;7zXP^JaCWSi*Fs%z2PSvufZU>-rTby zyq|IWN-w>PAoB z5p~YClC{zdHq=+~Mc~X=F-QR$mn^#s62MfA$oE=RExA8hb|tBj&oek#RW-vhlO9Gd%K9QhV44$3uGBMH_?~=p9COPmsXm0y0o@Y|X4}YuZCe-1QJjw^}e_ zejKrRMF`Cn!Q?mrW#Kj2IEd<#Vo@)rG-0a;dWPGcm)Q;Z5r&3 z5)_{IK9!{lZVpijZ-G1<71!Q=5LFR9;^RVxNjuQgzwSOGh(s z%~6f&IYZNRjigb;SD}Qx16^W63Jx(xC-n7qhI|*v+w zf2?d?xOeeR>heKnXruqw+JWWuryCEK*6@p$chA0lRv3^T{Cre>&-TZvH&1@(m2R`& LdM;h*YrXw1=cA`8 literal 0 HcmV?d00001 diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-0 b/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-0 new file mode 100644 index 0000000000000000000000000000000000000000..f091992a2bc6d37a3757af5c004240d8aa001a90 GIT binary patch literal 1006 zcmbVLNlwEs6a|SbEGpCkcwOZH-I>OShzuwbh$>ia;9MhD%9nArsO(^IBM#G#%paU|JHrB<2gz{(3 zMhX$%W2#I@oaULb5F62qGQz2(4`Z4{1?cwY-%8j=wxe05*j#$W83$#j0-S0(!X`9O z1t~a-vr!Y}XxBAVlCNP*&%jVsk&30JQ|WC)sz^uf7T61DyP7K^V8?}SrTRyC7&nO< z>M=AE`%w4l#~sW#Eq# G|M3YpBt<*` literal 0 HcmV?d00001 diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-1 b/test/test_data/vortex/append_java_compat.db/append_java_compat/manifest/manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-1 new file mode 100644 index 0000000000000000000000000000000000000000..2a79d6a174fcc6f8df399f5ec179a262408635ce GIT binary patch literal 1110 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhPz%iElZBN9=H)&?^*Ji_a@USMvblde;GVdnHZ8<1f({Y%JLl6`A~4+ty`bpIl-H` zskhlfRuw8u71{Rw_r6+r`FM^8D_uip_y}&yoBrzQvOU`Z;tV&mElK9TpyHcYX4u5U MaNvP4g9y4k08&zOBme*a literal 0 HcmV?d00001 diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/schema/schema-0 b/test/test_data/vortex/append_java_compat.db/append_java_compat/schema/schema-0 new file mode 100644 index 00000000..75918adb --- /dev/null +++ b/test/test_data/vortex/append_java_compat.db/append_java_compat/schema/schema-0 @@ -0,0 +1,21 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "name", + "type" : "STRING" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "file.format" : "vortex" + }, + "timeMillis" : 1789455840340 +} \ No newline at end of file diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/EARLIEST b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/EARLIEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/LATEST b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/LATEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/snapshot-1 b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/snapshot-1 new file mode 100644 index 00000000..ef7d4ea1 --- /dev/null +++ b/test/test_data/vortex/append_java_compat.db/append_java_compat/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "70b40861-aaad-4ae5-8380-7e88af4f2100", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-85e17711-240b-41ef-86a7-c6aec4c25d68-1", + "deltaManifestListSize" : 1110, + "commitUser" : "420305a8-591b-49b8-93e8-fe76caea2cef", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1789456011021, + "totalRecordCount" : 5, + "deltaRecordCount" : 5, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/third_party/versions.txt b/third_party/versions.txt index 1afdf7a1..8c145235 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -72,6 +72,14 @@ PAIMON_MOSAIC_BUILD_VERSION=0.2.0 PAIMON_MOSAIC_BUILD_SHA256_CHECKSUM=8123eaadd293a7904b692eff900484691711dc0cb24878166a2f90a378d37a22 PAIMON_MOSAIC_PKG_NAME=apache-paimon-mosaic-${PAIMON_MOSAIC_BUILD_VERSION}-src.tgz +# Vortex has no Apache release and vortex-ffi is publish=false, so it is pinned to an upstream +# commit. The file format is back-compatible since vortex 0.36.0, so this reads files written by +# paimon-java's vortex-jni 0.73.0. The SHA256 is the self-computed checksum of the pinned commit's +# GitHub source archive (upstream publishes none); recompute it whenever the pin changes. +PAIMON_VORTEX_BUILD_VERSION=d82de0e1c8c46dc90a3b95a433a51668e283dfdc +PAIMON_VORTEX_BUILD_SHA256_CHECKSUM=af7dd32d1e92090293ca3bd70f5a3f97b00a5cec968a0c8e99c7612feeb35934 +PAIMON_VORTEX_PKG_NAME=vortex-${PAIMON_VORTEX_BUILD_VERSION}.tar.gz + PAIMON_AVRO_BUILD_VERSION=c499eefb48aa2db906c7bca14a047223806f36db PAIMON_AVRO_BUILD_SHA256_CHECKSUM=9771f1dcfe3c01aff7ff670e873e66d3406362f71941821d482de65f3d32d780 PAIMON_AVRO_PKG_NAME=avro-${PAIMON_AVRO_BUILD_VERSION}.tar.gz @@ -172,6 +180,7 @@ DEPENDENCIES=( "PAIMON_BENCHMARK_URL ${PAIMON_BENCHMARK_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/benchmark/archive/refs/tags/v${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz" "PAIMON_ARROW_URL ${PAIMON_ARROW_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/arrow/releases/download/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}.tar.gz" "PAIMON_MOSAIC_URL ${PAIMON_MOSAIC_PKG_NAME} https://downloads.apache.org/paimon/paimon-mosaic-${PAIMON_MOSAIC_BUILD_VERSION}/${PAIMON_MOSAIC_PKG_NAME}" + "PAIMON_VORTEX_URL ${PAIMON_VORTEX_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/vortex-data/vortex/archive/${PAIMON_VORTEX_BUILD_VERSION}.tar.gz" "PAIMON_AVRO_URL ${PAIMON_AVRO_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/avro/archive/${PAIMON_AVRO_BUILD_VERSION}.tar.gz" "PAIMON_AWS_C_AUTH_URL ${PAIMON_AWS_C_AUTH_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-auth/archive/refs/tags/${PAIMON_AWS_C_AUTH_BUILD_VERSION}.tar.gz" "PAIMON_AWS_C_CAL_URL ${PAIMON_AWS_C_CAL_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-cal/archive/refs/tags/${PAIMON_AWS_C_CAL_BUILD_VERSION}.tar.gz" From 1f0a62c2919c9baa75604d0fafc4411590210ca3 Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Mon, 21 Sep 2026 03:55:28 +0000 Subject: [PATCH 2/9] fix(ci): install libclang so Vortex bindgen builds succeed in CI --- .github/workflows/build_and_test.yaml | 2 +- .github/workflows/gcc8_test.yaml | 2 +- .github/workflows/release_candidate.yaml | 2 +- ci/scripts/setup_flatc.sh | 23 +++++++++++++++++++---- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 2db6ef04..dd84a11e 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -114,7 +114,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y gdb libcurl4-openssl-dev libssl-dev - - name: Install flatc (Vortex) + - name: Install flatc and libclang (Vortex) shell: bash run: ci/scripts/setup_flatc.sh - name: Build Paimon diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index 6b3c99e2..bdd6a039 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -67,7 +67,7 @@ jobs: uses: ./.github/actions/setup-ccache with: cache-key-prefix: ccache-gcc8-test - - name: Install flatc (Vortex) + - name: Install flatc and libclang (Vortex) shell: bash run: ci/scripts/setup_flatc.sh - name: Build Paimon diff --git a/.github/workflows/release_candidate.yaml b/.github/workflows/release_candidate.yaml index ce6b74c5..a1f8dafa 100644 --- a/.github/workflows/release_candidate.yaml +++ b/.github/workflows/release_candidate.yaml @@ -135,7 +135,7 @@ jobs: shell: bash run: ci/scripts/setup_rust.sh - - name: Install flatc (Vortex) + - name: Install flatc and libclang (Vortex) shell: bash run: ci/scripts/setup_flatc.sh diff --git a/ci/scripts/setup_flatc.sh b/ci/scripts/setup_flatc.sh index f853fa6c..47e8df8a 100755 --- a/ci/scripts/setup_flatc.sh +++ b/ci/scripts/setup_flatc.sh @@ -15,17 +15,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Build and install flatc (the FlatBuffers compiler) at the version Vortex requires. +# Build and install flatc (the FlatBuffers compiler) at the version Vortex requires, and make sure +# libclang is present. # # Vortex compiles its FlatBuffers schemas (.fbs) into Rust at build time, and # cmake_modules/ThirdpartyToolchain.cmake requires a flatc whose version matches Vortex's # flatbuffers crate (find_program(... flatc REQUIRED)). Distribution packages are too old, so the -# pinned version is built from source. flatc is a build-time tool only; it is not needed at runtime. +# pinned version is built from source. Separately, Vortex's native dependencies run bindgen in +# their build scripts, which needs libclang at build time; some CI images do not ship it. +# flatc and libclang are build-time tools only; neither is needed at runtime. # -# The dev container (see .devcontainer/) already has flatc preinstalled; this script is for the +# The dev container (see .devcontainer/) already has these preinstalled; this script is for the # GitHub Actions runners and is called before ci/scripts/build_paimon.sh. # -# Idempotent: a no-op when a matching flatc is already on PATH. +# Idempotent: a no-op when a matching flatc is already on PATH and libclang is already present. set -eux @@ -58,4 +61,16 @@ export PATH="${FLATC_INSTALL_DIR}:${PATH}" echo "${FLATC_INSTALL_DIR}" >>"${GITHUB_PATH:-/dev/null}" || true echo "FLATC=${FLATC_INSTALL_DIR}/flatc" >>"${GITHUB_ENV:-/dev/null}" || true +# Vortex's native dependencies invoke bindgen in their build scripts, which loads libclang at +# build time. Images that only carry GCC (e.g. the gcc8 test container) may lack it. +if ! ldconfig -p 2>/dev/null | grep -q 'libclang'; then + if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1; then + apt_prefix="sudo" + else + apt_prefix="" + fi + ${apt_prefix} apt-get update -y + ${apt_prefix} apt-get install -y libclang-dev +fi + "${FLATC_INSTALL_DIR}/flatc" --version From 2c20c4626fcd185f7a65cec08a7abde01353fd39 Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Mon, 21 Sep 2026 06:42:06 +0000 Subject: [PATCH 3/9] fix(ci): suppress un-instrumented Vortex Rust artifacts under LSAN/TSAN --- build_support/lsan-suppressions.txt | 5 +++++ build_support/tsan-suppressions.txt | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/build_support/lsan-suppressions.txt b/build_support/lsan-suppressions.txt index 927afb39..294091dd 100644 --- a/build_support/lsan-suppressions.txt +++ b/build_support/lsan-suppressions.txt @@ -17,3 +17,8 @@ # False positive from atexit() registration in libc leak:*__new_exitfn* + +# Vortex links an un-instrumented Rust staticlib whose runtime keeps allocations alive at process +# exit; these are not paimon leaks. Rust crate frames are prefixed vortex_, which does not +# match paimon::vortex:: C++ frames, so genuine paimon leaks are still reported. +leak:vortex_ diff --git a/build_support/tsan-suppressions.txt b/build_support/tsan-suppressions.txt index a5b1d655..53783cab 100644 --- a/build_support/tsan-suppressions.txt +++ b/build_support/tsan-suppressions.txt @@ -18,3 +18,11 @@ # Prebuilt shared libraries are not TSAN-instrumented. Suppress reports from the whole library. race:liblumina.so race:libjindosdk_c.so.6 + +# Vortex links an un-instrumented Rust staticlib. The handoff between its blocking-pool threads +# (which invoke the paimon read/write callbacks) and the calling thread happens inside Rust, so +# TSAN cannot see the happens-before edge and reports false data races on the shared buffers. +# These patterns match only the Rust frames, so races confined to paimon C++ are still reported. +race:vortex_buffer +race:vortex_io +race:CallbackReadAt From 11c19268f423f2d522e2e4ea5f6bc931e1bbf3a8 Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Tue, 22 Sep 2026 06:05:39 +0000 Subject: [PATCH 4/9] fix(format): pin Vortex to 0.75.0 for bidirectional Java interop Pin the vendored Vortex from commit d82de0e to tag 0.75.0 so paimon-cpp interoperates bidirectionally with paimon-java's vortex-jni 0.73.0: 0.75 reads 0.73-written files and, verified end-to-end, 0.73 reads 0.75-written files. Newer pins (>=0.77) break the cpp->java direction with "Encoding not found in registry". Adapt the FFI call sites to the 0.75 signatures (error message accessor, the arrow/dtype converters drop their session argument, VortexFile::data_source) and fix a latent ownership bug uncovered by the downgrade: at 0.75 vx_data_source_dtype returns a BORROWED pointer (arc_wrapper new_ref, no refcount bump), so it must not be released with vx_dtype_free. Freeing it spuriously decremented the data source's DType Arc, a use-after-free that surfaced later as a segfault in the scan path. The pointer is now treated as borrowed; it stays valid for the schema conversion, which the data source outlives. --- cmake_modules/vortex.diff | 6 +++--- crates/vortex_callback_io/callback_io.rs | 21 +++++++------------ src/paimon/format/vortex/vortex_ffi_util.cpp | 7 ++++--- .../vortex/vortex_file_batch_reader.cpp | 10 +++++++-- .../format/vortex/vortex_format_writer.cpp | 7 +++---- third_party/versions.txt | 14 +++++++------ 6 files changed, 33 insertions(+), 32 deletions(-) diff --git a/cmake_modules/vortex.diff b/cmake_modules/vortex.diff index bbd277d3..26aca103 100644 --- a/cmake_modules/vortex.diff +++ b/cmake_modules/vortex.diff @@ -1,9 +1,9 @@ diff --git a/vortex-ffi/src/lib.rs b/vortex-ffi/src/lib.rs --- a/vortex-ffi/src/lib.rs +++ b/vortex-ffi/src/lib.rs -@@ -11,4 +11,5 @@ +@@ -8,4 +8,5 @@ mod array; +mod callback_io; + mod array_iterator; + mod binary; mod data_source; - mod dtype; - mod error; diff --git a/crates/vortex_callback_io/callback_io.rs b/crates/vortex_callback_io/callback_io.rs index 099b66e7..e6487bcb 100644 --- a/crates/vortex_callback_io/callback_io.rs +++ b/crates/vortex_callback_io/callback_io.rs @@ -73,7 +73,6 @@ use vortex::io::runtime::BlockingRuntime; use vortex::io::runtime::Handle; use vortex::io::runtime::Task; use vortex::io::session::RuntimeSessionExt; -use vortex::layout::scan::multi::MultiLayoutDataSource; use crate::RUNTIME; use crate::array::vx_array; @@ -273,13 +272,7 @@ unsafe fn data_source_new_callback( }); let file = RUNTIME.block_on(async { session.open_options().open(reader).await })?; - let data_source = MultiLayoutDataSource::new_with_first( - file.layout_reader()?, - Vec::new(), - vec![Some(size)], - &session, - ); - Ok(vx_data_source::new(data_source)) + Ok(vx_data_source::new(file.data_source()?)) } /// Create a data source that reads through host callbacks. @@ -558,7 +551,6 @@ mod tests { use crate::sink::vx_array_sink_close; use crate::sink::vx_array_sink_open_file; use crate::sink::vx_array_sink_push; - use crate::string::vx_view; /// Host context standing in for paimon's `InputStream`: positional reads against a file. struct FileContext { @@ -598,18 +590,19 @@ mod tests { unsafe { let session = vx_session_new(); - let vx_dtype_ptr = vx_dtype::new(dtype.clone()); + let vx_dtype_ptr = vx_dtype::new(Arc::new(dtype.clone())); let mut error = ptr::null_mut(); + let c_path = std::ffi::CString::new(path.clone()).unwrap(); let sink = vx_array_sink_open_file( session, - vx_view::from_str(&path), + c_path.as_ptr(), vx_dtype_ptr, &raw mut error, ); assert!(error.is_null()); let array = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); - let vx_array_ptr = vx_array::new(array.into_array()); + let vx_array_ptr = vx_array::new(Arc::new(array.into_array())); vx_array_sink_push(sink, vx_array_ptr, &raw mut error); assert!(error.is_null()); vx_array_sink_close(sink, &raw mut error); @@ -726,7 +719,7 @@ mod tests { unsafe { let session = vx_session_new(); - let vx_dtype_ptr = vx_dtype::new(dtype.clone()); + let vx_dtype_ptr = vx_dtype::new(Arc::new(dtype.clone())); let mut error = ptr::null_mut(); let out_ctx = Box::into_raw(Box::new(Arc::clone(&shared))); @@ -746,7 +739,7 @@ mod tests { let array = PrimitiveArray::new(buffer![10i32, 20i32, 30i32, 40i32], Validity::NonNullable); - let vx_array_ptr = vx_array::new(array.into_array()); + let vx_array_ptr = vx_array::new(Arc::new(array.into_array())); vx_callback_sink_push(sink, vx_array_ptr, &raw mut error); assert!(error.is_null()); vx_callback_sink_close(sink, &raw mut error); diff --git a/src/paimon/format/vortex/vortex_ffi_util.cpp b/src/paimon/format/vortex/vortex_ffi_util.cpp index 83e7c296..e67f35e5 100644 --- a/src/paimon/format/vortex/vortex_ffi_util.cpp +++ b/src/paimon/format/vortex/vortex_ffi_util.cpp @@ -25,9 +25,10 @@ Status VortexFfiError(const std::string& operation, vx_error* error) { if (error == nullptr) { return Status::OK(); } - vx_view message = vx_error_message(error); - std::string text = - message.ptr == nullptr ? std::string() : std::string(message.ptr, message.len); + const vx_string* message = vx_error_get_message(error); + const char* ptr = message == nullptr ? nullptr : vx_string_ptr(message); + const size_t len = message == nullptr ? 0 : vx_string_len(message); + std::string text = ptr == nullptr ? std::string() : std::string(ptr, len); vx_error_free(error); return Status::Invalid(operation, ": ", text); } diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.cpp b/src/paimon/format/vortex/vortex_file_batch_reader.cpp index 5079143d..96b08361 100644 --- a/src/paimon/format/vortex/vortex_file_batch_reader.cpp +++ b/src/paimon/format/vortex/vortex_file_batch_reader.cpp @@ -233,13 +233,19 @@ Result> VortexFileBatchReader::Create( } // The data source dtype is available without consuming a scan. - VxDtypePtr dtype(vx_data_source_dtype(data_source.get()), vx_dtype_free); + // NOTE(vortex 0.75): vx_data_source_dtype returns a BORROWED pointer (arc_wrapper new_ref: + // no refcount bump), so it must NOT be freed here; doing so spuriously decrements the data + // source's DType Arc and causes a use-after-free/segfault later in the scan. It stays valid + // as long as `data_source` lives, which outlives this schema conversion. (Upstream 0.77 + // changed vx_data_source_dtype to return an owned clone that MUST be freed; if the pin moves + // to >=0.77, wrap the result with vx_dtype_free again.) + const vx_dtype* dtype = vx_data_source_dtype(data_source.get()); if (dtype == nullptr) { return Status::IOError("failed to read Vortex data source dtype"); } ::ArrowSchema ffi_schema = {}; error = nullptr; - if (vx_dtype_to_arrow_schema(session.get(), dtype.get(), &ffi_schema, &error) != 0) { + if (vx_dtype_to_arrow_schema(dtype, &ffi_schema, &error) != 0) { return VortexFfiError("convert Vortex dtype to Arrow schema", error); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, diff --git a/src/paimon/format/vortex/vortex_format_writer.cpp b/src/paimon/format/vortex/vortex_format_writer.cpp index 5e56ae38..928ec845 100644 --- a/src/paimon/format/vortex/vortex_format_writer.cpp +++ b/src/paimon/format/vortex/vortex_format_writer.cpp @@ -53,7 +53,7 @@ Result> VortexFormatWriter::Create( PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &ffi_schema)); vx_error* error = nullptr; // vx_dtype_from_arrow_schema consumes ffi_schema on both success and failure. - VxDtypePtr dtype(vx_dtype_from_arrow_schema(session.get(), &ffi_schema, &error), vx_dtype_free); + VxDtypePtr dtype(vx_dtype_from_arrow_schema(&ffi_schema, &error), vx_dtype_free); if (dtype == nullptr) { return VortexFfiError("convert Arrow schema to Vortex dtype", error); } @@ -89,9 +89,8 @@ Status VortexFormatWriter::AddBatch(::ArrowArray* batch) { PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, &ffi_schema)); vx_error* error = nullptr; // vx_array_from_arrow consumes both `batch` and `ffi_schema` on success and on failure. - VxArrayPtr array( - vx_array_from_arrow(session_.get(), batch, &ffi_schema, /*nullable=*/false, &error), - vx_array_free); + VxArrayPtr array(vx_array_from_arrow(batch, &ffi_schema, /*nullable=*/false, &error), + vx_array_free); if (array == nullptr) { return VortexFfiError("convert Arrow batch to Vortex array", error); } diff --git a/third_party/versions.txt b/third_party/versions.txt index 8c145235..6a273b6b 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -72,12 +72,14 @@ PAIMON_MOSAIC_BUILD_VERSION=0.2.0 PAIMON_MOSAIC_BUILD_SHA256_CHECKSUM=8123eaadd293a7904b692eff900484691711dc0cb24878166a2f90a378d37a22 PAIMON_MOSAIC_PKG_NAME=apache-paimon-mosaic-${PAIMON_MOSAIC_BUILD_VERSION}-src.tgz -# Vortex has no Apache release and vortex-ffi is publish=false, so it is pinned to an upstream -# commit. The file format is back-compatible since vortex 0.36.0, so this reads files written by -# paimon-java's vortex-jni 0.73.0. The SHA256 is the self-computed checksum of the pinned commit's -# GitHub source archive (upstream publishes none); recompute it whenever the pin changes. -PAIMON_VORTEX_BUILD_VERSION=d82de0e1c8c46dc90a3b95a433a51668e283dfdc -PAIMON_VORTEX_BUILD_SHA256_CHECKSUM=af7dd32d1e92090293ca3bd70f5a3f97b00a5cec968a0c8e99c7612feeb35934 +# Vortex has no Apache release and vortex-ffi is publish=false, so it is pinned to an upstream tag. +# 0.75.0 is chosen for BIDIRECTIONAL Vortex interop with paimon-java's vortex-jni 0.73.0: 0.75 reads +# 0.73-written files (back-compatible since 0.36.0) and, empirically, 0.73 reads 0.75-written files +# (the 0.75 writer emits no encoding/layout unknown to 0.73). Newer pins (>=0.77) break that forward +# direction ("Encoding not found in registry"). The SHA256 is the self-computed checksum of the +# tag's GitHub source archive (upstream publishes none); recompute it whenever the pin changes. +PAIMON_VORTEX_BUILD_VERSION=0.75.0 +PAIMON_VORTEX_BUILD_SHA256_CHECKSUM=a0ceeed81e1b76dc9b591acde7d8e9ba96da78279a217fca12cd6c7c6b44b493 PAIMON_VORTEX_PKG_NAME=vortex-${PAIMON_VORTEX_BUILD_VERSION}.tar.gz PAIMON_AVRO_BUILD_VERSION=c499eefb48aa2db906c7bca14a047223806f36db From e0a18fef1333645dfb85c87035bb75abd95c7acd Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Tue, 22 Sep 2026 06:06:05 +0000 Subject: [PATCH 5/9] fix(format): recurse into LARGE_LIST/FIXED_SIZE_LIST in NormalizeViewArray NormalizeViewArray only recursed through STRING_VIEW/BINARY_VIEW/STRUCT/LIST, so a FIXED_SIZE_LIST (or LARGE_LIST) with view-typed elements fell into the default branch and was returned unchanged. NormalizeViewType already normalizes those element types, so the read schema declared e.g. fixed_size_list while the data stayed fixed_size_list (Vortex hardcodes Utf8 -> Utf8View on export). The ArrowSchema mismatch made callers reinterpret the view buffers as standard strings, silently corrupting data on read. Add LARGE_LIST and FIXED_SIZE_LIST branches symmetric to the LIST branch, including the unchanged-values fast path, reusing the source null bitmap, null count, offset, length and (for LARGE_LIST) value offsets. Covered by WriteThenReadListOfString, WriteThenReadFixedSizeListOfString and WriteThenReadListOfStruct, whose data includes nulls, empty strings and strings long enough to exceed the 12-byte inline view threshold. --- .../vortex/vortex_file_batch_reader.cpp | 24 +++++++ .../format/vortex/vortex_file_format_test.cpp | 63 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.cpp b/src/paimon/format/vortex/vortex_file_batch_reader.cpp index 96b08361..203f4f7b 100644 --- a/src/paimon/format/vortex/vortex_file_batch_reader.cpp +++ b/src/paimon/format/vortex/vortex_file_batch_reader.cpp @@ -149,6 +149,30 @@ Result> NormalizeViewArray(const std::shared_ptrtype()->field(0)->WithType(values->type())), list.length(), list.value_offsets(), values, list.null_bitmap(), list.null_count(), list.offset()); } + case arrow::Type::LARGE_LIST: { + const auto& list = checked_cast(*array); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr values, + NormalizeViewArray(list.values(), pool)); + if (values.get() == list.values().get()) { + return array; + } + return std::make_shared( + arrow::large_list(array->type()->field(0)->WithType(values->type())), list.length(), + list.value_offsets(), values, list.null_bitmap(), list.null_count(), list.offset()); + } + case arrow::Type::FIXED_SIZE_LIST: { + const auto& list = checked_cast(*array); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr values, + NormalizeViewArray(list.values(), pool)); + if (values.get() == list.values().get()) { + return array; + } + const auto& fsl_type = checked_cast(*array->type()); + return std::make_shared( + arrow::fixed_size_list(array->type()->field(0)->WithType(values->type()), + fsl_type.list_size()), + list.length(), values, list.null_bitmap(), list.null_count(), list.offset()); + } default: return array; } diff --git a/src/paimon/format/vortex/vortex_file_format_test.cpp b/src/paimon/format/vortex/vortex_file_format_test.cpp index b07e0402..889d8dc3 100644 --- a/src/paimon/format/vortex/vortex_file_format_test.cpp +++ b/src/paimon/format/vortex/vortex_file_format_test.cpp @@ -205,6 +205,69 @@ TEST_F(VortexFileFormatTest, WriteThenReadSupportedTypes) { AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5}); } +// Regression: a list column round-trips through Vortex's Utf8View export. The leaf strings +// include a null element, an empty list, a null list, an empty string and a string long enough to +// exceed the 12-byte StringView inline threshold (exercising the non-inline view buffer path). +TEST_F(VortexFileFormatTest, WriteThenReadListOfString) { + std::string path = PathUtil::JoinPath(directory_->Str(), "list-of-string.vortex"); + arrow::FieldVector fields = {arrow::field("tags", arrow::list(arrow::utf8()))}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [["one","two",null]], + [[]], + [null], + [["","a-longer-string-that-exceeds-the-inline-view-threshold","x"]], + [[null,null]] + ])") + .ValueOrDie(); + + ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2)); + AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 5}); +} + +// Regression for the FIXED_SIZE_LIST branch of NormalizeViewArray: without recursion into the +// elements, a fixed_size_list column comes back as fixed_size_list (Vortex +// hardcodes Utf8 -> Utf8View), mismatching the normalized read schema and silently corrupting the +// data. +TEST_F(VortexFileFormatTest, WriteThenReadFixedSizeListOfString) { + std::string path = PathUtil::JoinPath(directory_->Str(), "fsl-of-string.vortex"); + arrow::FieldVector fields = {arrow::field("triples", arrow::fixed_size_list(arrow::utf8(), 3))}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [["a","bb","ccc"]], + [null], + [["","a-longer-string-that-exceeds-the-inline-view-threshold","z"]], + [[null,"q",null]] + ])") + .ValueOrDie(); + + ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2)); + AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3, 4}); +} + +// Regression for a struct nested inside a list: the LIST branch recurses into the struct, whose +// utf8 leaf is a view; the combination must normalize back to list>. +TEST_F(VortexFileFormatTest, WriteThenReadListOfStruct) { + std::string path = PathUtil::JoinPath(directory_->Str(), "list-of-struct.vortex"); + std::shared_ptr item = + arrow::struct_({arrow::field("name", arrow::utf8()), arrow::field("id", arrow::int32())}); + arrow::FieldVector fields = {arrow::field("items", arrow::list(item))}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [[{"name":"a","id":1},{"name":null,"id":2}]], + [[]], + [null], + [[{"name":"a-longer-string-that-exceeds-the-inline-view-threshold","id":null}]] + ])") + .ValueOrDie(); + + ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2)); + AssertReadWithBatchSizes(path, schema, expected, {1, 2, 3}); +} + TEST_F(VortexFileFormatTest, ExtractStatisticsReportsRowCount) { std::string path = PathUtil::JoinPath(directory_->Str(), "statistics.vortex"); arrow::FieldVector fields = {arrow::field("id", arrow::int32()), From d6644fd4bf216d52acabf32c80c865eee2ba2881 Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Tue, 22 Sep 2026 07:20:40 +0000 Subject: [PATCH 6/9] fix(format): harden Vortex reader/writer per review Address four review findings plus a related write-path abort: - Scan in storage order. The scan used Vortex's default ordered=false, which may emit chunks out of order via buffer_unordered; NextBatch assigns physical row positions by batch order, so unordered chunks would misalign deletion vectors and primary-key merge. Set ScanOptions.ordered(true), matching paimon-java's VortexRecordsReader. - Recurse into nested fields when projecting to the read schema. ProjectToReadSchema only selected top-level columns, so reading a pruned nested field (e.g. r.b from r:ROW) kept the full child under a pruned parent type, mismatching the exported Arrow schema and silently corrupting the output. Project struct/list/fixed-size-list children recursively. - Fix a use-after-free on the stream error path. get_last_error returns a pointer owned by the ArrowArrayStream; it was read after ReleaseStream() invalidated it. Copy the message before releasing. - Rebase sliced batches to offset 0 before vx_array_from_arrow. arrow-rs cannot import a sliced (offset > 0) top-level struct (the parent offset is re-applied to already-offset children, tripping an arrow-data slice assertion that aborts). Offset-0 batches pass through untouched, so the hot path is unchanged. - Attribute the adapted sink entry points in callback_io.rs to vortex-ffi's sink.rs (Apache-2.0, Copyright the Vortex contributors) per Apache-2.0 section 4. Adds regression tests: nested-struct projection, row order across batches, and sliced top-level struct write/read. --- crates/vortex_callback_io/callback_io.rs | 5 + .../vortex/vortex_file_batch_reader.cpp | 117 +++++++++++++----- .../format/vortex/vortex_file_format_test.cpp | 74 +++++++++++ .../format/vortex/vortex_format_writer.cpp | 24 +++- 4 files changed, 190 insertions(+), 30 deletions(-) diff --git a/crates/vortex_callback_io/callback_io.rs b/crates/vortex_callback_io/callback_io.rs index e6487bcb..b0a45c98 100644 --- a/crates/vortex_callback_io/callback_io.rs +++ b/crates/vortex_callback_io/callback_io.rs @@ -31,6 +31,11 @@ //! The write side needs its own handle type rather than reusing `vx_array_sink`, whose fields are //! private to `crate::sink`. //! +//! The sink entry points below (`vx_callback_sink_push` / `_close` / `_abort`) are adapted from +//! vortex-ffi's `src/sink.rs` (SPDX-License-Identifier: Apache-2.0, Copyright the Vortex +//! contributors), reworked to drive the write pipeline through host callbacks instead of a +//! Vortex-resolved filesystem path. +//! //! This file is maintained in the paimon-cpp tree (`crates/vortex_callback_io/`) and copied into //! `vortex-ffi/src/` at build time; see `cmake_modules/vortex.diff`. //! diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.cpp b/src/paimon/format/vortex/vortex_file_batch_reader.cpp index 203f4f7b..11b877d9 100644 --- a/src/paimon/format/vortex/vortex_file_batch_reader.cpp +++ b/src/paimon/format/vortex/vortex_file_batch_reader.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include "arrow/api.h" @@ -178,34 +179,85 @@ Result> NormalizeViewArray(const std::shared_ptr> ProjectArrayToType( + const std::shared_ptr& array, + const std::shared_ptr& target_type) { + if (array->type()->Equals(target_type)) { + return array; + } + switch (target_type->id()) { + case arrow::Type::STRUCT: { + const auto& struct_array = checked_cast(*array); + const auto& struct_type = checked_cast(*array->type()); + const auto& target_struct = checked_cast(*target_type); + arrow::ArrayVector children; + children.reserve(target_struct.num_fields()); + for (const std::shared_ptr& target_field : target_struct.fields()) { + const int32_t index = struct_type.GetFieldIndex(target_field->name()); + if (index < 0) { + return Status::Invalid( + fmt::format("Vortex read field '{}' is not present in the file schema", + target_field->name())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr child, + ProjectArrayToType(struct_array.field(index), target_field->type())); + children.push_back(std::move(child)); + } + return std::make_shared( + target_type, struct_array.length(), children, struct_array.null_bitmap(), + struct_array.null_count(), struct_array.offset()); + } + case arrow::Type::LIST: { + const auto& list = checked_cast(*array); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr values, + ProjectArrayToType(list.values(), target_type->field(0)->type())); + return std::make_shared( + arrow::list(target_type->field(0)->WithType(values->type())), list.length(), + list.value_offsets(), values, list.null_bitmap(), list.null_count(), list.offset()); + } + case arrow::Type::LARGE_LIST: { + const auto& list = checked_cast(*array); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr values, + ProjectArrayToType(list.values(), target_type->field(0)->type())); + return std::make_shared( + arrow::large_list(target_type->field(0)->WithType(values->type())), list.length(), + list.value_offsets(), values, list.null_bitmap(), list.null_count(), list.offset()); + } + case arrow::Type::FIXED_SIZE_LIST: { + const auto& list = checked_cast(*array); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr values, + ProjectArrayToType(list.values(), target_type->field(0)->type())); + const auto& fsl_type = checked_cast(*target_type); + return std::make_shared( + arrow::fixed_size_list(target_type->field(0)->WithType(values->type()), + fsl_type.list_size()), + list.length(), values, list.null_bitmap(), list.null_count(), list.offset()); + } + default: + return array; + } +} + // Projects a struct array to the columns named by `read_schema`, selected by field name and ordered -// as in `read_schema`, so NextBatch returns exactly the read schema (the FileBatchReader contract; -// paimon's FieldMappingReader maps fields but does not re-project). Returns `array` unchanged when -// it already matches, or when `read_schema` is null (SetReadSchema not yet called). +// as in `read_schema` (recursing into nested fields), so NextBatch returns exactly the read schema +// (the FileBatchReader contract; paimon's FieldMappingReader maps fields but does not re-project). +// Returns `array` unchanged when it already matches, or when `read_schema` is null (SetReadSchema +// not yet called). Result> ProjectToReadSchema( const std::shared_ptr& array, const std::shared_ptr& read_schema) { if (read_schema == nullptr) { return array; } - const std::shared_ptr target_type = arrow::struct_(read_schema->fields()); - if (array->type()->Equals(target_type)) { - return array; - } - const auto& struct_array = checked_cast(*array); - const auto& struct_type = checked_cast(*array->type()); - arrow::ArrayVector children; - children.reserve(read_schema->num_fields()); - for (const std::shared_ptr& read_field : read_schema->fields()) { - const int32_t index = struct_type.GetFieldIndex(read_field->name()); - if (index < 0) { - return Status::Invalid(fmt::format( - "Vortex read field '{}' is not present in the file schema", read_field->name())); - } - children.push_back(struct_array.field(index)); - } - return std::make_shared(target_type, struct_array.length(), children, - struct_array.null_bitmap(), - struct_array.null_count(), struct_array.offset()); + return ProjectArrayToType(array, arrow::struct_(read_schema->fields())); } } // namespace @@ -300,10 +352,17 @@ VortexFileBatchReader::~VortexFileBatchReader() { Result VortexFileBatchReader::OpenNextPartitionStream() { if (scan_ == nullptr) { vx_error* error = nullptr; - // NULL options: scan all rows and columns (no projection/predicate pushdown). - VxScanPtr scan(vx_data_source_scan(data_source_.get(), /*options=*/nullptr, - /*estimate=*/nullptr, &error), - vx_scan_free); + // Scan all rows and columns (Step 1 does no projection/predicate pushdown), but require + // storage order. Vortex defaults to ordered=false and may emit chunks out of order via + // buffer_unordered; NextBatch assigns physical row positions by batch order + // (rows_emitted_), so unordered chunks would misalign deletion vectors and primary-key + // merge. paimon-java's VortexRecordsReader likewise builds its scan with + // ScanOptions.ordered(true). + vx_scan_options options{}; // zero-init: all columns, no filter/row-range/selection/limit + options.ordered = true; + VxScanPtr scan( + vx_data_source_scan(data_source_.get(), &options, /*estimate=*/nullptr, &error), + vx_scan_free); if (scan == nullptr) { return VortexCallbackError("create Vortex scan", error, input_context_->GetCallbackStatus()); @@ -342,7 +401,10 @@ Result> VortexFileBatchReader::ReadNextArray() { ::ArrowArray ffi_array = {}; const int32_t rc = current_stream_.get_next(¤t_stream_, &ffi_array); if (rc != 0) { + // get_last_error returns a pointer owned by the stream; the Arrow C stream contract + // invalidates it once the stream is released, so copy it before ReleaseStream(). const char* message = current_stream_.get_last_error(¤t_stream_); + std::string message_copy = message == nullptr ? "unknown" : message; // A read failure surfaces here as an opaque stream error, so prefer the status the IO // callback stashed. Status callback_status = input_context_->GetCallbackStatus(); @@ -351,8 +413,7 @@ Result> VortexFileBatchReader::ReadNextArray() { return callback_status.WithMessage("read Vortex batch", ": ", callback_status.message()); } - return Status::IOError("Vortex Arrow stream error: ", - message == nullptr ? "unknown" : message); + return Status::IOError("Vortex Arrow stream error: ", message_copy); } if (ffi_array.release == nullptr) { ReleaseStream(); // Partition exhausted; try the next one. diff --git a/src/paimon/format/vortex/vortex_file_format_test.cpp b/src/paimon/format/vortex/vortex_file_format_test.cpp index 889d8dc3..6e85e70f 100644 --- a/src/paimon/format/vortex/vortex_file_format_test.cpp +++ b/src/paimon/format/vortex/vortex_file_format_test.cpp @@ -313,6 +313,80 @@ TEST_F(VortexFileFormatTest, ProjectedReadReturnsOnlyRequestedColumns) { ASSERT_TRUE(actual->Equals(expected)) << actual->ToString() << "\nvs\n" << expected->ToString(); } +// Regression: nested projection must prune sub-fields, not merely relabel the parent type. Reading +// only r.b from a file whose r is ROW must yield r as ROW with matching data; before the +// recursive fix the full ROW child was kept under a ROW type, so the exported ArrowSchema +// disagreed with the array layout and silently corrupted the output. The b column also exercises +// view normalization inside a nested struct. +TEST_F(VortexFileFormatTest, ProjectedReadPrunesNestedStruct) { + std::string path = PathUtil::JoinPath(directory_->Str(), "nested-projection.vortex"); + std::shared_ptr r_type = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false), + arrow::field("r", r_type)}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr written = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(fields), + R"([[1,{"a":10,"b":"x"}],[2,{"a":20,"b":null}],[3,{"a":30,"b":"z"}]])") + .ValueOrDie(); + // batch_size=2 also slices the top-level struct column on write (offset > 0), exercising the + // writer's offset rebasing together with the nested-projection read fix. + ASSERT_OK(WriteFile(path, schema, written, /*batch_size=*/2)); + + // Read only r.b: drop the top-level id column and prune r.a. + std::shared_ptr r_pruned = arrow::struct_({arrow::field("b", arrow::utf8())}); + arrow::FieldVector projected_fields = {arrow::field("r", r_pruned)}; + std::shared_ptr projected = arrow::schema(projected_fields); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(projected_fields), + R"([[{"b":"x"}],[{"b":null}],[{"b":"z"}]])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + ReadFile(path, projected, /*batch_size=*/2, /*expected_row_count=*/3)); + ASSERT_TRUE(actual->Equals(expected)) << actual->ToString() << "\nvs\n" << expected->ToString(); +} + +// Regression guard for ordered scans: 200 sequential rows span many Vortex chunks, and a small read +// batch size forces many batches. With ScanOptions.ordered(true) the rows must come back in storage +// order, so the concatenated result equals the written 0..199 sequence exactly (physical row +// positions, which the upper layer assigns by batch order, stay aligned for deletion vectors and +// primary-key merge). +TEST_F(VortexFileFormatTest, ReadPreservesRowOrderAcrossBatches) { + std::string path = PathUtil::JoinPath(directory_->Str(), "row-order.vortex"); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false)}; + std::shared_ptr schema = arrow::schema(fields); + std::string json = "["; + for (int32_t i = 0; i < 200; ++i) { + json += (i > 0 ? "," : "") + std::string("[") + std::to_string(i) + "]"; + } + json += "]"; + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json).ValueOrDie(); + ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/16)); + AssertReadWithBatchSizes(path, schema, expected, {1, 7, 64, 200}); +} + +// Regression: writing a top-level struct column in slices must not abort. arrow-rs cannot import a +// sliced (offset > 0) top-level struct (the parent offset is re-applied to already-offset children, +// tripping an arrow-data slice assertion), so VortexFormatWriter::AddBatch rebases such a batch to +// offset 0 before handing it to Vortex. batch_size=2 over 4 rows produces a slice at offset 2. +TEST_F(VortexFileFormatTest, WriteThenReadSlicedStructColumn) { + std::string path = PathUtil::JoinPath(directory_->Str(), "sliced-struct.vortex"); + std::shared_ptr r_type = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false), + arrow::field("r", r_type)}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr expected = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(fields), + R"([[1,{"a":10,"b":"x"}],[2,{"a":20,"b":null}],[3,{"a":30,"b":"z"}],[4,{"a":40,"b":"w"}]])") + .ValueOrDie(); + ASSERT_OK(WriteFile(path, schema, expected, /*batch_size=*/2)); + AssertReadWithBatchSizes(path, schema, expected, {1, 2, 4}); +} + // The IO callback bridge must surface a paimon IO failure as that paimon error, not Vortex's // opaque "stream error": the callback can only hand a status code back across the FFI boundary, so // the reader/writer stashes the real error and prefers it. `IOHook` injects a failure at the Nth diff --git a/src/paimon/format/vortex/vortex_format_writer.cpp b/src/paimon/format/vortex/vortex_format_writer.cpp index 928ec845..35379873 100644 --- a/src/paimon/format/vortex/vortex_format_writer.cpp +++ b/src/paimon/format/vortex/vortex_format_writer.cpp @@ -22,7 +22,10 @@ #include #include "arrow/c/bridge.h" +#include "arrow/memory_pool.h" +#include "arrow/type.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/vortex/vortex_ffi_util.h" #include "paimon/fs/file_system.h" @@ -85,11 +88,28 @@ Status VortexFormatWriter::AddBatch(::ArrowArray* batch) { if (finished_ || sink_ == nullptr) { return Status::Invalid("cannot add a batch after Vortex writer is finished"); } + // vx_array_from_arrow imports through arrow-rs, which cannot represent a sliced (offset > 0) + // top-level struct: the parent offset gets re-applied to children that Arrow C++ already + // exported with it, tripping an arrow-data slice assertion (end <= len) that aborts the + // process. Rebase such a batch to offset 0 first, mirroring the read path. Offset-0 batches + // (the common case) are handed over untouched, so the hot path adds no copy. + ::ArrowArray* vortex_batch = batch; + ::ArrowArray rebased_batch{}; + if (batch->offset != 0) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr imported, + arrow::ImportArray(batch, arrow::struct_(schema_->fields()))); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr normalized, + ArrowUtils::NormalizeArrayOffsets(imported, arrow::default_memory_pool())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*normalized, &rebased_batch)); + vortex_batch = &rebased_batch; + } ::ArrowSchema ffi_schema = {}; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema_, &ffi_schema)); vx_error* error = nullptr; - // vx_array_from_arrow consumes both `batch` and `ffi_schema` on success and on failure. - VxArrayPtr array(vx_array_from_arrow(batch, &ffi_schema, /*nullable=*/false, &error), + // vx_array_from_arrow consumes both `vortex_batch` and `ffi_schema` on success and on failure. + VxArrayPtr array(vx_array_from_arrow(vortex_batch, &ffi_schema, /*nullable=*/false, &error), vx_array_free); if (array == nullptr) { return VortexFfiError("convert Arrow batch to Vortex array", error); From 0b5fdff921c3734fb5675145d142d80630bb0d12 Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Tue, 22 Sep 2026 09:11:42 +0000 Subject: [PATCH 7/9] fix(ci): clear clang-tidy warnings and suppress un-instrumented Vortex TSAN races The Debug/TSAN matrix jobs failed the check-clang-tidy gate and the TSAN race gate on the Vortex files: - clang-tidy (modernize-use-using / modernize-use-auto): convert the hand-written C-ABI typedefs in vortex_ffi.h to `using` aliases / plain structs, and use `auto` for the cast-initialized locals in vortex_io_callbacks.cpp and vortex_stats_extractor.cpp. - TSAN: Vortex deserializes its footer/dtype and drives segment reads from internal runtime threads inside the un-instrumented Rust staticlib, so TSAN cannot see the happens-before edges and reports false races whose only non-paimon frames are the vortex_array/vortex_error/vortex_file/vortex_layout/vortex_mask Rust modules. Suppress those module-scoped patterns (they match Rust frames only, so races confined to paimon C++ are still reported), following the existing vortex_buffer/vortex_io/CallbackReadAt precedent. --- build_support/tsan-suppressions.txt | 15 ++++++++++++ src/paimon/format/vortex/vortex_ffi.h | 24 +++++++++---------- .../format/vortex/vortex_io_callbacks.cpp | 6 ++--- .../format/vortex/vortex_stats_extractor.cpp | 2 +- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/build_support/tsan-suppressions.txt b/build_support/tsan-suppressions.txt index 53783cab..8269e2fb 100644 --- a/build_support/tsan-suppressions.txt +++ b/build_support/tsan-suppressions.txt @@ -26,3 +26,18 @@ race:libjindosdk_c.so.6 race:vortex_buffer race:vortex_io race:CallbackReadAt +# Vortex also deserializes its footer/dtype and drives segment reads from internal runtime threads; +# that concurrency lives inside the same un-instrumented Rust staticlib (dtype flatbuffers decode, +# shared-future polling, segment cache and mask handling), so TSAN reports false races whose only +# non-paimon frames are these Rust modules. Patterns match Rust frames only. +race:vortex_array +race:vortex_error +race:vortex_file +race:vortex_layout +race:vortex_mask +# The Arrow C-bridge hands buffer ownership to/from the Rust side (arrow_buffer Allocation and +# bytes::Bytes), and Vortex formats its error messages on its runtime threads; those +# synchronizations also live in the un-instrumented Rust staticlib, so suppress those Rust frames. +race:arrow_buffer +race:bytes::bytes::Bytes +race:alloc::string::String diff --git a/src/paimon/format/vortex/vortex_ffi.h b/src/paimon/format/vortex/vortex_ffi.h index 9eba29ef..58f6584f 100644 --- a/src/paimon/format/vortex/vortex_ffi.h +++ b/src/paimon/format/vortex/vortex_ffi.h @@ -37,9 +37,9 @@ // vortex.h's API refers to the Arrow C-data-interface structs via these typedef // names, which it skips defining under USE_OWN_ARROW. Provide them from Arrow's // definitions, exactly as vortex.h's own header comment instructs. -typedef struct ArrowSchema FFI_ArrowSchema; -typedef struct ArrowArray FFI_ArrowArray; -typedef struct ArrowArrayStream FFI_ArrowArrayStream; +using FFI_ArrowSchema = struct ArrowSchema; +using FFI_ArrowArray = struct ArrowArray; +using FFI_ArrowArrayStream = struct ArrowArrayStream; extern "C" { #include "vortex.h" // NOLINT(build/include_subdir) @@ -57,21 +57,21 @@ extern "C" { /// boundary, so the callback is expected to keep its error detail on `ctx`. /// /// Must be thread-safe: Vortex issues concurrent positional reads for the same context. -typedef int32_t (*vx_read_at_fn)(void* ctx, uint64_t offset, uint8_t* dst, size_t length); +using vx_read_at_fn = int32_t (*)(void* ctx, uint64_t offset, uint8_t* dst, size_t length); /// Release `ctx`. Called exactly once, when the owning data source is freed (including when it /// fails to open). -typedef void (*vx_release_fn)(void* ctx); +using vx_release_fn = void (*)(void* ctx); /// Host callbacks backing a positional reader. -typedef struct vx_input_callbacks { +struct vx_input_callbacks { /// Opaque host context, passed back to every callback. void* ctx; /// Positional read. Required. vx_read_at_fn read_at_fn; /// Context destructor. Optional; may be null. vx_release_fn release_fn; -} vx_input_callbacks; +}; /// Create a data source that reads through `callbacks`. `size` is the total file length in bytes. /// Returns null and sets `err` on failure. @@ -82,13 +82,13 @@ const vx_data_source* vx_data_source_new_callback(const vx_session* session, /// Append `length` bytes from `src` to the host sink. Returns 0 on success and non-zero on failure; /// a partial write must be reported as a failure. Writes for one sink are sequential, never /// concurrent. -typedef int32_t (*vx_write_fn)(void* ctx, const uint8_t* src, size_t length); +using vx_write_fn = int32_t (*)(void* ctx, const uint8_t* src, size_t length); /// Flush whatever the host has buffered. Returns 0 on success and non-zero on failure. -typedef int32_t (*vx_flush_fn)(void* ctx); +using vx_flush_fn = int32_t (*)(void* ctx); /// Host callbacks backing a sequential writer. -typedef struct vx_output_callbacks { +struct vx_output_callbacks { /// Opaque host context, passed back to every callback. void* ctx; /// Sequential write. Required. @@ -97,11 +97,11 @@ typedef struct vx_output_callbacks { vx_flush_fn flush_fn; /// Context destructor. Optional; may be null. vx_release_fn release_fn; -} vx_output_callbacks; +}; /// A sink writing a Vortex file through host callbacks. Mirrors `vx_array_sink`, which can only /// target a local filesystem path. -typedef struct vx_callback_sink vx_callback_sink; +struct vx_callback_sink; /// Open a sink writing through `callbacks`. Returns null and sets `err` on failure. Write errors /// are reported by `vx_callback_sink_close`, since the bytes are produced by a background task. diff --git a/src/paimon/format/vortex/vortex_io_callbacks.cpp b/src/paimon/format/vortex/vortex_io_callbacks.cpp index e80d2bbd..ca05988b 100644 --- a/src/paimon/format/vortex/vortex_io_callbacks.cpp +++ b/src/paimon/format/vortex/vortex_io_callbacks.cpp @@ -75,7 +75,7 @@ Status VortexInputContext::GetCallbackStatus() const { int32_t VortexInputContext::ReadAt(void* ctx, uint64_t offset, uint8_t* dst, size_t length) noexcept { - VortexInputContext* context = ContextFrom(ctx); + auto* context = ContextFrom(ctx); if (context == nullptr || dst == nullptr) { if (context != nullptr) { context->SetCallbackStatus(Status::Invalid("invalid Vortex read request")); @@ -158,7 +158,7 @@ int64_t VortexOutputContext::BytesWritten() const { } int32_t VortexOutputContext::Write(void* ctx, const uint8_t* src, size_t length) noexcept { - VortexOutputContext* context = ContextFrom(ctx); + auto* context = ContextFrom(ctx); if (context == nullptr || src == nullptr) { if (context != nullptr) { context->SetCallbackStatus(Status::Invalid("invalid Vortex write request")); @@ -190,7 +190,7 @@ int32_t VortexOutputContext::Write(void* ctx, const uint8_t* src, size_t length) } int32_t VortexOutputContext::Flush(void* ctx) noexcept { - VortexOutputContext* context = ContextFrom(ctx); + auto* context = ContextFrom(ctx); if (context == nullptr) { return -1; } diff --git a/src/paimon/format/vortex/vortex_stats_extractor.cpp b/src/paimon/format/vortex/vortex_stats_extractor.cpp index e7a3f32c..c4a20b85 100644 --- a/src/paimon/format/vortex/vortex_stats_extractor.cpp +++ b/src/paimon/format/vortex/vortex_stats_extractor.cpp @@ -66,7 +66,7 @@ VortexStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& fil if (row_count.type == VX_ESTIMATE_UNKNOWN) { return Status::Invalid("Vortex file did not report a row count"); } - const int64_t rows = static_cast(row_count.estimate); + const auto rows = static_cast(row_count.estimate); // Empty column stats: Vortex keeps its statistics internal and does not surface them to Paimon. return std::make_pair(ColumnStatsVector(), FileInfo(rows)); } From 0d744686f2b796e46fae10ad45fe34cc698fe088 Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Tue, 22 Sep 2026 13:21:10 +0000 Subject: [PATCH 8/9] fix(format): address second-round review (flush serialization, pool, validation, CI deps) - Serialize host OutputStream access between the Vortex writer task and a caller-thread Flush(): VortexOutputContext gains a stream mutex and a FlushStream() entry, and VortexFormatWriter::Flush() plus the write/flush callbacks all take it, so Write and Flush can no longer interleave on a stream with no concurrent-access contract. The remaining internally buffered data is drained when the sink closes in Finish(). - Thread the caller's Arrow memory pool into VortexFormatWriter (Create now takes it and the builder passes GetArrowPool(pool_)), so sliced-batch rebasing allocates from the Paimon pool instead of Arrow's default pool. - Validate Vortex-incompatible schema types wherever Vortex can be selected (default file format, file.format-per-level, changelog-file.format), mirroring the Lance validation, instead of only the default format. - setup_flatc.sh: install libclang before the flatc early-exit, so an environment that already has a matching flatc but lacks libclang no longer silently skips the bindgen build dependency. - Tests: FlushBetweenBatchesThenRead (flush interleaved with background writes) and SchemaValidationTest.TestVortexDataTypes (per-level/changelog validation). --- ci/scripts/setup_flatc.sh | 24 ++++++------- src/paimon/core/schema/schema_validation.cpp | 36 +++++++++++++------ .../core/schema/schema_validation_test.cpp | 31 ++++++++++++++++ .../format/vortex/vortex_file_format_test.cpp | 34 ++++++++++++++++++ .../format/vortex/vortex_format_writer.cpp | 24 +++++++------ .../format/vortex/vortex_format_writer.h | 9 +++-- .../format/vortex/vortex_io_callbacks.cpp | 15 ++++++-- .../format/vortex/vortex_io_callbacks.h | 14 ++++++++ .../format/vortex/vortex_writer_builder.cpp | 3 +- 9 files changed, 151 insertions(+), 39 deletions(-) diff --git a/ci/scripts/setup_flatc.sh b/ci/scripts/setup_flatc.sh index 47e8df8a..ce8981f2 100755 --- a/ci/scripts/setup_flatc.sh +++ b/ci/scripts/setup_flatc.sh @@ -36,6 +36,18 @@ set -eux FLATBUFFERS_VERSION=${FLATBUFFERS_VERSION:-25.12.19} FLATC_INSTALL_DIR=${FLATC_INSTALL_DIR:-"${HOME}/.local/bin"} +# Vortex's native dependencies invoke bindgen in their build scripts, which loads libclang at +# build time. Images that only carry GCC (e.g. the gcc8 test container) may lack it. +if ! ldconfig -p 2>/dev/null | grep -q 'libclang'; then + if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1; then + apt_prefix="sudo" + else + apt_prefix="" + fi + ${apt_prefix} apt-get update -y + ${apt_prefix} apt-get install -y libclang-dev +fi + # Skip when a matching flatc is already available. if command -v flatc >/dev/null 2>&1 && [[ "$(flatc --version 2>/dev/null | awk '{print $NF}')" == "${FLATBUFFERS_VERSION}" ]]; then @@ -61,16 +73,4 @@ export PATH="${FLATC_INSTALL_DIR}:${PATH}" echo "${FLATC_INSTALL_DIR}" >>"${GITHUB_PATH:-/dev/null}" || true echo "FLATC=${FLATC_INSTALL_DIR}/flatc" >>"${GITHUB_ENV:-/dev/null}" || true -# Vortex's native dependencies invoke bindgen in their build scripts, which loads libclang at -# build time. Images that only carry GCC (e.g. the gcc8 test container) may lack it. -if ! ldconfig -p 2>/dev/null | grep -q 'libclang'; then - if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1; then - apt_prefix="sudo" - else - apt_prefix="" - fi - ${apt_prefix} apt-get update -y - ${apt_prefix} apt-get install -y libclang-dev -fi - "${FLATC_INSTALL_DIR}/flatc" --version diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index b3284fb3..d1a589a2 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -935,21 +935,35 @@ Status SchemaValidation::ValidateVortexDataField(const std::shared_ptrIdentifier()) != "vortex") { - return Status::OK(); - } - const std::vector inline_blob_fields = options.GetBlobInlineFields(); const std::set inline_blob_field_set(inline_blob_fields.begin(), inline_blob_fields.end()); - // Mirror the Mosaic path: only validate fields stored in the normal data file. A non-inline - // BLOB lives in a separate blob file, so it is skipped here; an inline BLOB is rejected. - for (const DataField& field : schema.Fields()) { - if (BlobUtils::IsBlobField(field.ArrowField()) && - inline_blob_field_set.count(field.Name()) == 0) { - continue; + // Mirror the Lance path: Vortex can be selected not only as the default file format but also + // per level or for the changelog, so validate the schema against every place it can be chosen. + auto validate_format = [&](const std::string&, const std::string& file_format) -> Status { + if (!StringUtils::EqualsIgnoreCase(file_format, "vortex")) { + return Status::OK(); + } + // Mirror the Mosaic path: only validate fields stored in the normal data file. A non-inline + // BLOB lives in a separate blob file, so it is skipped here; an inline BLOB is rejected. + for (const DataField& field : schema.Fields()) { + if (BlobUtils::IsBlobField(field.ArrowField()) && + inline_blob_field_set.count(field.Name()) == 0) { + continue; + } + PAIMON_RETURN_NOT_OK(ValidateVortexDataField(field.ArrowField())); } - PAIMON_RETURN_NOT_OK(ValidateVortexDataField(field.ArrowField())); + return Status::OK(); + }; + + PAIMON_RETURN_NOT_OK( + validate_format(Options::FILE_FORMAT, options.GetFileFormat()->Identifier())); + PAIMON_RETURN_NOT_OK( + ValidatePerLevelOption(options.ToMap(), Options::FILE_FORMAT_PER_LEVEL, validate_format)); + std::shared_ptr changelog_format = options.GetChangelogFileFormat(); + if (changelog_format) { + PAIMON_RETURN_NOT_OK( + validate_format(Options::CHANGELOG_FILE_FORMAT, changelog_format->Identifier())); } return Status::OK(); } diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index f6c08b86..747d8038 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -332,6 +332,37 @@ TEST(SchemaValidationTest, TestLanceDataTypes) { } #endif +#ifdef PAIMON_ENABLE_VORTEX +TEST(SchemaValidationTest, TestVortexDataTypes) { + std::shared_ptr table_schema; + // Vortex as the default file format: unsupported types are rejected at validation time. + std::map vortex_options = {{Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, "vortex"}}; + ASSERT_OK_AND_ASSIGN( + table_schema, + TableSchema::Create( + /*schema_id=*/0, + arrow::schema({arrow::field("map", arrow::map(arrow::int32(), arrow::utf8()))}), + /*partition_keys=*/{}, /*primary_keys=*/{}, vortex_options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "type MAP"); + + // Vortex selected per level or for the changelog must be validated too, not only the default. + for (const auto& [option_key, option_value] : std::vector>{ + {Options::FILE_FORMAT_PER_LEVEL, "1:vortex"}, + {Options::CHANGELOG_FILE_FORMAT, "vortex"}}) { + std::map alternate_format_options = { + {Options::BUCKET, "-1"}, {Options::FILE_FORMAT, "parquet"}, {option_key, option_value}}; + ASSERT_OK_AND_ASSIGN( + table_schema, + TableSchema::Create( + /*schema_id=*/0, + arrow::schema({arrow::field("map", arrow::map(arrow::int32(), arrow::utf8()))}), + /*partition_keys=*/{}, /*primary_keys=*/{}, alternate_format_options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "type MAP"); + } +} +#endif + TEST(SchemaValidationTest, TestRowTracking) { auto f0 = arrow::field("f0", arrow::utf8()); auto f1 = arrow::field("f1", arrow::int32()); diff --git a/src/paimon/format/vortex/vortex_file_format_test.cpp b/src/paimon/format/vortex/vortex_file_format_test.cpp index 6e85e70f..44e97a64 100644 --- a/src/paimon/format/vortex/vortex_file_format_test.cpp +++ b/src/paimon/format/vortex/vortex_file_format_test.cpp @@ -387,6 +387,40 @@ TEST_F(VortexFileFormatTest, WriteThenReadSlicedStructColumn) { AssertReadWithBatchSizes(path, schema, expected, {1, 2, 4}); } +// Regression: Flush() between batches must be safe (it is serialized with the background writer +// task's writes on the host OutputStream) and must not lose or reorder any data. +TEST_F(VortexFileFormatTest, FlushBetweenBatchesThenRead) { + std::string path = PathUtil::JoinPath(directory_->Str(), "flush-between.vortex"); + arrow::FieldVector fields = {arrow::field("id", arrow::int32(), false), + arrow::field("name", arrow::utf8())}; + std::shared_ptr schema = arrow::schema(fields); + std::shared_ptr written = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([[1,"one"],[2,null],[3,"three"],[4,"four"]])") + .ValueOrDie(); + + ::ArrowSchema ffi_schema = {}; + ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer_builder, + format_->CreateWriterBuilder(&ffi_schema, /*batch_size=*/2)); + writer_builder->WithMemoryPool(pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr output, + file_system_->Create(path, /*overwrite=*/false)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + writer_builder->Build(output, "zstd")); + for (int64_t offset = 0; offset < written->length(); offset += 2) { + std::shared_ptr slice = written->Slice(offset, 2); + ::ArrowArray ffi_array = {}; + ASSERT_TRUE(arrow::ExportArray(*slice, &ffi_array).ok()); + ASSERT_OK(writer->AddBatch(&ffi_array)); + ASSERT_OK(writer->Flush()); + } + ASSERT_OK(writer->Finish()); + ASSERT_OK(output->Close()); + + AssertReadWithBatchSizes(path, schema, written, {1, 2, 4}); +} + // The IO callback bridge must surface a paimon IO failure as that paimon error, not Vortex's // opaque "stream error": the callback can only hand a status code back across the FFI boundary, so // the reader/writer stashes the real error and prefers it. `IOHook` injects a failure at the Nth diff --git a/src/paimon/format/vortex/vortex_format_writer.cpp b/src/paimon/format/vortex/vortex_format_writer.cpp index 35379873..cba3fbb0 100644 --- a/src/paimon/format/vortex/vortex_format_writer.cpp +++ b/src/paimon/format/vortex/vortex_format_writer.cpp @@ -35,18 +35,21 @@ namespace paimon::vortex { VortexFormatWriter::VortexFormatWriter(std::shared_ptr output, std::shared_ptr schema, VxSessionPtr session, std::shared_ptr output_context, - vx_callback_sink* sink) + vx_callback_sink* sink, + std::shared_ptr arrow_pool) : output_(std::move(output)), schema_(std::move(schema)), + arrow_pool_(std::move(arrow_pool)), session_(std::move(session)), output_context_(std::move(output_context)), sink_(sink), metrics_(std::make_shared()) {} Result> VortexFormatWriter::Create( - const std::shared_ptr& output, const std::shared_ptr& schema) { - if (output == nullptr || schema == nullptr) { - return Status::Invalid("Vortex writer requires non-null output and schema"); + const std::shared_ptr& output, const std::shared_ptr& schema, + const std::shared_ptr& arrow_pool) { + if (output == nullptr || schema == nullptr || arrow_pool == nullptr) { + return Status::Invalid("Vortex writer requires non-null output, schema and arrow pool"); } VxSessionPtr session(vx_session_new(), vx_session_free); if (session == nullptr) { @@ -69,7 +72,7 @@ Result> VortexFormatWriter::Create( output_context->GetCallbackStatus()); } return std::unique_ptr(new VortexFormatWriter( - output, schema, std::move(session), std::move(output_context), sink)); + output, schema, std::move(session), std::move(output_context), sink, arrow_pool)); } VortexFormatWriter::~VortexFormatWriter() { @@ -99,9 +102,8 @@ Status VortexFormatWriter::AddBatch(::ArrowArray* batch) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::shared_ptr imported, arrow::ImportArray(batch, arrow::struct_(schema_->fields()))); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr normalized, - ArrowUtils::NormalizeArrayOffsets(imported, arrow::default_memory_pool())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized, + ArrowUtils::NormalizeArrayOffsets(imported, arrow_pool_.get())); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*normalized, &rebased_batch)); vortex_batch = &rebased_batch; } @@ -127,8 +129,10 @@ Status VortexFormatWriter::Flush() { if (finished_) { return Status::OK(); } - // Vortex buffers internally, so this only flushes what its sink has already handed over. - return output_->Flush(); + // Flush through the output context so it is serialized with the background writer task's writes + // (OutputStream has no concurrent Write/Flush contract). Vortex buffers internally, so this + // flushes the bytes handed over so far; the rest is drained when the sink closes in Finish(). + return output_context_->FlushStream(); } Status VortexFormatWriter::Finish() { diff --git a/src/paimon/format/vortex/vortex_format_writer.h b/src/paimon/format/vortex/vortex_format_writer.h index 4df57405..cbff524e 100644 --- a/src/paimon/format/vortex/vortex_format_writer.h +++ b/src/paimon/format/vortex/vortex_format_writer.h @@ -30,6 +30,7 @@ #include "paimon/result.h" namespace arrow { +class MemoryPool; class Schema; } // namespace arrow namespace paimon { @@ -48,7 +49,8 @@ namespace paimon::vortex { class VortexFormatWriter : public FormatWriter { public: static Result> Create( - const std::shared_ptr& output, const std::shared_ptr& schema); + const std::shared_ptr& output, const std::shared_ptr& schema, + const std::shared_ptr& arrow_pool); ~VortexFormatWriter() override; @@ -62,10 +64,13 @@ class VortexFormatWriter : public FormatWriter { private: VortexFormatWriter(std::shared_ptr output, std::shared_ptr schema, VxSessionPtr session, std::shared_ptr output_context, - vx_callback_sink* sink); + vx_callback_sink* sink, std::shared_ptr arrow_pool); std::shared_ptr output_; std::shared_ptr schema_; + // The caller's Arrow pool, used for any buffer (re)allocation on the write path so it counts + // against Paimon's memory accounting instead of Arrow's default pool. + std::shared_ptr arrow_pool_; VxSessionPtr session_; std::shared_ptr output_context_; // Raw pointer on purpose: close (in Finish) and abort (in the destructor) are mutually diff --git a/src/paimon/format/vortex/vortex_io_callbacks.cpp b/src/paimon/format/vortex/vortex_io_callbacks.cpp index ca05988b..90b5ceac 100644 --- a/src/paimon/format/vortex/vortex_io_callbacks.cpp +++ b/src/paimon/format/vortex/vortex_io_callbacks.cpp @@ -171,8 +171,12 @@ int32_t VortexOutputContext::Write(void* ctx, const uint8_t* src, size_t length) return -1; } auto write_length = static_cast(length); - Result result = - context->output_->Write(reinterpret_cast(src), write_length); + // The Vortex writer task is the only other accessor of `output_`; hold the stream lock so a + // concurrent caller-thread FlushStream cannot interleave with this write. + Result result = [&] { + std::lock_guard stream_lock(context->stream_mutex_); + return context->output_->Write(reinterpret_cast(src), write_length); + }(); if (!result.ok()) { context->SetCallbackStatus(result.status()); return -1; @@ -194,7 +198,7 @@ int32_t VortexOutputContext::Flush(void* ctx) noexcept { if (context == nullptr) { return -1; } - Status status = context->output_->Flush(); + Status status = context->FlushStream(); if (!status.ok()) { context->SetCallbackStatus(status); return -1; @@ -202,4 +206,9 @@ int32_t VortexOutputContext::Flush(void* ctx) noexcept { return 0; } +Status VortexOutputContext::FlushStream() { + std::lock_guard stream_lock(stream_mutex_); + return output_->Flush(); +} + } // namespace paimon::vortex diff --git a/src/paimon/format/vortex/vortex_io_callbacks.h b/src/paimon/format/vortex/vortex_io_callbacks.h index 7e066e11..c52f96e7 100644 --- a/src/paimon/format/vortex/vortex_io_callbacks.h +++ b/src/paimon/format/vortex/vortex_io_callbacks.h @@ -143,11 +143,25 @@ class VortexOutputContext { /// @return The number of bytes written to the paimon stream so far. int64_t BytesWritten() const; + /// Flush the paimon stream, serialized with the background writer task's writes. + /// + /// `OutputStream` carries no thread-safety contract for concurrent `Write`/`Flush`, and + /// Vortex's writer task invokes the write callback from its own thread, so a caller-thread + /// flush must take the same lock the write callback holds. Vortex buffers internally, so this + /// flushes the bytes handed over so far; whatever is still buffered is drained when the sink is + /// closed (`Finish`). + /// + /// @return OK on success, or the flush error. + Status FlushStream(); + private: void SetCallbackStatus(const Status& status); std::shared_ptr output_; mutable std::mutex mutex_; + // Serializes access to `output_` between the Vortex writer task (write/flush callbacks) and a + // caller-thread `FlushStream`; `mutex_` only guards the status/byte-count bookkeeping. + std::mutex stream_mutex_; Status callback_status_; int64_t bytes_written_ = 0; }; diff --git a/src/paimon/format/vortex/vortex_writer_builder.cpp b/src/paimon/format/vortex/vortex_writer_builder.cpp index 4400ae27..e06eb630 100644 --- a/src/paimon/format/vortex/vortex_writer_builder.cpp +++ b/src/paimon/format/vortex/vortex_writer_builder.cpp @@ -21,6 +21,7 @@ #include +#include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/format/vortex/vortex_format_writer.h" #include "paimon/status.h" @@ -33,7 +34,7 @@ Result> VortexWriterBuilder::Build( } // The paimon compression hint is not mapped to the sink; Vortex picks its own encodings. (void)compression; - return VortexFormatWriter::Create(output, schema_); + return VortexFormatWriter::Create(output, schema_, GetArrowPool(pool_)); } } // namespace paimon::vortex From 6780907032963eeff7a02fe42896c15d50728969 Mon Sep 17 00:00:00 2001 From: zhouyc-ali Date: Tue, 22 Sep 2026 16:17:37 +0000 Subject: [PATCH 9/9] fix(ci): silence bugprone-forward-declaration-namespace on the MemoryPool fwd decl The Debug build's check-clang-tidy gate analyses every changed translation unit. Any TU that includes paimon/type_fwd.h and also uses arrow::MemoryPool (as the Vortex writer now does for its rebasing pool) trips bugprone-forward-declaration-namespace on the paimon::MemoryPool forward declaration, even though paimon::MemoryPool is a real public type defined in paimon/memory/. Annotate the forward declaration with NOLINT plus a comment explaining that it is intentional. --- include/paimon/type_fwd.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/paimon/type_fwd.h b/include/paimon/type_fwd.h index e469934c..0382cdc9 100644 --- a/include/paimon/type_fwd.h +++ b/include/paimon/type_fwd.h @@ -38,7 +38,10 @@ class Result; class Status; -class MemoryPool; +// paimon::MemoryPool is a real public type (see paimon/memory/); this forward declaration is +// intentional. The NOLINT silences bugprone-forward-declaration-namespace, which otherwise fires in +// any translation unit that includes this header and happens to use arrow::MemoryPool. +class MemoryPool; // NOLINT(bugprone-forward-declaration-namespace) class FileFormat;