From 36e60dc86e7d57dbe776cdd65e15426bee412955 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 2 Sep 2026 15:35:05 +0800 Subject: [PATCH 01/16] feat(rest): access table data with the temporary credentials issued by the REST catalog --- include/paimon/catalog/catalog.h | 15 + include/paimon/catalog_options.h | 9 + src/paimon/CMakeLists.txt | 2 + src/paimon/common/catalog_options.cpp | 2 + .../core/catalog/file_system_catalog_test.cpp | 17 + .../table/system/global_system_tables.cpp | 13 +- .../core/table/system/system_table_test.cpp | 75 ++++ src/paimon/rest/resource_paths.cpp | 5 + src/paimon/rest/resource_paths.h | 3 + src/paimon/rest/resource_paths_test.cpp | 3 + src/paimon/rest/rest_api.cpp | 5 + src/paimon/rest/rest_api.h | 8 + src/paimon/rest/rest_catalog.cpp | 45 +- src/paimon/rest/rest_catalog.h | 24 +- src/paimon/rest/rest_catalog_test.cpp | 80 ++++ src/paimon/rest/rest_messages.cpp | 18 + src/paimon/rest/rest_messages.h | 28 ++ src/paimon/rest/rest_messages_test.cpp | 36 ++ src/paimon/rest/rest_token_file_system.cpp | 184 +++++++++ src/paimon/rest/rest_token_file_system.h | 127 ++++++ .../rest/rest_token_file_system_test.cpp | 385 ++++++++++++++++++ 21 files changed, 1075 insertions(+), 9 deletions(-) create mode 100644 src/paimon/rest/rest_token_file_system.cpp create mode 100644 src/paimon/rest/rest_token_file_system.h create mode 100644 src/paimon/rest/rest_token_file_system_test.cpp diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 742352b0..40f52527 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -187,6 +187,21 @@ class PAIMON_EXPORT Catalog { /// @return A shared pointer to the file system instance. virtual std::shared_ptr GetFileSystem() const = 0; + /// Returns the file system used to access the data of a specified table. + /// + /// @note A catalog that hands out per-table temporary credentials returns a file + /// system that refreshes them, so the returned instance must be used for the + /// table it was requested for. Pass it to `ReadContextBuilder::WithFileSystem`, + /// `ScanContextBuilder::WithFileSystem` or `WriteContextBuilder::WithFileSystem`. + /// + /// @param identifier The identifier (database and table name) of the table. + /// @return A shared pointer to the file system instance; the catalog-level file system + /// by default. + virtual Result> GetTableFileSystem( + const Identifier& identifier) const { + return GetFileSystem(); + } + /// Returns the catalog-level options that were passed during catalog creation. /// /// @return A const reference to the map of catalog options (key-value pairs). diff --git a/include/paimon/catalog_options.h b/include/paimon/catalog_options.h index 05b1af27..f781fa18 100644 --- a/include/paimon/catalog_options.h +++ b/include/paimon/catalog_options.h @@ -65,6 +65,15 @@ struct PAIMON_EXPORT CatalogOptions { /// "dlf.signing-algorithm" - DLF signer ("default" or "openapi"). static const char DLF_SIGNING_ALGORITHM[]; + /// "dlf.oss-endpoint" - OSS endpoint that overrides the "fs.oss.endpoint" of a data + /// token issued by the REST catalog. + static const char DLF_OSS_ENDPOINT[]; + + /// "data-token.enabled" - Whether table data is accessed with the temporary + /// credentials issued by the REST catalog instead of the credentials configured in + /// the catalog options. Defaults to false. + static const char DATA_TOKEN_ENABLED[]; + /// "table-default." - Prefix of the catalog options that provide table option /// defaults: "table-default.=" applies "=" to a created /// table when the caller left "" unset. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 45824cf9..35b613fa 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -488,6 +488,7 @@ if(PAIMON_ENABLE_REST) rest/dlf_auth.cpp rest/rest_catalog.cpp rest/rest_messages.cpp + rest/rest_token_file_system.cpp rest/rest_util.cpp) endif() @@ -1031,6 +1032,7 @@ if(PAIMON_BUILD_TESTS) rest/dlf_auth_test.cpp rest/rest_catalog_test.cpp rest/rest_messages_test.cpp + rest/rest_token_file_system_test.cpp rest/rest_util_test.cpp STATIC_LINK_LIBS paimon_shared diff --git a/src/paimon/common/catalog_options.cpp b/src/paimon/common/catalog_options.cpp index 1a856dad..30be6be5 100644 --- a/src/paimon/common/catalog_options.cpp +++ b/src/paimon/common/catalog_options.cpp @@ -33,6 +33,8 @@ const char CatalogOptions::DLF_TOKEN_LOADER[] = "dlf.token-loader"; const char CatalogOptions::DLF_TOKEN_ECS_METADATA_URL[] = "dlf.token-ecs-metadata-url"; const char CatalogOptions::DLF_TOKEN_ECS_ROLE_NAME[] = "dlf.token-ecs-role-name"; const char CatalogOptions::DLF_SIGNING_ALGORITHM[] = "dlf.signing-algorithm"; +const char CatalogOptions::DLF_OSS_ENDPOINT[] = "dlf.oss-endpoint"; +const char CatalogOptions::DATA_TOKEN_ENABLED[] = "data-token.enabled"; const char CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX[] = "table-default."; } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 1b30c9cf..18793419 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -44,6 +44,23 @@ namespace paimon::test { +TEST(FileSystemCatalogTest, TestGetTableFileSystem) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + + // a catalog without per-table credentials serves the table data with the catalog wide + // file system, whatever the table is + ASSERT_OK_AND_ASSIGN(auto table_fs, catalog.GetTableFileSystem(Identifier("db1", "t1"))); + ASSERT_EQ(catalog.GetFileSystem(), table_fs); + ASSERT_OK_AND_ASSIGN(auto other_fs, catalog.GetTableFileSystem(Identifier("db2", "t2"))); + ASSERT_EQ(catalog.GetFileSystem(), other_fs); +} + TEST(FileSystemCatalogTest, TestDatabaseExists) { std::map options; options[Options::FILE_SYSTEM] = "local"; diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index bd484edc..647da76c 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -515,8 +515,19 @@ Result> PartitionsSystemTable::BuildRows() const { return table_path_result.status(); } + // The files of each table are served with the credentials of that table, which a + // catalog issuing temporary ones per table only hands out through the catalog. + Result> table_fs_result = + context_.catalog->GetTableFileSystem(id); + if (!table_fs_result.ok()) { + if (table_fs_result.status().IsNotExist()) { + continue; + } + return table_fs_result.status(); + } + Result file_stats_result = - AggregateFileStats(context_.fs, table_path_result.value(), *data_schema); + AggregateFileStats(table_fs_result.value(), table_path_result.value(), *data_schema); if (!file_stats_result.ok()) { if (file_stats_result.status().IsNotExist()) { continue; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 15092fec..7591e53b 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -30,9 +30,14 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/generic_row.h" +#include "paimon/core/catalog/file_system_catalog.h" +#include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/table/system/system_table_scan.h" #include "paimon/defs.h" @@ -282,6 +287,76 @@ TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) "global system table requires catalog context: tables"); } +namespace { + +/// A catalog that serves the files of every table through `GetTableFileSystem`, like one +/// issuing temporary credentials per table does, and records what was asked of it. +class PerTableFileSystemCatalog : public FileSystemCatalog { + public: + PerTableFileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, + const std::map& options) + : FileSystemCatalog(fs, warehouse, options) {} + + Result> GetTableFileSystem( + const Identifier& identifier) const override { + requested.push_back(identifier.GetFullName()); + if (failure) { + return failure.value(); + } + return GetFileSystem(); + } + + mutable std::vector requested; + std::optional failure; +}; + +} // namespace + +TEST(SystemTableTest, TestGlobalPartitionsUsesTheFileSystemOfEachTable) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + + PerTableFileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + arrow::Schema typed_schema( + {arrow::field("dt", arrow::utf8()), arrow::field("v", arrow::int32(), /*nullable=*/true)}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(typed_schema, &c_schema).ok()); + Status created = catalog.CreateTable(Identifier("db1", "t1"), &c_schema, + /*partition_keys=*/{"dt"}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false); + ArrowSchemaRelease(&c_schema); + ASSERT_OK(created); + + GlobalSystemTableContext context; + context.catalog = &catalog; + context.fs = core_options.GetFileSystem(); + context.warehouse = dir->Str(); + context.catalog_options = options; + PartitionsSystemTable partitions(context); + + // The table holds no data, so it contributes no partition, but its credentials are + // still the ones asked for before its files are listed. + ASSERT_OK_AND_ASSIGN(std::vector rows, partitions.BuildRows()); + ASSERT_TRUE(rows.empty()); + ASSERT_EQ((std::vector{"db1.t1"}), catalog.requested); + + // A table whose credentials cannot be issued fails the query, so that a partial result + // is never mistaken for the whole one. + catalog.failure = Status::Invalid("no credentials for the table"); + ASSERT_NOK_WITH_MSG(partitions.BuildRows(), "no credentials for the table"); + + // A table that went away between being listed and being read is skipped instead, like + // one whose location has already been removed. + catalog.failure = Status::NotExist("table dropped"); + ASSERT_OK_AND_ASSIGN(std::vector skipped, partitions.BuildRows()); + ASSERT_TRUE(skipped.empty()); +} + TEST(SystemTableTest, TestScanMetricsAreSnapshots) { SystemTableScan scan("/tmp/table"); std::shared_ptr metrics = scan.GetMetrics(); diff --git a/src/paimon/rest/resource_paths.cpp b/src/paimon/rest/resource_paths.cpp index 75ce9f7b..e1ee40f6 100644 --- a/src/paimon/rest/resource_paths.cpp +++ b/src/paimon/rest/resource_paths.cpp @@ -73,4 +73,9 @@ std::string ResourcePaths::TableSnapshot(const std::string& database_name, return Table(database_name, table_name) + "/snapshot"; } +std::string ResourcePaths::TableToken(const std::string& database_name, + const std::string& table_name) const { + return Table(database_name, table_name) + "/token"; +} + } // namespace paimon diff --git a/src/paimon/rest/resource_paths.h b/src/paimon/rest/resource_paths.h index 2fc58e20..9294beff 100644 --- a/src/paimon/rest/resource_paths.h +++ b/src/paimon/rest/resource_paths.h @@ -43,6 +43,9 @@ class ResourcePaths { std::string TableSnapshot(const std::string& database_name, const std::string& table_name) const; + /// Path of the temporary file system credentials of one table. + std::string TableToken(const std::string& database_name, const std::string& table_name) const; + private: /// "/v1" or "/v1/{encoded prefix}". std::string base_; diff --git a/src/paimon/rest/resource_paths_test.cpp b/src/paimon/rest/resource_paths_test.cpp index 560b1d19..bc8a1355 100644 --- a/src/paimon/rest/resource_paths_test.cpp +++ b/src/paimon/rest/resource_paths_test.cpp @@ -33,12 +33,15 @@ TEST(ResourcePathsTest, WithPrefix) { ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1/snapshots", paths.Snapshots("db", "t1")); ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1/commit", paths.CommitTable("db", "t1")); ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1/snapshot", paths.TableSnapshot("db", "t1")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1/token", paths.TableToken("db", "t1")); + ASSERT_EQ("/v1/my+prefix/databases/db%231/tables/t+1/token", paths.TableToken("db#1", "t 1")); } TEST(ResourcePathsTest, WithoutPrefix) { ResourcePaths paths(""); ASSERT_EQ("/v1/databases", paths.Databases()); ASSERT_EQ("/v1/tables/rename", paths.RenameTable()); + ASSERT_EQ("/v1/databases/db/tables/t1/token", paths.TableToken("db", "t1")); } } // namespace paimon::test diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp index 86a4eb5c..af5a7858 100644 --- a/src/paimon/rest/rest_api.cpp +++ b/src/paimon/rest/rest_api.cpp @@ -339,4 +339,9 @@ Result RestApi::CommitSnapshot(const Identifier& identifier, return entity.IsSuccess(); } +Result RestApi::LoadTableToken(const Identifier& identifier) const { + return GetEntity( + resource_paths_.TableToken(identifier.GetDatabaseName(), identifier.GetTableName()), {}); +} + } // namespace paimon diff --git a/src/paimon/rest/rest_api.h b/src/paimon/rest/rest_api.h index 500de03d..6efdc180 100644 --- a/src/paimon/rest/rest_api.h +++ b/src/paimon/rest/rest_api.h @@ -80,6 +80,9 @@ class RestApi { static constexpr const char* kOptionUrlPrefix = "prefix"; /// Options with this prefix are sent as http headers (with the prefix stripped). static constexpr const char* kHeaderOptionPrefix = "header."; + /// Credentials are refreshed once they expire in less than this, so that a request + /// signed with them cannot expire while it is in flight. + static constexpr int64_t kTokenExpirationSafeTimeMillis = 3600000; /// Creates the api client. /// @@ -125,6 +128,11 @@ class RestApi { const Snapshot& snapshot, const std::vector& statistics) const; + /// Loads the temporary file system credentials of one table. A 403 means the caller + /// has no permission for the table and is reported as an error status carrying a + /// `RestErrorDetail`. + Result LoadTableToken(const Identifier& identifier) const; + /// Maps a non-successful http response to a status: 404 becomes `NotExist`, 409 /// becomes `Exist`, 400 becomes `Invalid`, 501 becomes `NotImplemented` and the /// other codes become `IOError`. A redirect returned while `follow_redirects` is diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 64396a29..12e2cc89 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -27,6 +27,7 @@ #include "paimon/catalog_options.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/rapidjson_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/catalog_utils.h" @@ -38,6 +39,7 @@ #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" +#include "paimon/rest/rest_token_file_system.h" #include "paimon/rest/rest_util.h" #include "paimon/table/format/format_table.h" #include "rapidjson/document.h" @@ -72,14 +74,23 @@ Result ToLoadIdentifier(const Identifier& identifier) { } // namespace -RestCatalog::RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, - const std::string& warehouse) +RestCatalog::RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, + const std::string& warehouse, bool data_token_enabled) : api_(std::move(api)), fs_(fs), warehouse_(warehouse), + data_token_enabled_(data_token_enabled), table_default_options_(RestUtil::ExtractPrefixMap( api_->GetMergedOptions(), CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX)), - logger_(Logger::GetLogger("RestCatalog")) {} + logger_(Logger::GetLogger("RestCatalog")) { + if (data_token_enabled_) { + GenericLruCache>::Options cache_options; + cache_options.max_weight = kMaxTableFileSystems; + table_fs_cache_ = + std::make_unique>>( + std::move(cache_options)); + } +} Result> RestCatalog::Create( const std::string& warehouse, const std::map& options, @@ -89,8 +100,11 @@ Result> RestCatalog::Create( RestApi::Create(options, warehouse, /*config_required=*/true, http_config)); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(api->GetMergedOptions(), file_system)); - return std::unique_ptr( - new RestCatalog(std::move(api), core_options.GetFileSystem(), warehouse)); + PAIMON_ASSIGN_OR_RAISE(bool data_token_enabled, + OptionsUtils::GetValueFromMap( + api->GetMergedOptions(), CatalogOptions::DATA_TOKEN_ENABLED, false)); + return std::unique_ptr(new RestCatalog( + std::move(api), core_options.GetFileSystem(), warehouse, data_token_enabled)); } const std::map& RestCatalog::GetOptions() const { @@ -391,6 +405,10 @@ Result> RestCatalog::LoadTableSchema(const Identifier& i if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } + // The file system is only used to build the system table, whose schema does not + // read any file, so the catalog wide one is enough here. The credentials of the + // table are applied when its rows are read, through the file system of the read or + // scan context. PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema, dynamic_options)); @@ -438,6 +456,23 @@ std::shared_ptr RestCatalog::GetFileSystem() const { return fs_; } +Result> RestCatalog::GetTableFileSystem( + const Identifier& identifier) const { + if (!data_token_enabled_) { + return fs_; + } + // The credentials are issued for the data table, so a system table shares those of + // the table it belongs to. + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + return table_fs_cache_->Get( + load_identifier.GetFullName(), + [this, &load_identifier](const std::string&) -> Result> { + std::shared_ptr fs = std::make_shared( + api_, api_->GetMergedOptions(), load_identifier); + return fs; + }); +} + Result> RestCatalog::ListSnapshots(const Identifier& identifier, const std::string& branch) const { PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "listSnapshots")); diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 1c0b6b52..015381dd 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -25,6 +25,7 @@ #include #include "paimon/catalog/catalog.h" +#include "paimon/common/utils/generic_lru_cache.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/logging.h" #include "paimon/rest/rest_api.h" @@ -72,6 +73,12 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { Result> LoadTableSchema(const Identifier& identifier) const override; std::string GetRootPath() const override; std::shared_ptr GetFileSystem() const override; + /// Returns a file system that refreshes the temporary credentials the server issues + /// for the table when `CatalogOptions::DATA_TOKEN_ENABLED` is set, and the + /// catalog-level file system otherwise. The instance of one table is reused, so that + /// its credentials are loaded once instead of per call. + Result> GetTableFileSystem( + const Identifier& identifier) const override; Result> GetTable(const Identifier& identifier) const override; Result> ListSnapshots(const Identifier& identifier, const std::string& branch) const override; @@ -98,8 +105,13 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& identifier) const override; private: - RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, - const std::string& warehouse); + /// Upper bound of the retained data token file systems, matching the Java client. A + /// table evicted from the cache is served by a new instance, which loads credentials + /// of its own, so the bound trades a token request for a bounded footprint. + static constexpr int64_t kMaxTableFileSystems = 1000; + + RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, + const std::string& warehouse, bool data_token_enabled); /// Loads the schema and catalog table ID from the same response. Result> LoadTableSchema(const Identifier& identifier, @@ -114,12 +126,18 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { static Result> ToTableSchema( const GetTableResponse& response, const std::optional& branch); - std::unique_ptr api_; + std::shared_ptr api_; std::shared_ptr fs_; std::string warehouse_; + /// Whether table data is accessed with the credentials the server issues per table. + bool data_token_enabled_ = false; /// The "table-default." options of the merged config, applied to `CreateTable` /// options when absent. std::map table_default_options_; + /// Data token file systems, keyed by the full name of the identifier sent to the + /// server. Only created when `data_token_enabled_` is set. Evicting an entry does not + /// invalidate a file system a caller still holds, since the entries are shared. + std::unique_ptr>> table_fs_cache_; std::shared_ptr logger_; }; diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 197877e2..5db836d6 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -507,6 +507,86 @@ TEST_F(RestCatalogTest, CreateMergesServerConfig) { ASSERT_NE(nullptr, catalog->GetFileSystem()); } +TEST_F(RestCatalogTest, TableFileSystemWithoutDataToken) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, + catalog->GetTableFileSystem(Identifier("db1", "t1"))); + // without the data token the catalog wide credentials are used for the data as well + ASSERT_EQ(catalog->GetFileSystem(), fs); +} + +TEST_F(RestCatalogTest, TableFileSystemWithDataToken) { + options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, + catalog->GetTableFileSystem(Identifier("db1", "t1"))); + ASSERT_NE(nullptr, fs); + ASSERT_NE(catalog->GetFileSystem(), fs); + + // one instance per table, so its credentials are loaded once and then shared + ASSERT_OK_AND_ASSIGN(std::shared_ptr again, + catalog->GetTableFileSystem(Identifier("db1", "t1"))); + ASSERT_EQ(fs, again); + // a system table reads the files of the table it belongs to + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table_fs, + catalog->GetTableFileSystem(Identifier("db1", "t1$snapshots"))); + ASSERT_EQ(fs, system_table_fs); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr other_table_fs, + catalog->GetTableFileSystem(Identifier("db1", "t2"))); + ASSERT_NE(fs, other_table_fs); +} + +TEST_F(RestCatalogTest, TableFileSystemNormalizesTheBranch) { + options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, + catalog->GetTableFileSystem(Identifier("db1", "t1"))); + + // the main branch is the table itself, so it shares the credentials + ASSERT_OK_AND_ASSIGN(std::shared_ptr main_branch_fs, + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_main"))); + ASSERT_EQ(fs, main_branch_fs); + + // another branch is addressed as its own object on the server, so it gets its own + // credentials + ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_fs, + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1"))); + ASSERT_NE(fs, branch_fs); + ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_fs_again, + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1"))); + ASSERT_EQ(branch_fs, branch_fs_again); +} + +TEST_F(RestCatalogTest, TableFileSystemCacheIsBounded) { + options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + + // Building a file system asks the server for nothing, the credentials are loaded when + // it is first used, so a catalog scanning many tables would grow without a bound. + constexpr int32_t kTables = 1001; + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, + catalog->GetTableFileSystem(Identifier("db1", "t0"))); + for (int32_t i = 1; i < kTables; ++i) { + ASSERT_OK(catalog->GetTableFileSystem(Identifier("db1", fmt::format("t{}", i))).status()); + } + + // The table used least recently was evicted, so it is served by a new instance that + // loads credentials of its own. + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_again, + catalog->GetTableFileSystem(Identifier("db1", "t0"))); + ASSERT_NE(first, first_again); + + // The table used most recently is still there. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr last, + catalog->GetTableFileSystem(Identifier("db1", fmt::format("t{}", kTables - 1)))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr last_again, + catalog->GetTableFileSystem(Identifier("db1", fmt::format("t{}", kTables - 1)))); + ASSERT_EQ(last, last_again); +} + TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); diff --git a/src/paimon/rest/rest_messages.cpp b/src/paimon/rest/rest_messages.cpp index 6cc2041f..e8d891dd 100644 --- a/src/paimon/rest/rest_messages.cpp +++ b/src/paimon/rest/rest_messages.cpp @@ -56,6 +56,8 @@ constexpr const char kFieldSource[] = "source"; constexpr const char kFieldDestination[] = "destination"; constexpr const char kFieldSuccess[] = "success"; constexpr const char kFieldSnapshot[] = "snapshot"; +constexpr const char kFieldToken[] = "token"; +constexpr const char kFieldExpiresAtMillis[] = "expiresAtMillis"; void AddOptionalStringMember(rapidjson::Value* obj, const char* key, const std::optional& value, @@ -360,6 +362,22 @@ void GetTableResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { audit_.ParseFrom(obj); } +rapidjson::Value GetTableTokenResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldToken), + RapidJsonUtil::SerializeValue(token_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldExpiresAtMillis), + RapidJsonUtil::SerializeValue(expires_at_millis_, allocator).Move(), *allocator); + return obj; +} + +void GetTableTokenResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldToken, {}); + expires_at_millis_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldExpiresAtMillis, 0); +} + rapidjson::Value CreateTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) { rapidjson::Value obj(rapidjson::kObjectType); diff --git a/src/paimon/rest/rest_messages.h b/src/paimon/rest/rest_messages.h index bc3e909b..8333b1fd 100644 --- a/src/paimon/rest/rest_messages.h +++ b/src/paimon/rest/rest_messages.h @@ -324,6 +324,34 @@ class GetTableResponse : public Jsonizable { RestAuditFields audit_; }; +/// The temporary file system credentials of one table. `token` carries file system +/// options (e.g. "fs.oss.accessKeyId") that are merged over the catalog options. +class GetTableTokenResponse : public Jsonizable { + public: + GetTableTokenResponse(const std::map& token, + int64_t expires_at_millis) + : token_(token), expires_at_millis_(expires_at_millis) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::map& GetToken() const { + return token_; + } + /// Expiration of the credentials; 0 when the server did not report one, which + /// makes them expire immediately. + int64_t GetExpiresAtMillis() const { + return expires_at_millis_; + } + + GetTableTokenResponse() = default; + + private: + std::map token_; + int64_t expires_at_millis_ = 0; +}; + /// `schema_json` uses the same schema JSON layout as `GetTableResponse`. class CreateTableRequest : public Jsonizable { public: diff --git a/src/paimon/rest/rest_messages_test.cpp b/src/paimon/rest/rest_messages_test.cpp index 53fbd5a5..69ccde68 100644 --- a/src/paimon/rest/rest_messages_test.cpp +++ b/src/paimon/rest/rest_messages_test.cpp @@ -196,6 +196,42 @@ TEST(RestMessagesTest, GetTableResponseParse) { ASSERT_STREQ("f0", schema["fields"][0]["name"].GetString()); } +TEST(RestMessagesTest, GetTableTokenResponseParse) { + std::string json = R"({ + "token": {"fs.oss.accessKeyId": "ak", "fs.oss.securityToken": "st"}, + "expiresAtMillis": 1700000000000, + "futureField": [1, 2] + })"; + ASSERT_OK_AND_ASSIGN(GetTableTokenResponse response, + GetTableTokenResponse::FromJsonString(json)); + ASSERT_EQ(2u, response.GetToken().size()); + ASSERT_EQ("ak", response.GetToken().at("fs.oss.accessKeyId")); + ASSERT_EQ("st", response.GetToken().at("fs.oss.securityToken")); + ASSERT_EQ(1700000000000, response.GetExpiresAtMillis()); +} + +TEST(RestMessagesTest, GetTableTokenResponseLenientParse) { + // an absent expiration makes the credentials expire immediately rather than fail + ASSERT_OK_AND_ASSIGN(GetTableTokenResponse no_expiration, + GetTableTokenResponse::FromJsonString(R"({"token": {"k": "v"}})")); + ASSERT_EQ("v", no_expiration.GetToken().at("k")); + ASSERT_EQ(0, no_expiration.GetExpiresAtMillis()); + + ASSERT_OK_AND_ASSIGN(GetTableTokenResponse no_token, + GetTableTokenResponse::FromJsonString(R"({"expiresAtMillis": 5})")); + ASSERT_TRUE(no_token.GetToken().empty()); + ASSERT_EQ(5, no_token.GetExpiresAtMillis()); +} + +TEST(RestMessagesTest, GetTableTokenResponseRoundTrip) { + GetTableTokenResponse response({{"fs.oss.accessKeyId", "ak"}, {"fs.oss.endpoint", "ep"}}, + 1700000000000); + ASSERT_OK_AND_ASSIGN(std::string json, response.ToJsonString()); + ASSERT_OK_AND_ASSIGN(GetTableTokenResponse parsed, GetTableTokenResponse::FromJsonString(json)); + ASSERT_EQ(response.GetToken(), parsed.GetToken()); + ASSERT_EQ(1700000000000, parsed.GetExpiresAtMillis()); +} + TEST(RestMessagesTest, UnknownFieldsAreIgnored) { // forward compatibility: fields a newer server adds must be ignored std::string json = R"({ diff --git a/src/paimon/rest/rest_token_file_system.cpp b/src/paimon/rest/rest_token_file_system.cpp new file mode 100644 index 00000000..f6ea0c13 --- /dev/null +++ b/src/paimon/rest/rest_token_file_system.cpp @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_token_file_system.h" + +#include +#include + +#include "paimon/catalog_options.h" +#include "paimon/core/core_options.h" + +namespace paimon { + +namespace { +/// Declared here rather than taken from the OSS file system, which is an optional build +/// component this module must not depend on. +constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; +} // namespace + +RestTokenFileSystem::RestTokenFileSystem(const std::shared_ptr& api, + const std::map& catalog_options, + const Identifier& identifier, Clock clock) + : api_(api), + catalog_options_(catalog_options), + identifier_(identifier), + clock_(std::move(clock)), + logger_(Logger::GetLogger("RestTokenFileSystem")) {} + +bool RestTokenFileSystem::ShouldRefresh() const { + if (!token_) { + return true; + } + int64_t now_millis = + std::chrono::duration_cast(clock_().time_since_epoch()).count(); + return token_->expires_at_millis - now_millis < RestApi::kTokenExpirationSafeTimeMillis; +} + +std::map RestTokenFileSystem::MergeTokenOptions( + const std::map& token) const { + std::map merged = token; + // The DLF OSS endpoint overrides the standard one, since the credentials are issued + // for the DLF endpoint rather than for the endpoint the catalog was configured with. + auto dlf_oss_endpoint = catalog_options_.find(CatalogOptions::DLF_OSS_ENDPOINT); + if (dlf_oss_endpoint != catalog_options_.end() && !dlf_oss_endpoint->second.empty()) { + merged[kOssEndpointOption] = dlf_oss_endpoint->second; + } + return merged; +} + +Status RestTokenFileSystem::Refresh() const { + PAIMON_LOG_INFO(logger_, "begin refresh data token for identifier [%s]", + identifier_.ToString().c_str()); + PAIMON_ASSIGN_OR_RAISE(GetTableTokenResponse response, api_->LoadTableToken(identifier_)); + PAIMON_LOG_INFO(logger_, "end refresh data token for identifier [%s] expiresAtMillis [%ld]", + identifier_.ToString().c_str(), + static_cast(response.GetExpiresAtMillis())); + + RestToken token{MergeTokenOptions(response.GetToken()), response.GetExpiresAtMillis()}; + // The credentials are the only file system options that change, so the file system is + // rebuilt from the catalog options with the credentials merged over them. + std::map fs_options = catalog_options_; + for (const auto& [key, value] : token.token) { + fs_options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(fs_options, /*specified_file_system=*/nullptr)); + std::shared_ptr fs = core_options.GetFileSystem(); + if (fs == nullptr) { + return Status::Invalid("failed to build the file system of the data token of ", + identifier_.ToString()); + } + + retained_fs_.push_back(fs); + while (retained_fs_.size() > kMaxRetainedFileSystems) { + retained_fs_.pop_front(); + } + token_ = std::move(token); + fs_ = std::move(fs); + return Status::OK(); +} + +Result> RestTokenFileSystem::Delegate() const { + { + std::shared_lock read_lock(mutex_); + if (!ShouldRefresh()) { + return fs_; + } + } + + std::unique_lock write_lock(mutex_); + // Double-check, another thread may have refreshed while this one waited for the lock. + if (!ShouldRefresh()) { + return fs_; + } + PAIMON_RETURN_NOT_OK(Refresh()); + return fs_; +} + +Result RestTokenFileSystem::ValidToken() const { + { + std::shared_lock read_lock(mutex_); + if (!ShouldRefresh()) { + return token_.value(); + } + } + + std::unique_lock write_lock(mutex_); + if (!ShouldRefresh()) { + return token_.value(); + } + PAIMON_RETURN_NOT_OK(Refresh()); + return token_.value(); +} + +Result> RestTokenFileSystem::Open(const std::string& path) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->Open(path); +} + +Result> RestTokenFileSystem::Open( + const FileStatus& file_status) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->Open(file_status); +} + +Result> RestTokenFileSystem::Create(const std::string& path, + bool overwrite) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->Create(path, overwrite); +} + +Status RestTokenFileSystem::Mkdirs(const std::string& path) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->Mkdirs(path); +} + +Status RestTokenFileSystem::Rename(const std::string& src, const std::string& dst) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->Rename(src, dst); +} + +Status RestTokenFileSystem::Delete(const std::string& path, bool recursive) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->Delete(path, recursive); +} + +Result RestTokenFileSystem::GetFileStatus(const std::string& path) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->GetFileStatus(path); +} + +Status RestTokenFileSystem::ListDir(const std::string& directory, + std::vector* file_status_list) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->ListDir(directory, file_status_list); +} + +Status RestTokenFileSystem::ListFileStatus(const std::string& path, + std::vector* file_status_list) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->ListFileStatus(path, file_status_list); +} + +Result RestTokenFileSystem::Exists(const std::string& path) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr fs, Delegate()); + return fs->Exists(path); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_token_file_system.h b/src/paimon/rest/rest_token_file_system.h new file mode 100644 index 00000000..289648a5 --- /dev/null +++ b/src/paimon/rest/rest_token_file_system.h @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/fs/file_system.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_api.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// The temporary credentials of one table, as issued by the REST catalog. +struct RestToken { + /// File system options, e.g. "fs.oss.accessKeyId". + std::map token; + int64_t expires_at_millis = 0; + + /// Orders tokens so that one can key the file system cache. + bool operator<(const RestToken& other) const { + if (expires_at_millis != other.expires_at_millis) { + return expires_at_millis < other.expires_at_millis; + } + return token < other.token; + } +}; + +/// A `FileSystem` that accesses table data with the temporary credentials issued by the +/// REST catalog for one table, reloading them before they expire. Every operation is +/// delegated to the file system built from the credentials merged over the catalog +/// options, so the schemes configured for the catalog keep working. +class RestTokenFileSystem : public FileSystem { + public: + using Clock = std::function; + + /// The file systems of the most recent credentials are retained so that a stream + /// opened just before a rotation is not left with a destroyed file system. + static constexpr size_t kMaxRetainedFileSystems = 4; + + /// @param api Client of the catalog that issues the credentials. Shared because this + /// file system commonly outlives the catalog it was obtained from. + /// @param catalog_options Options the credentials are merged over. + /// @param identifier The table the credentials are requested for. + /// @param clock Source of the current time, overridable for tests. + RestTokenFileSystem(const std::shared_ptr& api, + const std::map& catalog_options, + const Identifier& identifier, Clock clock = std::chrono::system_clock::now); + + ~RestTokenFileSystem() override = default; + + Result> Open(const std::string& path) const override; + Result> Open(const FileStatus& file_status) const override; + Result> Create(const std::string& path, + bool overwrite) const override; + + Status Mkdirs(const std::string& path) const override; + Status Rename(const std::string& src, const std::string& dst) const override; + Status Delete(const std::string& path, bool recursive = true) const override; + Result GetFileStatus(const std::string& path) const override; + Status ListDir(const std::string& directory, + std::vector* file_status_list) const override; + Status ListFileStatus(const std::string& path, + std::vector* file_status_list) const override; + Result Exists(const std::string& path) const override; + + /// Returns credentials that are not about to expire, reloading them when needed. Lets + /// a caller that brings its own file system use the credentials of this table. + Result ValidToken() const; + + private: + /// Returns the file system of the current credentials, reloading them when they + /// expire in less than `RestApi::kTokenExpirationSafeTimeMillis`. + Result> Delegate() const; + + /// Reloads the credentials and builds their file system. Called with the write lock + /// of `mutex_` held. + Status Refresh() const; + + /// Whether `token_` is absent or expires within the safe time. + bool ShouldRefresh() const; + + /// `catalog_options_` with `token` merged over it. + std::map MergeTokenOptions( + const std::map& token) const; + + std::shared_ptr api_; + std::map catalog_options_; + Identifier identifier_; + Clock clock_; + std::shared_ptr logger_; + + mutable std::shared_mutex mutex_; + mutable std::optional token_; + mutable std::shared_ptr fs_; + /// Retains the file systems of the recent credentials, oldest first. + mutable std::deque> retained_fs_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp new file mode 100644 index 00000000..cad1ae62 --- /dev/null +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_token_file_system.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/defs.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/rest/rest_api.h" +#include "paimon/rest/rest_messages.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +constexpr const char kToken[] = "test-token"; +constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; + +// The credentials the mock server hands out, plus the number of times it was asked for +// them. +struct MockTokenState { + std::map token = {{"fs.oss.accessKeyId", "ak-1"}}; + int64_t expires_at_millis = 0; + // when set, the token endpoint fails with this http code + std::optional force_error_code; + // guards all fields above: the handler runs on the server's accept thread while + // tests seed and inspect the state + std::mutex mutex; + std::atomic request_count{0}; +}; + +MockRestServer::Response HandleTokenRequest(MockTokenState* state, + const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request.path != "/v1/databases/db1/tables/t1/token") { + ErrorResponse error("", "", "unknown path " + request.path, 404); + response.code = 404; + response.body = error.ToJsonString().value(); + return response; + } + state->request_count++; + + std::lock_guard lock(state->mutex); + if (state->force_error_code) { + ErrorResponse error(ErrorResponse::kResourceTypeTable, "t1", "no permission", + state->force_error_code.value()); + response.code = state->force_error_code.value(); + response.body = error.ToJsonString().value(); + return response; + } + GetTableTokenResponse token(state->token, state->expires_at_millis); + response.body = token.ToJsonString().value(); + return response; +} + +int64_t ToMillis(std::chrono::system_clock::time_point time) { + return std::chrono::duration_cast(time.time_since_epoch()).count(); +} + +} // namespace + +class RestTokenFileSystemTest : public ::testing::Test { + protected: + void SetUp() override { + state_ = std::make_shared(); + // credentials that are valid well beyond the safe time, so nothing refreshes + // unless a test moves the clock + state_->expires_at_millis = kExpiresAtMillis; + now_millis_ = kNowMillis; + ASSERT_OK_AND_ASSIGN( + server_, MockRestServer::Start([state = state_](const MockRestServer::Request& req) { + return HandleTokenRequest(state.get(), req); + })); + temp_dir_ = UniqueTestDirectory::Create("local"); + ASSERT_NE(nullptr, temp_dir_); + + catalog_options_ = { + {CatalogOptions::URI, server_->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + {Options::FILE_SYSTEM, "local"}, + }; + } + + void TearDown() override { + if (server_) { + server_->Stop(); + } + } + + std::shared_ptr CreateFileSystem() { + Result> api = + RestApi::Create(catalog_options_, "", /*config_required=*/false); + if (!api.ok()) { + return nullptr; + } + std::shared_ptr shared_api(std::move(api).value()); + return std::make_shared( + shared_api, catalog_options_, Identifier("db1", "t1"), [this] { + return std::chrono::system_clock::time_point( + std::chrono::milliseconds(now_millis_.load())); + }); + } + + // Writes `content` to a file of the temp directory with the local file system and + // returns its path. + std::string WriteFile(const std::string& name, const std::string& content) { + std::string path = temp_dir_->Str() + "/" + name; + Result> out = + temp_dir_->GetFileSystem()->Create(path, /*overwrite=*/true); + EXPECT_OK(out.status()); + if (!out.ok()) { + return path; + } + std::unique_ptr stream = std::move(out).value(); + EXPECT_OK(stream->Write(content.data(), content.size()).status()); + EXPECT_OK(stream->Close()); + return path; + } + + // Epoch millis the injected clock starts at; an arbitrary point far enough from 0 + // that subtracting the safe time stays positive. + static constexpr int64_t kNowMillis = 1700000000000; + // Expiration the mock server reports, far beyond the safe time of `kNowMillis`. + static constexpr int64_t kExpiresAtMillis = + kNowMillis + 10 * RestApi::kTokenExpirationSafeTimeMillis; + + std::shared_ptr state_; + std::unique_ptr server_; + std::unique_ptr temp_dir_; + std::map catalog_options_; + std::atomic now_millis_{kNowMillis}; +}; + +TEST_F(RestTokenFileSystemTest, DelegatesWithTheLoadedToken) { + std::string path = WriteFile("data", "paimon"); + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + std::string content; + ASSERT_OK(fs->ReadFile(path, &content)); + ASSERT_EQ("paimon", content); + ASSERT_EQ(1, state_->request_count.load()); + + // the other operations reach the same delegate + ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(path)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(FileStatus status, fs->GetFileStatus(path)); + ASSERT_EQ(6, status.GetLen()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr in, fs->Open(status)); + ASSERT_OK(in->Close()); + std::vector basic_file_status_list; + ASSERT_OK(fs->ListDir(temp_dir_->Str(), &basic_file_status_list)); + ASSERT_EQ(1u, basic_file_status_list.size()); + std::vector file_status_list; + ASSERT_OK(fs->ListFileStatus(temp_dir_->Str(), &file_status_list)); + ASSERT_EQ(1u, file_status_list.size()); + ASSERT_OK(fs->Rename(path, path + ".renamed")); + ASSERT_OK(fs->Mkdirs(temp_dir_->Str() + "/sub")); + ASSERT_OK(fs->Delete(temp_dir_->Str() + "/sub")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr out, + fs->Create(temp_dir_->Str() + "/written", /*overwrite=*/true)); + ASSERT_OK(out->Close()); + + // credentials that are not about to expire are loaded once + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, ValidTokenCarriesOnlyTheServerCredentials) { + catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = "dlf-endpoint"; + { + std::lock_guard lock(state_->mutex); + state_->token = {{"fs.oss.accessKeyId", "ak-1"}, {kOssEndpointOption, "server-endpoint"}}; + } + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); + ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); + // the endpoint the credentials were issued for wins over the one the server reported + ASSERT_EQ("dlf-endpoint", token.token.at(kOssEndpointOption)); + ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); + // the catalog options are not part of the token, so its secrets stay private + ASSERT_EQ(0u, token.token.count(CatalogOptions::TOKEN)); + ASSERT_EQ(2u, token.token.size()); +} + +TEST_F(RestTokenFileSystemTest, ReloadsWithinTheSafeTime) { + std::string path = WriteFile("data", "paimon"); + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + ASSERT_OK_AND_ASSIGN(RestToken first, fs->ValidToken()); + ASSERT_EQ("ak-1", first.token.at("fs.oss.accessKeyId")); + ASSERT_EQ(1, state_->request_count.load()); + + // one millisecond before the safe time the credentials are still used as they are + now_millis_ = kExpiresAtMillis - RestApi::kTokenExpirationSafeTimeMillis - 1; + std::string content; + ASSERT_OK(fs->ReadFile(path, &content)); + ASSERT_EQ("paimon", content); + ASSERT_EQ(1, state_->request_count.load()); + + // a stream opened with the current credentials must survive their rotation, since it + // does not own the file system it came from + ASSERT_OK_AND_ASSIGN(std::unique_ptr in, fs->Open(path)); + + int64_t next_expiration = kExpiresAtMillis + RestApi::kTokenExpirationSafeTimeMillis; + { + std::lock_guard lock(state_->mutex); + state_->token = {{"fs.oss.accessKeyId", "ak-2"}}; + state_->expires_at_millis = next_expiration; + } + now_millis_ = kExpiresAtMillis - 1; + ASSERT_OK_AND_ASSIGN(RestToken second, fs->ValidToken()); + ASSERT_EQ("ak-2", second.token.at("fs.oss.accessKeyId")); + ASSERT_EQ(next_expiration, second.expires_at_millis); + ASSERT_EQ(2, state_->request_count.load()); + + content.assign(6, '\0'); + ASSERT_OK_AND_ASSIGN(int64_t read_length, in->Read(content.data(), 6)); + ASSERT_EQ(6, read_length); + ASSERT_EQ("paimon", content); + ASSERT_OK(in->Close()); + + // the refreshed credentials are reused + ASSERT_OK(fs->ReadFile(path, &content)); + ASSERT_EQ(2, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, ExpiredTokenReloadsOnEveryCall) { + // an expiration the server did not report makes the credentials expire immediately + { + std::lock_guard lock(state_->mutex); + state_->expires_at_millis = 0; + } + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + ASSERT_OK(fs->ValidToken().status()); + ASSERT_OK(fs->ValidToken().status()); + ASSERT_EQ(2, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, ForbiddenIsReportedToTheCaller) { + { + std::lock_guard lock(state_->mutex); + state_->force_error_code = 403; + } + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + Status status = fs->Exists("any-path").status(); + ASSERT_NOK(status); + ASSERT_NOK_WITH_MSG(status, "no permission"); + ASSERT_NE(nullptr, status.detail()); + ASSERT_EQ(std::string(RestErrorDetail::kTypeId), status.detail()->type_id()); + ASSERT_EQ(403, checked_pointer_cast(status.detail())->GetCode()); + + // a later success is not blocked by the earlier failure + { + std::lock_guard lock(state_->mutex); + state_->force_error_code.reset(); + } + std::string path = WriteFile("data", "paimon"); + ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(path)); + ASSERT_TRUE(exists); +} + +TEST_F(RestTokenFileSystemTest, DefaultClockIsTheSystemClock) { + // the default clock is only exercised here: the other tests inject their own + { + std::lock_guard lock(state_->mutex); + state_->expires_at_millis = ToMillis(std::chrono::system_clock::now()) + + 10 * RestApi::kTokenExpirationSafeTimeMillis; + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(catalog_options_, "", /*config_required=*/false)); + RestTokenFileSystem fs(std::shared_ptr(std::move(api)), catalog_options_, + Identifier("db1", "t1")); + ASSERT_OK(fs.ValidToken().status()); + ASSERT_OK(fs.ValidToken().status()); + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, EmptyDlfOssEndpointIsNotApplied) { + catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = ""; + { + std::lock_guard lock(state_->mutex); + state_->token = {{kOssEndpointOption, "server-endpoint"}}; + } + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + // an unset dlf endpoint leaves the endpoint the server reported alone + ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); + ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); + ASSERT_EQ(1u, token.token.size()); +} + +TEST_F(RestTokenFileSystemTest, TokenOverridesTheCatalogFileSystemOptions) { + // the credentials must reach the options the delegate is built from: the catalog is + // configured with the local file system, yet an option of the token replaces it + { + std::lock_guard lock(state_->mutex); + state_->token = {{Options::FILE_SYSTEM, "no-such-file-system"}}; + } + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + std::string path = WriteFile("data", "paimon"); + Status status = fs->Exists(path).status(); + ASSERT_NOK(status); + ASSERT_NOK_WITH_MSG(status, "no-such-file-system"); + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, ConcurrentFirstAccessLoadsTheTokenOnce) { + std::string path = WriteFile("data", "paimon"); + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + constexpr size_t kThreads = 8; + std::atomic failures{0}; + std::vector threads; + threads.reserve(kThreads); + for (size_t i = 0; i < kThreads; ++i) { + threads.emplace_back([&] { + Result exists = fs->Exists(path); + if (!exists.ok() || !exists.value()) { + failures++; + } + }); + } + for (std::thread& thread : threads) { + thread.join(); + } + + ASSERT_EQ(0, failures.load()); + // the double-checked refresh keeps the threads that waited for the lock from each + // loading credentials of their own + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST(RestTokenTest, OrdersByExpirationThenCredentials) { + RestToken early{{{"k", "v"}}, 1}; + RestToken late{{{"k", "v"}}, 2}; + ASSERT_TRUE(early < late); + ASSERT_FALSE(late < early); + + RestToken other_credentials{{{"k", "w"}}, 1}; + ASSERT_TRUE(early < other_credentials); + ASSERT_FALSE(other_credentials < early); + ASSERT_FALSE(early < early); +} + +} // namespace paimon::test From 5da21bea00621034cb0478aa42f7e49b29c3a227 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Tue, 15 Sep 2026 11:14:21 +0800 Subject: [PATCH 02/16] fix --- src/paimon/rest/rest_catalog.cpp | 20 +- src/paimon/rest/rest_catalog.h | 20 +- src/paimon/rest/rest_catalog_test.cpp | 171 ++++++-- src/paimon/rest/rest_messages.cpp | 6 +- src/paimon/rest/rest_messages.h | 3 +- src/paimon/rest/rest_messages_test.cpp | 19 +- src/paimon/rest/rest_token_file_system.cpp | 62 +-- src/paimon/rest/rest_token_file_system.h | 60 ++- .../rest/rest_token_file_system_test.cpp | 402 +++++++++++++++++- 9 files changed, 639 insertions(+), 124 deletions(-) diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 12e2cc89..7102aad1 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -84,11 +84,7 @@ RestCatalog::RestCatalog(std::shared_ptr api, const std::shared_ptrGetMergedOptions(), CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX)), logger_(Logger::GetLogger("RestCatalog")) { if (data_token_enabled_) { - GenericLruCache>::Options cache_options; - cache_options.max_weight = kMaxTableFileSystems; - table_fs_cache_ = - std::make_unique>>( - std::move(cache_options)); + token_fs_cache_ = RestTokenFileSystem::CreateFileSystemCache(); } } @@ -464,13 +460,13 @@ Result> RestCatalog::GetTableFileSystem( // The credentials are issued for the data table, so a system table shares those of // the table it belongs to. PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); - return table_fs_cache_->Get( - load_identifier.GetFullName(), - [this, &load_identifier](const std::string&) -> Result> { - std::shared_ptr fs = std::make_shared( - api_, api_->GetMergedOptions(), load_identifier); - return fs; - }); + // Building the file system asks the server for nothing, the credentials are loaded on + // the first access, so nothing is keyed by the table here: the file systems built from + // the credentials are what `token_fs_cache_` reuses and bounds. Keying an instance by + // its table would keep serving the credentials of a dropped table to a table recreated + // at another location. + return std::make_shared(api_, api_->GetMergedOptions(), load_identifier, + token_fs_cache_); } Result> RestCatalog::ListSnapshots(const Identifier& identifier, diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 015381dd..1af74938 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -29,6 +29,7 @@ #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/logging.h" #include "paimon/rest/rest_api.h" +#include "paimon/rest/rest_token_file_system.h" #include "paimon/result.h" #include "paimon/status.h" @@ -75,8 +76,10 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { std::shared_ptr GetFileSystem() const override; /// Returns a file system that refreshes the temporary credentials the server issues /// for the table when `CatalogOptions::DATA_TOKEN_ENABLED` is set, and the - /// catalog-level file system otherwise. The instance of one table is reused, so that - /// its credentials are loaded once instead of per call. + /// catalog-level file system otherwise. Every call returns an instance bound to the + /// table it was asked for, whose credentials are loaded on the first access; the file + /// systems built from them are shared through a bounded cache, so the tables the + /// server issues the same credentials for share one file system. Result> GetTableFileSystem( const Identifier& identifier) const override; Result> GetTable(const Identifier& identifier) const override; @@ -105,11 +108,6 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { const Identifier& identifier) const override; private: - /// Upper bound of the retained data token file systems, matching the Java client. A - /// table evicted from the cache is served by a new instance, which loads credentials - /// of its own, so the bound trades a token request for a bounded footprint. - static constexpr int64_t kMaxTableFileSystems = 1000; - RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled); @@ -134,10 +132,10 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// The "table-default." options of the merged config, applied to `CreateTable` /// options when absent. std::map table_default_options_; - /// Data token file systems, keyed by the full name of the identifier sent to the - /// server. Only created when `data_token_enabled_` is set. Evicting an entry does not - /// invalidate a file system a caller still holds, since the entries are shared. - std::unique_ptr>> table_fs_cache_; + /// The file systems of the data tokens, keyed by the credentials they were built from + /// and shared by the data token file systems this catalog hands out. Only created when + /// `data_token_enabled_` is set. + std::shared_ptr token_fs_cache_; std::shared_ptr logger_; }; diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 5db836d6..4c5581a1 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -62,6 +62,9 @@ namespace { constexpr const char kToken[] = "test-token"; constexpr const char kPrefix[] = "paimon"; constexpr const char kWarehouse[] = "wh1"; +// Expiration of the data tokens the mock issues, far enough in the future that they are +// never refreshed within a test. +constexpr int64_t kDataTokenExpiresAtMillis = 4102444800000; // The in-memory catalog state behind the mock rest server. struct MockCatalogState { @@ -70,10 +73,13 @@ struct MockCatalogState { int64_t schema_id = 0; std::string path; std::string id = "1"; + std::map token = {{"fs.oss.accessKeyId", "ak-1"}}; }; std::map> databases; // headers of the last request, with lower-cased names std::map last_headers; + // paths the data token endpoint was called at, in order + std::vector token_requests; // when set, every request except "/v1/config" fails with this http code std::optional force_error_code; // how many times a single table has been fetched, so a caller that needs the path and the @@ -236,6 +242,28 @@ MockRestServer::Response HandleCatalogRequest(MockCatalogState* state, std::string remainder = rest.substr(databases_prefix.size()); size_t tables_pos = remainder.find("/tables"); + const std::string token_suffix = "/token"; + if (tables_pos != std::string::npos && remainder.size() > token_suffix.size() && + remainder.compare(remainder.size() - token_suffix.size(), token_suffix.size(), + token_suffix) == 0) { + // The mock issues credentials for any table, so that a test can ask for the token + // of a branch or of a table it did not seed. + state->token_requests.push_back(request.path); + std::map credentials = {{"fs.oss.accessKeyId", "ak-1"}}; + auto database = state->databases.find(remainder.substr(0, tables_pos)); + if (database != state->databases.end()) { + size_t table_start = tables_pos + std::strlen("/tables/"); + std::string table_name = + remainder.substr(table_start, remainder.size() - table_start - token_suffix.size()); + auto table = database->second.find(table_name); + if (table != database->second.end()) { + credentials = table->second.token; + } + } + GetTableTokenResponse token(credentials, kDataTokenExpiresAtMillis); + return JsonResponse(200, token.ToJsonString().value()); + } + if (tables_pos == std::string::npos) { const std::string& db_name = remainder; auto db_iter = state->databases.find(db_name); @@ -496,6 +524,16 @@ class RestCatalogTest : public ::testing::Test { std::shared_ptr state_; std::unique_ptr server_; std::map options_; + + // Path the data token of `database`.`table` is requested at. + static std::string TokenPath(const std::string& database, const std::string& table) { + return fmt::format("/v1/{}/databases/{}/tables/{}/token", kPrefix, database, table); + } + + std::vector TokenRequests() { + std::lock_guard lock(state_->mutex); + return state_->token_requests; + } }; TEST_F(RestCatalogTest, CreateMergesServerConfig) { @@ -523,68 +561,125 @@ TEST_F(RestCatalogTest, TableFileSystemWithDataToken) { ASSERT_NE(nullptr, fs); ASSERT_NE(catalog->GetFileSystem(), fs); - // one instance per table, so its credentials are loaded once and then shared - ASSERT_OK_AND_ASSIGN(std::shared_ptr again, - catalog->GetTableFileSystem(Identifier("db1", "t1"))); - ASSERT_EQ(fs, again); - // a system table reads the files of the table it belongs to + // building it asks the server for nothing: the credentials of the table it is bound to + // are loaded when it is first used + ASSERT_TRUE(TokenRequests().empty()); + ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists("/no-such-file")); + ASSERT_FALSE(exists); + ASSERT_EQ(std::vector({TokenPath("db1", "t1")}), TokenRequests()); + + // a system table reads the files of the table it belongs to, so it is served the + // credentials of that table ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table_fs, catalog->GetTableFileSystem(Identifier("db1", "t1$snapshots"))); - ASSERT_EQ(fs, system_table_fs); - + ASSERT_OK(system_table_fs->Exists("/no-such-file").status()); ASSERT_OK_AND_ASSIGN(std::shared_ptr other_table_fs, catalog->GetTableFileSystem(Identifier("db1", "t2"))); - ASSERT_NE(fs, other_table_fs); + ASSERT_OK(other_table_fs->Exists("/no-such-file").status()); + ASSERT_EQ(std::vector( + {TokenPath("db1", "t1"), TokenPath("db1", "t1"), TokenPath("db1", "t2")}), + TokenRequests()); +} + +TEST_F(RestCatalogTest, TableFileSystemIsBoundToTheTableItWasAskedFor) { + options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + + // the database and the table are addressed separately, so identifiers that print the + // same must not be served the credentials of one another + ASSERT_OK_AND_ASSIGN(std::shared_ptr dotted_database, + catalog->GetTableFileSystem(Identifier("db1.a", "t1"))); + ASSERT_OK(dotted_database->Exists("/no-such-file").status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr dotted_table, + catalog->GetTableFileSystem(Identifier("db1", "a.t1"))); + ASSERT_OK(dotted_table->Exists("/no-such-file").status()); + ASSERT_EQ(std::vector({TokenPath("db1.a", "t1"), TokenPath("db1", "a.t1")}), + TokenRequests()); } TEST_F(RestCatalogTest, TableFileSystemNormalizesTheBranch) { options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, - catalog->GetTableFileSystem(Identifier("db1", "t1"))); // the main branch is the table itself, so it shares the credentials ASSERT_OK_AND_ASSIGN(std::shared_ptr main_branch_fs, catalog->GetTableFileSystem(Identifier("db1", "t1$branch_main"))); - ASSERT_EQ(fs, main_branch_fs); + ASSERT_OK(main_branch_fs->Exists("/no-such-file").status()); // another branch is addressed as its own object on the server, so it gets its own // credentials ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_fs, catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1"))); - ASSERT_NE(fs, branch_fs); - ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_fs_again, - catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1"))); - ASSERT_EQ(branch_fs, branch_fs_again); + ASSERT_OK(branch_fs->Exists("/no-such-file").status()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_system_fs, + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1$snapshots"))); + ASSERT_OK(branch_system_fs->Exists("/no-such-file").status()); + ASSERT_EQ(std::vector({TokenPath("db1", "t1"), TokenPath("db1", "t1$branch_b1"), + TokenPath("db1", "t1$branch_b1")}), + TokenRequests()); } -TEST_F(RestCatalogTest, TableFileSystemCacheIsBounded) { +TEST_F(RestCatalogTest, TableFileSystemIsNotRetainedPerTable) { options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); - // Building a file system asks the server for nothing, the credentials are loaded when - // it is first used, so a catalog scanning many tables would grow without a bound. - constexpr int32_t kTables = 1001; + // The catalog remembers no file system of a table, so a table dropped and recreated at + // another location is never served the credentials of the dropped one. What is cached + // and bounded are the file systems built from the credentials, keyed by them. ASSERT_OK_AND_ASSIGN(std::shared_ptr first, - catalog->GetTableFileSystem(Identifier("db1", "t0"))); - for (int32_t i = 1; i < kTables; ++i) { - ASSERT_OK(catalog->GetTableFileSystem(Identifier("db1", fmt::format("t{}", i))).status()); - } - - // The table used least recently was evicted, so it is served by a new instance that - // loads credentials of its own. - ASSERT_OK_AND_ASSIGN(std::shared_ptr first_again, - catalog->GetTableFileSystem(Identifier("db1", "t0"))); - ASSERT_NE(first, first_again); - - // The table used most recently is still there. - ASSERT_OK_AND_ASSIGN( - std::shared_ptr last, - catalog->GetTableFileSystem(Identifier("db1", fmt::format("t{}", kTables - 1)))); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr last_again, - catalog->GetTableFileSystem(Identifier("db1", fmt::format("t{}", kTables - 1)))); - ASSERT_EQ(last, last_again); + catalog->GetTableFileSystem(Identifier("db1", "t1"))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second, + catalog->GetTableFileSystem(Identifier("db1", "t1"))); + ASSERT_NE(first, second); + + ASSERT_OK(first->Exists("/no-such-file").status()); + ASSERT_OK(second->Exists("/no-such-file").status()); + ASSERT_EQ(std::vector({TokenPath("db1", "t1"), TokenPath("db1", "t1")}), + TokenRequests()); +} + +TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) { + options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + Identifier identifier("db1", "t1"); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + ASSERT_OK_AND_ASSIGN(std::string old_location, catalog->GetTableLocation(identifier)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, + catalog->GetTableFileSystem(identifier)); + std::shared_ptr first_token_fs = + std::dynamic_pointer_cast(first); + ASSERT_NE(nullptr, first_token_fs); + ASSERT_OK_AND_ASSIGN(RestToken old_token, first_token_fs->ValidToken()); + ASSERT_EQ("ak-1", old_token.token.at("fs.oss.accessKeyId")); + + ASSERT_OK(catalog->DropTable(identifier, /*ignore_if_not_exists=*/false)); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(identifier)); + ASSERT_FALSE(exists); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + std::string new_location = old_location + "-recreated"; + { + // the server hands the recreated table another location and other credentials, while + // the expiration of the credentials of the dropped table stays the same + std::lock_guard lock(state_->mutex); + MockCatalogState::TableData& table = state_->databases.at("db1").at("t1"); + table.path = new_location; + table.token = {{"fs.oss.accessKeyId", "ak-2"}}; + } + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(identifier)); + ASSERT_EQ(new_location, location); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second, + catalog->GetTableFileSystem(identifier)); + ASSERT_NE(first, second); + std::shared_ptr second_token_fs = + std::dynamic_pointer_cast(second); + ASSERT_NE(nullptr, second_token_fs); + ASSERT_OK_AND_ASSIGN(RestToken new_token, second_token_fs->ValidToken()); + ASSERT_EQ("ak-2", new_token.token.at("fs.oss.accessKeyId")); + ASSERT_EQ(old_token.expires_at_millis, new_token.expires_at_millis); + ASSERT_EQ(kDataTokenExpiresAtMillis, new_token.expires_at_millis); + ASSERT_EQ(std::vector({TokenPath("db1", "t1"), TokenPath("db1", "t1")}), + TokenRequests()); } TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { diff --git a/src/paimon/rest/rest_messages.cpp b/src/paimon/rest/rest_messages.cpp index e8d891dd..f9b7048c 100644 --- a/src/paimon/rest/rest_messages.cpp +++ b/src/paimon/rest/rest_messages.cpp @@ -373,8 +373,10 @@ rapidjson::Value GetTableTokenResponse::ToJson(rapidjson::Document::AllocatorTyp } void GetTableTokenResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { - token_ = RapidJsonUtil::DeserializeKeyValue>( - obj, kFieldToken, {}); + // A response without credentials must not be turned into a fall back to the catalog + // credentials, so "token" is required while an explicitly empty object is accepted. + token_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldToken); expires_at_millis_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldExpiresAtMillis, 0); } diff --git a/src/paimon/rest/rest_messages.h b/src/paimon/rest/rest_messages.h index 8333b1fd..76c47b8f 100644 --- a/src/paimon/rest/rest_messages.h +++ b/src/paimon/rest/rest_messages.h @@ -325,7 +325,8 @@ class GetTableResponse : public Jsonizable { }; /// The temporary file system credentials of one table. `token` carries file system -/// options (e.g. "fs.oss.accessKeyId") that are merged over the catalog options. +/// options (e.g. "fs.oss.accessKeyId") that are merged over the catalog options, and is +/// required: a response that omits it is rejected instead of read as "no credentials". class GetTableTokenResponse : public Jsonizable { public: GetTableTokenResponse(const std::map& token, diff --git a/src/paimon/rest/rest_messages_test.cpp b/src/paimon/rest/rest_messages_test.cpp index 69ccde68..3a9726d3 100644 --- a/src/paimon/rest/rest_messages_test.cpp +++ b/src/paimon/rest/rest_messages_test.cpp @@ -217,10 +217,21 @@ TEST(RestMessagesTest, GetTableTokenResponseLenientParse) { ASSERT_EQ("v", no_expiration.GetToken().at("k")); ASSERT_EQ(0, no_expiration.GetExpiresAtMillis()); - ASSERT_OK_AND_ASSIGN(GetTableTokenResponse no_token, - GetTableTokenResponse::FromJsonString(R"({"expiresAtMillis": 5})")); - ASSERT_TRUE(no_token.GetToken().empty()); - ASSERT_EQ(5, no_token.GetExpiresAtMillis()); + // credentials the server reports as explicitly empty are parsed as such + ASSERT_OK_AND_ASSIGN( + GetTableTokenResponse empty_token, + GetTableTokenResponse::FromJsonString(R"({"token": {}, "expiresAtMillis": 5})")); + ASSERT_TRUE(empty_token.GetToken().empty()); + ASSERT_EQ(5, empty_token.GetExpiresAtMillis()); +} + +TEST(RestMessagesTest, GetTableTokenResponseRequiresTheToken) { + // a missing or null token is a malformed response, telling it apart from an empty one + // keeps it from being served as "no credentials", which would fall back to the + // credentials configured for the catalog + ASSERT_NOK(GetTableTokenResponse::FromJsonString(R"({"expiresAtMillis": 5})").status()); + ASSERT_NOK( + GetTableTokenResponse::FromJsonString(R"({"token": null, "expiresAtMillis": 5})").status()); } TEST(RestMessagesTest, GetTableTokenResponseRoundTrip) { diff --git a/src/paimon/rest/rest_token_file_system.cpp b/src/paimon/rest/rest_token_file_system.cpp index f6ea0c13..baa2d7ad 100644 --- a/src/paimon/rest/rest_token_file_system.cpp +++ b/src/paimon/rest/rest_token_file_system.cpp @@ -18,6 +18,7 @@ #include "paimon/rest/rest_token_file_system.h" +#include #include #include @@ -32,12 +33,31 @@ namespace { constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; } // namespace +size_t RestToken::Hash::operator()(const RestToken& rest_token) const { + size_t result = std::hash()(rest_token.expires_at_millis); + for (const auto& [key, value] : rest_token.token) { + result = result * 31 + std::hash()(key); + result = result * 31 + std::hash()(value); + } + return result; +} + +std::shared_ptr RestTokenFileSystem::CreateFileSystemCache() { + RestTokenFileSystemCache::Options cache_options; + cache_options.max_weight = kMaxCachedFileSystems; + cache_options.expire_after_access_ms = kFileSystemCacheExpireAfterAccessMillis; + return std::make_shared(std::move(cache_options)); +} + RestTokenFileSystem::RestTokenFileSystem(const std::shared_ptr& api, const std::map& catalog_options, - const Identifier& identifier, Clock clock) + const Identifier& identifier, + std::shared_ptr fs_cache, + Clock clock) : api_(api), catalog_options_(catalog_options), identifier_(identifier), + fs_cache_(fs_cache != nullptr ? std::move(fs_cache) : CreateFileSystemCache()), clock_(std::move(clock)), logger_(Logger::GetLogger("RestTokenFileSystem")) {} @@ -62,7 +82,7 @@ std::map RestTokenFileSystem::MergeTokenOptions( return merged; } -Status RestTokenFileSystem::Refresh() const { +Status RestTokenFileSystem::RefreshToken() const { PAIMON_LOG_INFO(logger_, "begin refresh data token for identifier [%s]", identifier_.ToString().c_str()); PAIMON_ASSIGN_OR_RAISE(GetTableTokenResponse response, api_->LoadTableToken(identifier_)); @@ -70,9 +90,12 @@ Status RestTokenFileSystem::Refresh() const { identifier_.ToString().c_str(), static_cast(response.GetExpiresAtMillis())); - RestToken token{MergeTokenOptions(response.GetToken()), response.GetExpiresAtMillis()}; - // The credentials are the only file system options that change, so the file system is - // rebuilt from the catalog options with the credentials merged over them. + token_ = RestToken{MergeTokenOptions(response.GetToken()), response.GetExpiresAtMillis()}; + return Status::OK(); +} + +Result> RestTokenFileSystem::BuildFileSystem( + const RestToken& token) const { std::map fs_options = catalog_options_; for (const auto& [key, value] : token.token) { fs_options[key] = value; @@ -84,31 +107,13 @@ Status RestTokenFileSystem::Refresh() const { return Status::Invalid("failed to build the file system of the data token of ", identifier_.ToString()); } - - retained_fs_.push_back(fs); - while (retained_fs_.size() > kMaxRetainedFileSystems) { - retained_fs_.pop_front(); - } - token_ = std::move(token); - fs_ = std::move(fs); - return Status::OK(); + return fs; } Result> RestTokenFileSystem::Delegate() const { - { - std::shared_lock read_lock(mutex_); - if (!ShouldRefresh()) { - return fs_; - } - } - - std::unique_lock write_lock(mutex_); - // Double-check, another thread may have refreshed while this one waited for the lock. - if (!ShouldRefresh()) { - return fs_; - } - PAIMON_RETURN_NOT_OK(Refresh()); - return fs_; + PAIMON_ASSIGN_OR_RAISE(RestToken token, ValidToken()); + return fs_cache_->Get( + token, [this](const RestToken& cached_token) { return BuildFileSystem(cached_token); }); } Result RestTokenFileSystem::ValidToken() const { @@ -120,10 +125,11 @@ Result RestTokenFileSystem::ValidToken() const { } std::unique_lock write_lock(mutex_); + // Double-check, another thread may have refreshed while this one waited for the lock. if (!ShouldRefresh()) { return token_.value(); } - PAIMON_RETURN_NOT_OK(Refresh()); + PAIMON_RETURN_NOT_OK(RefreshToken()); return token_.value(); } diff --git a/src/paimon/rest/rest_token_file_system.h b/src/paimon/rest/rest_token_file_system.h index 289648a5..b1272682 100644 --- a/src/paimon/rest/rest_token_file_system.h +++ b/src/paimon/rest/rest_token_file_system.h @@ -19,8 +19,8 @@ #pragma once #include +#include #include -#include #include #include #include @@ -30,6 +30,7 @@ #include #include "paimon/catalog/identifier.h" +#include "paimon/common/utils/generic_lru_cache.h" #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/rest/rest_api.h" @@ -44,15 +45,24 @@ struct RestToken { std::map token; int64_t expires_at_millis = 0; - /// Orders tokens so that one can key the file system cache. - bool operator<(const RestToken& other) const { - if (expires_at_millis != other.expires_at_millis) { - return expires_at_millis < other.expires_at_millis; - } - return token < other.token; + /// Credentials are interchangeable only when they grant the same access until the same + /// point in time, which is what makes them a file system cache key. + bool operator==(const RestToken& other) const { + return expires_at_millis == other.expires_at_millis && token == other.token; } + + struct Hash { + size_t operator()(const RestToken& rest_token) const; + }; }; +/// File systems keyed by the credentials they were built from, so that the tables the +/// server issues the same credentials for share one file system. Sharing this cache +/// between the `RestTokenFileSystem` instances of many tables keeps a rotation of one +/// table's credentials from rebuilding the file systems of the others. +using RestTokenFileSystemCache = + GenericLruCache, RestToken::Hash>; + /// A `FileSystem` that accesses table data with the temporary credentials issued by the /// REST catalog for one table, reloading them before they expire. Every operation is /// delegated to the file system built from the credentials merged over the catalog @@ -61,18 +71,27 @@ class RestTokenFileSystem : public FileSystem { public: using Clock = std::function; - /// The file systems of the most recent credentials are retained so that a stream - /// opened just before a rotation is not left with a destroyed file system. - static constexpr size_t kMaxRetainedFileSystems = 4; + /// Bounds of the file system cache, matching the Java client: the file system of + /// credentials that were not used for this long is dropped, which also keeps a stream + /// opened just before a rotation from losing the file system it came from. + static constexpr int64_t kFileSystemCacheExpireAfterAccessMillis = 10 * 3600 * 1000; + static constexpr int64_t kMaxCachedFileSystems = 1000; + + /// Creates a cache the file systems of many tables can share. + static std::shared_ptr CreateFileSystemCache(); /// @param api Client of the catalog that issues the credentials. Shared because this /// file system commonly outlives the catalog it was obtained from. /// @param catalog_options Options the credentials are merged over. /// @param identifier The table the credentials are requested for. + /// @param fs_cache Cache of the delegates, shared with the file systems of the other + /// tables of the same catalog. A private one is created when null. /// @param clock Source of the current time, overridable for tests. RestTokenFileSystem(const std::shared_ptr& api, const std::map& catalog_options, - const Identifier& identifier, Clock clock = std::chrono::system_clock::now); + const Identifier& identifier, + std::shared_ptr fs_cache = nullptr, + Clock clock = std::chrono::system_clock::now); ~RestTokenFileSystem() override = default; @@ -92,7 +111,9 @@ class RestTokenFileSystem : public FileSystem { Result Exists(const std::string& path) const override; /// Returns credentials that are not about to expire, reloading them when needed. Lets - /// a caller that brings its own file system use the credentials of this table. + /// a caller that brings its own file system use the credentials of this table, so it + /// builds no file system of its own and needs nothing but the catalog options the + /// credentials are requested with. Result ValidToken() const; private: @@ -100,13 +121,17 @@ class RestTokenFileSystem : public FileSystem { /// expire in less than `RestApi::kTokenExpirationSafeTimeMillis`. Result> Delegate() const; - /// Reloads the credentials and builds their file system. Called with the write lock - /// of `mutex_` held. - Status Refresh() const; + /// Reloads the credentials from the server. Called with the write lock of `mutex_` + /// held. + Status RefreshToken() const; /// Whether `token_` is absent or expires within the safe time. bool ShouldRefresh() const; + /// Builds the file system of `token`: the credentials are the only file system options + /// that change, so it is built from the catalog options with them merged over. + Result> BuildFileSystem(const RestToken& token) const; + /// `catalog_options_` with `token` merged over it. std::map MergeTokenOptions( const std::map& token) const; @@ -114,14 +139,13 @@ class RestTokenFileSystem : public FileSystem { std::shared_ptr api_; std::map catalog_options_; Identifier identifier_; + std::shared_ptr fs_cache_; Clock clock_; std::shared_ptr logger_; + /// Guards `token_`. mutable std::shared_mutex mutex_; mutable std::optional token_; - mutable std::shared_ptr fs_; - /// Retains the file systems of the recent credentials, oldest first. - mutable std::deque> retained_fs_; }; } // namespace paimon diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index cad1ae62..f46e0a39 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -20,18 +20,24 @@ #include #include +#include #include #include #include #include #include #include +#include #include +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/catalog_options.h" +#include "paimon/common/factories/io_hook.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" #include "paimon/rest/rest_messages.h" @@ -51,6 +57,9 @@ struct MockTokenState { int64_t expires_at_millis = 0; // when set, the token endpoint fails with this http code std::optional force_error_code; + // when set, the token endpoint answers with this body and http 200, which lets a test + // return a malformed response without going through serialization + std::optional response_body; // guards all fields above: the handler runs on the server's accept thread while // tests seed and inspect the state std::mutex mutex; @@ -76,6 +85,10 @@ MockRestServer::Response HandleTokenRequest(MockTokenState* state, response.body = error.ToJsonString().value(); return response; } + if (state->response_body) { + response.body = state->response_body.value(); + return response; + } GetTableTokenResponse token(state->token, state->expires_at_millis); response.body = token.ToJsonString().value(); return response; @@ -85,6 +98,26 @@ int64_t ToMillis(std::chrono::system_clock::time_point time) { return std::chrono::duration_cast(time.time_since_epoch()).count(); } +// A local file system that runs a callback right after a file was created, so a test can +// rotate the credentials in the middle of a multi step operation without depending on +// thread scheduling or on waiting for real time to pass. +class RefreshOnCreateFileSystem : public LocalFileSystem { + public: + explicit RefreshOnCreateFileSystem(std::function on_create) + : on_create_(std::move(on_create)) {} + + Result> Create(const std::string& path, + bool overwrite) const override { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr out, + LocalFileSystem::Create(path, overwrite)); + on_create_(); + return out; + } + + private: + std::function on_create_; +}; + } // namespace class RestTokenFileSystemTest : public ::testing::Test { @@ -116,7 +149,8 @@ class RestTokenFileSystemTest : public ::testing::Test { } } - std::shared_ptr CreateFileSystem() { + std::shared_ptr CreateFileSystem( + std::shared_ptr fs_cache = nullptr) { Result> api = RestApi::Create(catalog_options_, "", /*config_required=*/false); if (!api.ok()) { @@ -124,7 +158,7 @@ class RestTokenFileSystemTest : public ::testing::Test { } std::shared_ptr shared_api(std::move(api).value()); return std::make_shared( - shared_api, catalog_options_, Identifier("db1", "t1"), [this] { + shared_api, catalog_options_, Identifier("db1", "t1"), std::move(fs_cache), [this] { return std::chrono::system_clock::time_point( std::chrono::milliseconds(now_millis_.load())); }); @@ -370,16 +404,364 @@ TEST_F(RestTokenFileSystemTest, ConcurrentFirstAccessLoadsTheTokenOnce) { ASSERT_EQ(1, state_->request_count.load()); } -TEST(RestTokenTest, OrdersByExpirationThenCredentials) { - RestToken early{{{"k", "v"}}, 1}; - RestToken late{{{"k", "v"}}, 2}; - ASSERT_TRUE(early < late); - ASSERT_FALSE(late < early); +TEST_F(RestTokenFileSystemTest, ValidTokenNeedsNoDelegate) { + // a caller that brings its own file system only needs the credentials, so an option + // the delegate cannot be built from must not keep it from getting them + catalog_options_[Options::MANIFEST_FORMAT] = "no-such-format"; + std::string path = WriteFile("data", "paimon"); + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); + ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); + ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); + + // a file operation does need the delegate and reports why it cannot be built + Status status = fs->Exists(path).status(); + ASSERT_NOK(status); + ASSERT_NOK_WITH_MSG(status, "no-such-format"); +} + +TEST_F(RestTokenFileSystemTest, FileSystemsOfEqualCredentialsAreShared) { + std::string path = WriteFile("data", "paimon"); + std::shared_ptr cache = RestTokenFileSystem::CreateFileSystemCache(); + std::shared_ptr first = CreateFileSystem(cache); + std::shared_ptr second = CreateFileSystem(cache); + ASSERT_NE(nullptr, first); + ASSERT_NE(nullptr, second); + + ASSERT_OK(first->Exists(path).status()); + ASSERT_OK(second->Exists(path).status()); + // both loaded credentials of their own, which are equal and so share one delegate + ASSERT_EQ(2, state_->request_count.load()); + ASSERT_EQ(1u, cache->Size()); + + // the delegate of the rotated credentials is added while the previous one is kept, so + // a stream opened just before the rotation does not lose the file system it came from + { + std::lock_guard lock(state_->mutex); + state_->token = {{"fs.oss.accessKeyId", "ak-2"}}; + state_->expires_at_millis = kExpiresAtMillis + RestApi::kTokenExpirationSafeTimeMillis; + } + now_millis_ = kExpiresAtMillis - 1; + ASSERT_OK(first->Exists(path).status()); + ASSERT_EQ(2u, cache->Size()); +} + +TEST_F(RestTokenFileSystemTest, MalformedTokenResponseDoesNotAccessBackend) { + catalog_options_["fs.oss.accessKeyId"] = "catalog-ak"; + std::string path = WriteFile("data", "original"); + const std::vector bodies = { + fmt::format(R"({{"expiresAtMillis":{}}})", kExpiresAtMillis), + fmt::format(R"({{"token":null,"expiresAtMillis":{}}})", kExpiresAtMillis)}; + for (bool warm_cache : {false, true}) { + for (const auto& body : bodies) { + SCOPED_TRACE(body); + SCOPED_TRACE(warm_cache); + now_millis_ = kNowMillis; + { + std::lock_guard lock(state_->mutex); + state_->response_body.reset(); + state_->expires_at_millis = kExpiresAtMillis; + } + std::shared_ptr cache = + RestTokenFileSystem::CreateFileSystemCache(); + std::shared_ptr fs = CreateFileSystem(cache); + ASSERT_NE(nullptr, fs); + if (warm_cache) { + ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(path)); + ASSERT_TRUE(exists); + // the loaded credentials have not expired yet but entered the refresh + // window, so a refresh that fails must not fall back to them + now_millis_ = kExpiresAtMillis - RestApi::kTokenExpirationSafeTimeMillis + 1; + } + { + std::lock_guard lock(state_->mutex); + state_->response_body = body; + } + int32_t requests_before = state_->request_count.load(); + // the hook counts every access of the local files, so it shows that a failed + // refresh never reaches the backend + IOHook* hook = IOHook::GetInstance(); + ScopeGuard guard([hook]() { hook->Clear(); }); + hook->Reset(0, IOHook::Mode::SILENT); + Status token_status = fs->ValidToken().status(); + ASSERT_TRUE(token_status.IsInvalid()) << token_status.ToString(); + Status read_status = fs->Exists(path).status(); + ASSERT_TRUE(read_status.IsInvalid()) << read_status.ToString(); + Status write_status = fs->WriteFile(path, "modified", /*overwrite=*/true); + ASSERT_TRUE(write_status.IsInvalid()) << write_status.ToString(); + ASSERT_EQ(0, hook->IOCount()); + ASSERT_EQ(warm_cache ? 1u : 0u, cache->Size()); + ASSERT_EQ(requests_before + 3, state_->request_count.load()); + hook->Clear(); + + std::string content; + ASSERT_OK(temp_dir_->GetFileSystem()->ReadFile(path, &content)); + ASSERT_EQ("original", content); + { + std::lock_guard lock(state_->mutex); + state_->response_body.reset(); + state_->expires_at_millis = + kExpiresAtMillis + RestApi::kTokenExpirationSafeTimeMillis; + } + ASSERT_OK(fs->ReadFile(path, &content)); + ASSERT_EQ("original", content); + ASSERT_EQ(requests_before + 4, state_->request_count.load()); + } + } +} + +TEST_F(RestTokenFileSystemTest, ExplicitEmptyTokenAllowsFileOperations) { + { + std::lock_guard lock(state_->mutex); + state_->token.clear(); + } + std::shared_ptr cache = RestTokenFileSystem::CreateFileSystemCache(); + std::shared_ptr fs = CreateFileSystem(cache); + ASSERT_NE(nullptr, fs); + ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); + ASSERT_TRUE(token.token.empty()); + ASSERT_EQ(0u, cache->Size()); + std::string path = temp_dir_->Str() + "/empty-token"; + ASSERT_OK(fs->WriteFile(path, "data", /*overwrite=*/false)); + std::string content; + ASSERT_OK(fs->ReadFile(path, &content)); + ASSERT_EQ("data", content); + ASSERT_EQ(1u, cache->Size()); + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, DefaultCacheEvictsAndRebuildsBackendWithoutReloadingToken) { + std::shared_ptr cache = RestTokenFileSystem::CreateFileSystemCache(); + ASSERT_EQ(1000, cache->GetMaxWeight()); + std::shared_ptr fs = CreateFileSystem(cache); + ASSERT_NE(nullptr, fs); + std::string path = WriteFile("data", "paimon"); + ASSERT_OK(fs->Exists(path).status()); + ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); + std::optional> cached = cache->GetIfPresent(token); + ASSERT_TRUE(cached.has_value()); + std::weak_ptr old_backend = cached.value(); + cached.reset(); + + // fill the cache with other credentials to reach its real default capacity; the file + // system asked for the credentials stays in use, only the delegate it built is evicted + std::shared_ptr filler = temp_dir_->GetFileSystem(); + for (int32_t i = 0; i < 1000; ++i) { + RestToken other{{{"entry", fmt::format("{}", i)}}, kExpiresAtMillis}; + ASSERT_OK(cache->Get(other, [&filler](const RestToken&) { return filler; }).status()); + } + ASSERT_EQ(1000u, cache->Size()); + ASSERT_FALSE(cache->GetIfPresent(token).has_value()); + ASSERT_TRUE(old_backend.expired()); + ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(path)); + ASSERT_TRUE(exists); + ASSERT_TRUE(cache->GetIfPresent(token).has_value()); + ASSERT_EQ(1000u, cache->Size()); + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, CacheExpirationDoesNotReloadValidToken) { + using RemovalCause = RestTokenFileSystemCache::RemovalCause; + std::vector causes; + std::vector> removed_backends; + RestTokenFileSystemCache::Options options; + // an idle time of zero expires every entry right away, which the production default of + // ten hours cannot do within a test + options.expire_after_access_ms = 0; + options.removal_callback = [&](const RestToken&, const std::shared_ptr& backend, + RemovalCause cause) { + causes.push_back(cause); + removed_backends.push_back(backend); + }; + std::shared_ptr cache = + std::make_shared(options); + std::shared_ptr fs = CreateFileSystem(cache); + ASSERT_NE(nullptr, fs); + std::string path = WriteFile("data", "paimon"); + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(bool exists, fs->Exists(path)); + ASSERT_TRUE(exists); + ASSERT_EQ(0u, cache->Size()); + } + ASSERT_EQ((std::vector{RemovalCause::EXPIRED, RemovalCause::EXPIRED}), causes); + ASSERT_EQ(2u, removed_backends.size()); + ASSERT_TRUE(removed_backends[0].expired()); + ASSERT_TRUE(removed_backends[1].expired()); + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, LocalStreamsSurviveRotationAndCacheEviction) { + RestTokenFileSystemCache::Options options; + options.max_weight = 1; + std::shared_ptr cache = + std::make_shared(options); + std::shared_ptr fs = CreateFileSystem(cache); + ASSERT_NE(nullptr, fs); + std::string path = WriteFile("data", "paimon"); + std::string output_path = temp_dir_->Str() + "/output"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr in, fs->Open(path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr out, + fs->Create(output_path, /*overwrite=*/false)); + ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); + std::optional> cached = cache->GetIfPresent(token); + ASSERT_TRUE(cached.has_value()); + std::weak_ptr old_backend = cached.value(); + cached.reset(); + { + std::lock_guard lock(state_->mutex); + state_->token = {{"fs.oss.accessKeyId", "ak-2"}}; + state_->expires_at_millis = kExpiresAtMillis + RestApi::kTokenExpirationSafeTimeMillis; + } + now_millis_ = kExpiresAtMillis - 1; + ASSERT_OK(fs->Exists(path).status()); + ASSERT_EQ(1u, cache->Size()); + ASSERT_FALSE(cache->GetIfPresent(token).has_value()); + ASSERT_TRUE(old_backend.expired()); + ASSERT_EQ(2, state_->request_count.load()); + fs.reset(); + cache.reset(); + + // the streams of the local file system own their file handle, so they outlive the file + // system they came from; other backends do not have to make that guarantee + std::string content(6, '\0'); + ASSERT_OK_AND_ASSIGN(int64_t read_length, in->Read(content.data(), content.size())); + ASSERT_EQ(6, read_length); + ASSERT_EQ("paimon", content); + ASSERT_OK(in->Close()); + ASSERT_OK_AND_ASSIGN(int64_t written, out->Write(content.data(), content.size())); + ASSERT_EQ(6, written); + ASSERT_OK(out->Flush()); + ASSERT_OK(out->Close()); + ASSERT_OK(temp_dir_->GetFileSystem()->ReadFile(output_path, &content)); + ASSERT_EQ("paimon", content); +} + +TEST_F(RestTokenFileSystemTest, HighLevelFileOperationsAcrossTokenRefresh) { + std::shared_ptr cache = RestTokenFileSystem::CreateFileSystemCache(); + std::shared_ptr fs = CreateFileSystem(cache); + ASSERT_NE(nullptr, fs); + ASSERT_OK_AND_ASSIGN(RestToken first_token, fs->ValidToken()); + bool created = false; + // `AtomicStore` writes a temporary file and renames it, so the rotation happens between + // the steps of one operation + std::shared_ptr delegate = std::make_shared([&]() { + created = true; + std::lock_guard lock(state_->mutex); + state_->token = {{"fs.oss.accessKeyId", "ak-2"}}; + state_->expires_at_millis = kExpiresAtMillis + RestApi::kTokenExpirationSafeTimeMillis; + now_millis_ = kExpiresAtMillis - 1; + }); + ASSERT_OK(cache->Get(first_token, [&delegate](const RestToken&) { return delegate; }).status()); + std::string path = temp_dir_->Str() + "/atomic"; + ASSERT_OK(fs->AtomicStore(path, "original")); + ASSERT_TRUE(created); + ASSERT_EQ(2, state_->request_count.load()); + ASSERT_OK_AND_ASSIGN(RestToken second_token, fs->ValidToken()); + ASSERT_EQ("ak-2", second_token.token.at("fs.oss.accessKeyId")); + ASSERT_TRUE(cache->GetIfPresent(second_token).has_value()); + std::string content; + ASSERT_OK(fs->ReadFile(path, &content)); + ASSERT_EQ("original", content); + ASSERT_OK(fs->WriteFile(path, "updated", /*overwrite=*/true)); + ASSERT_NOK(fs->WriteFile(path, "forbidden", /*overwrite=*/false)); + ASSERT_NOK(fs->AtomicStore(path, "forbidden")); + ASSERT_OK(fs->ReadFile(path, &content)); + ASSERT_EQ("updated", content); + std::vector files; + ASSERT_OK(fs->ListFileStatus(temp_dir_->Str(), &files)); + ASSERT_EQ(1u, files.size()); + ASSERT_EQ(path, files[0].GetPath()); + ASSERT_EQ(2, state_->request_count.load()); +} + +TEST_F(RestTokenFileSystemTest, ConcurrentFileSystemsShareOneCachedBackend) { + std::string path = WriteFile("data", "paimon"); + std::shared_ptr cache = RestTokenFileSystem::CreateFileSystemCache(); + constexpr size_t kThreads = 8; + std::vector> file_systems; + file_systems.reserve(kThreads); + for (size_t i = 0; i < kThreads; ++i) { + std::shared_ptr fs = CreateFileSystem(cache); + ASSERT_NE(nullptr, fs); + file_systems.push_back(std::move(fs)); + } + + std::atomic failures{0}; + std::vector threads; + threads.reserve(kThreads); + for (const std::shared_ptr& file_system : file_systems) { + threads.emplace_back([&failures, &path, file_system] { + Result exists = file_system->Exists(path); + if (!exists.ok() || !exists.value()) { + failures++; + } + }); + } + for (std::thread& thread : threads) { + thread.join(); + } + + ASSERT_EQ(0, failures.load()); + // every file system loads credentials of its own, but since the server issues equal ones + // to all of them, the delegates they ask the cache for converge to a single entry + ASSERT_EQ(static_cast(kThreads), state_->request_count.load()); + ASSERT_EQ(1u, cache->Size()); + ASSERT_OK_AND_ASSIGN(RestToken token, file_systems.front()->ValidToken()); + ASSERT_TRUE(cache->GetIfPresent(token).has_value()); +} + +TEST_F(RestTokenFileSystemTest, RefreshedTokenThatCannotBuildDelegateFails) { + std::string path = WriteFile("data", "paimon"); + std::shared_ptr cache = RestTokenFileSystem::CreateFileSystemCache(); + std::shared_ptr fs = CreateFileSystem(cache); + ASSERT_NE(nullptr, fs); + ASSERT_OK(fs->Exists(path).status()); + ASSERT_EQ(1u, cache->Size()); + + // the refreshed credentials name a file system that cannot be built + { + std::lock_guard lock(state_->mutex); + state_->token = {{Options::FILE_SYSTEM, "no-such-file-system"}}; + state_->expires_at_millis = kExpiresAtMillis + RestApi::kTokenExpirationSafeTimeMillis; + } + now_millis_ = kExpiresAtMillis - 1; + Status status = fs->Exists(path).status(); + ASSERT_NOK(status); + ASSERT_NOK_WITH_MSG(status, "no-such-file-system"); + ASSERT_EQ(2, state_->request_count.load()); + // the refreshed credentials replaced the previous ones, so the delegate that worked + // before is not served as a fallback + std::string content; + Status read_status = fs->ReadFile(path, &content); + ASSERT_NOK(read_status); + ASSERT_NOK_WITH_MSG(read_status, "no-such-file-system"); + ASSERT_EQ(2, state_->request_count.load()); + + // credentials the delegate can be built from recover the file system + { + std::lock_guard lock(state_->mutex); + state_->token = {{"fs.oss.accessKeyId", "ak-2"}}; + state_->expires_at_millis = kExpiresAtMillis + 2 * RestApi::kTokenExpirationSafeTimeMillis; + } + now_millis_ = kExpiresAtMillis + RestApi::kTokenExpirationSafeTimeMillis - 1; + ASSERT_OK(fs->ReadFile(path, &content)); + ASSERT_EQ("paimon", content); + ASSERT_EQ(3, state_->request_count.load()); +} + +TEST(RestTokenTest, EqualCredentialsShareOneFileSystemKey) { + RestToken token{{{"k", "v"}}, 1}; + RestToken same{{{"k", "v"}}, 1}; + ASSERT_TRUE(token == same); + ASSERT_EQ(RestToken::Hash()(token), RestToken::Hash()(same)); + // neither a later expiration nor other credentials may be served the same file system + RestToken later{{{"k", "v"}}, 2}; RestToken other_credentials{{{"k", "w"}}, 1}; - ASSERT_TRUE(early < other_credentials); - ASSERT_FALSE(other_credentials < early); - ASSERT_FALSE(early < early); + ASSERT_FALSE(token == later); + ASSERT_FALSE(token == other_credentials); } } // namespace paimon::test From 5f6ecfb68d69b08dcfd8b389c0edfb5c3ad57103 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 16 Sep 2026 09:47:58 +0800 Subject: [PATCH 03/16] fix --- src/paimon/rest/dlf_auth_test.cpp | 153 ++++++++++++++++++ .../rest/rest_token_file_system_test.cpp | 2 +- 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/src/paimon/rest/dlf_auth_test.cpp b/src/paimon/rest/dlf_auth_test.cpp index afb9cec5..5e82ff50 100644 --- a/src/paimon/rest/dlf_auth_test.cpp +++ b/src/paimon/rest/dlf_auth_test.cpp @@ -21,10 +21,14 @@ #include #include +#include +#include #include #include #include #include +#include +#include #include #include #include @@ -465,4 +469,153 @@ TEST(DlfAuthProviderTest, RejectsIncompleteOrUnknownConfiguration) { ASSERT_NOK_WITH_MSG(AuthProvider::Create(options).status(), "DLF region"); } +TEST(DlfDefaultSignerTest, SignsSpecialCharQueryLikeJava) { + // Golden produced by the Java DLFDefaultSigner for identical inputs; it locks the + // form-encoding of query values (space->'+', '+'->%2B, '='->%3D, '&'->%26, '%'->%25) + // that the default signer signs over. + DlfDefaultSigner signer("cn-hangzhou"); + const std::string body = R"({"name":"database"})"; + DlfToken token("access-key-id", "access-key-secret", "securityToken", std::nullopt); + RestAuthParameter parameter = RestAuthParameter::Create( + "POST", "/v1/paimon/databases", + {{"k1", "a b"}, {"k2", "x+y"}, {"k3", "p=q&r"}, {"k4", "100%"}}, body); + const std::chrono::system_clock::time_point signing_time = + std::chrono::system_clock::from_time_t(1701605532); // 2023-12-03T12:12:12Z + + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders(body, signing_time, token.GetSecurityToken(), "host")); + ASSERT_EQ("20231203T121212Z", headers.at("x-dlf-date")); + ASSERT_EQ("F2vHiexkcYvt4XOY5o4tmQ==", headers.at("Content-MD5")); + ASSERT_OK_AND_ASSIGN(std::string authorization, + signer.Authorization(parameter, token, "host", headers)); + ASSERT_EQ( + "DLF4-HMAC-SHA256 Credential=access-key-id/20231203/cn-hangzhou/" + "DlfNext/aliyun_v4_request,Signature=" + "3054a64554697227800ff06197e50b2c3de958f6b5584e3b531957207e685b4b", + authorization); +} + +TEST(DlfOpenApiSignerTest, OmitsBodyAndSecurityTokenHeadersWhenAbsent) { + DlfOpenApiSigner signer; + ASSERT_OK_AND_ASSIGN( + DlfRequestSigner::Headers headers, + signer.SignHeaders("", FixedTime(), std::nullopt, "dlfnext.cn-beijing.aliyuncs.com")); + ASSERT_EQ("Wed, 16 Apr 2025 03:44:46 GMT", headers.at("Date")); + ASSERT_EQ("application/json", headers.at("Accept")); + ASSERT_EQ("dlfnext.cn-beijing.aliyuncs.com", headers.at("Host")); + ASSERT_EQ("HMAC-SHA1", headers.at("x-acs-signature-method")); + ASSERT_EQ("1.0", headers.at("x-acs-signature-version")); + ASSERT_EQ("2026-01-18", headers.at("x-acs-version")); + ASSERT_FALSE(headers.at("x-acs-signature-nonce").empty()); + ASSERT_EQ(0, headers.count("Content-MD5")); + ASSERT_EQ(0, headers.count("Content-Type")); + ASSERT_EQ(0, headers.count("x-acs-security-token")); + ASSERT_EQ(7, headers.size()); +} + +TEST(DlfOpenApiSignerTest, FormatsGmtDateAcrossWeekdayAndMonthBounds) { + // Guards the fixed weekday/month tables used for the RFC 1123 GMT Date header. + const std::vector> cases = { + {1736060889, "Sun, 05 Jan 2025 07:08:09 GMT"}, // single-digit day, weekday index 0 + {1709251199, "Thu, 29 Feb 2024 23:59:59 GMT"}, // leap day + {1672444800, "Sat, 31 Dec 2022 00:00:00 GMT"}, // month index 11, weekday index 6 + {1635768000, "Mon, 01 Nov 2021 12:00:00 GMT"}, // month index 10, weekday index 1 + }; + DlfOpenApiSigner signer; + for (const auto& [epoch, expected] : cases) { + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders("", std::chrono::system_clock::from_time_t(epoch), + std::nullopt, "dlfnext.cn-hangzhou.aliyuncs.com")); + ASSERT_EQ(expected, headers.at("Date")); + } +} + +TEST(DlfOpenApiSignerTest, GeneratesUniqueUuidShapedNonce) { + DlfOpenApiSigner signer; + static const std::regex kUuidPattern( + "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); + static const std::regex kTimestampPattern("[0-9]{10,}"); + std::set nonces; + for (int32_t i = 0; i < 500; ++i) { + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders("", FixedTime(), std::nullopt, "host")); + const std::string nonce = headers.at("x-acs-signature-nonce"); + ASSERT_TRUE(std::regex_search(nonce, kUuidPattern)); + ASSERT_TRUE(std::regex_search(nonce, kTimestampPattern)); + nonces.insert(nonce); + } + ASSERT_EQ(500, nonces.size()); +} + +TEST(DlfOpenApiSignerTest, ConcurrentNonceGenerationStaysUnique) { + DlfOpenApiSigner signer; + std::set nonces; + std::vector statuses; + std::mutex mutex; + std::vector threads; + for (int32_t t = 0; t < 8; ++t) { + threads.emplace_back([&] { + for (int32_t i = 0; i < 50; ++i) { + Result headers = + signer.SignHeaders("", FixedTime(), std::nullopt, "host"); + std::scoped_lock lock(mutex); + statuses.push_back(headers.ok() ? Status::OK() : headers.status()); + if (headers.ok()) { + nonces.insert(headers.value().at("x-acs-signature-nonce")); + } + } + }); + } + for (std::thread& thread : threads) { + thread.join(); + } + ASSERT_EQ(400, statuses.size()); + for (const Status& status : statuses) { + ASSERT_OK(status); + } + ASSERT_EQ(400, nonces.size()); +} + +TEST(DlfSignerTest, ExposesJavaCompatibleIdentifiers) { + ASSERT_EQ(std::string("default"), std::string(DlfDefaultSigner::kIdentifier)); + ASSERT_EQ(std::string("openapi"), std::string(DlfOpenApiSigner::kIdentifier)); +} + +TEST(DlfAuthProviderTest, ParsesSigningAlgorithmFromUriBoundaries) { + ASSERT_EQ("openapi", + DlfAuthProvider::ParseSigningAlgorithmFromUri("dlfnext.cn-hangzhou.aliyuncs.com")); + ASSERT_EQ("openapi", DlfAuthProvider::ParseSigningAlgorithmFromUri( + "dlfnext-vpc.cn-hangzhou.aliyuncs.com")); + ASSERT_EQ("openapi", DlfAuthProvider::ParseSigningAlgorithmFromUri( + "https://dlfnext.cn-hangzhou.aliyuncs.com")); + ASSERT_EQ("default", + DlfAuthProvider::ParseSigningAlgorithmFromUri("cn-hangzhou-vpc.dlf.aliyuncs.com")); + ASSERT_EQ("default", DlfAuthProvider::ParseSigningAlgorithmFromUri( + "cn-hangzhou-intranet.dlf.aliyuncs.com")); + ASSERT_EQ("default", DlfAuthProvider::ParseSigningAlgorithmFromUri( + "https://cn-hangzhou-vpc.dlf.aliyuncs.com")); + ASSERT_EQ("default", DlfAuthProvider::ParseSigningAlgorithmFromUri("unknown.example.com")); + ASSERT_EQ("default", DlfAuthProvider::ParseSigningAlgorithmFromUri("127.0.0.1")); + ASSERT_EQ("default", DlfAuthProvider::ParseSigningAlgorithmFromUri("http://127.0.0.1:8080")); + ASSERT_EQ("default", DlfAuthProvider::ParseSigningAlgorithmFromUri("")); +} + +TEST(DlfAuthProviderTest, ParsesRegionFromUriVariants) { + const std::vector hangzhou_uris = { + "https://cn-hangzhou-vpc.dlf.aliyuncs.com", "https://cn-hangzhou-intranet.dlf.aliyuncs.com", + "https://cn-hangzhou.dlf.aliyuncs.com", "https://pre-cn-hangzhou-vpc.dlf.aliyuncs.com"}; + for (const std::string& uri : hangzhou_uris) { + ASSERT_OK_AND_ASSIGN(std::string region, DlfAuthProvider::ParseRegionFromUri(uri)); + ASSERT_EQ("cn-hangzhou", region); + } + const std::vector us_east_uris = {"https://us-east-1-vpc.dlf.aliyuncs.com", + "https://us-east-1-intranet.dlf.aliyuncs.com"}; + for (const std::string& uri : us_east_uris) { + ASSERT_OK_AND_ASSIGN(std::string region, DlfAuthProvider::ParseRegionFromUri(uri)); + ASSERT_EQ("us-east-1", region); + } + ASSERT_NOK_WITH_MSG(DlfAuthProvider::ParseRegionFromUri("http://127.0.0.1:8080").status(), + "could not determine DLF region"); +} + } // namespace paimon::test diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index f46e0a39..f52029a8 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -407,7 +407,7 @@ TEST_F(RestTokenFileSystemTest, ConcurrentFirstAccessLoadsTheTokenOnce) { TEST_F(RestTokenFileSystemTest, ValidTokenNeedsNoDelegate) { // a caller that brings its own file system only needs the credentials, so an option // the delegate cannot be built from must not keep it from getting them - catalog_options_[Options::MANIFEST_FORMAT] = "no-such-format"; + catalog_options_["manifest.format"] = "no-such-format"; std::string path = WriteFile("data", "paimon"); std::shared_ptr fs = CreateFileSystem(); ASSERT_NE(nullptr, fs); From 2dc96a5ca300d28382447cfa4a90e9a8d530ff02 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Wed, 16 Sep 2026 15:19:53 +0800 Subject: [PATCH 04/16] fix --- docs/source/user_guide/catalog.rst | 10 +++--- include/paimon/write_context.h | 9 +++--- .../core/operation/file_store_commit.cpp | 8 ++++- .../core/operation/file_store_commit_test.cpp | 31 +++++++++++++++++++ .../core/operation/file_store_write.cpp | 8 ++++- .../core/operation/file_store_write_test.cpp | 31 +++++++++++++++++++ src/paimon/testing/mock/mock_catalog.h | 23 ++++++++++++++ 7 files changed, 109 insertions(+), 11 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 42883905..ef2eb897 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -162,10 +162,12 @@ already advanced the latest snapshot, the requested snapshot must be published under the table path. Recreating the writer restores offsets from the catalog's current snapshot. -Both builders use the catalog's file system for manifests and data, preserving -its object-store credentials. ``WithFileSystem`` overrides it; writers also -allow ``WithFileSystemSchemeToIdentifierMap`` to override file-system selection. -For format tables, use ``WriteContextBuilder(FormatTable)`` instead. +Both builders use the file system of the table being written or committed for +manifests and data, so a catalog that issues temporary credentials per table +serves them through ``Catalog::GetTableFileSystem``. ``WithFileSystem`` overrides +it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override +file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` +instead. The snapshot being committed carries a uuid generated on the client, and the commit names the snapshot it is based on by that snapshot's uuid, so the server diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index e0a940eb..2cacb325 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -263,11 +263,10 @@ class PAIMON_EXPORT WriteContextBuilder { /// snapshot. Use the same catalog and identifier as `CommitContextBuilder` to restore and /// refresh real-time progress. /// - /// A branch is addressed by the identifier, as `tbl$branch_dev`, so that the catalog answers - /// for that branch. Its schema is read from the branch directory rather than from the catalog, - /// just as a read of that branch reads it. - /// Data, manifests and historical metadata remain on the file system. The catalog supplies - /// that file system unless overridden by `WithFileSystem()` or + /// Only the main branch is supported. Data, manifests and historical metadata remain on the + /// file system. The catalog supplies that table's file system - including the per-table + /// temporary credentials a catalog that issues them hands out through + /// `Catalog::GetTableFileSystem` - unless overridden by `WithFileSystem()` or /// `WithFileSystemSchemeToIdentifierMap()`. /// @param catalog Non-null catalog, kept alive while the writer uses its snapshot loader. /// @param identifier The native table to write to. diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 2fae23ba..db9cee5c 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -195,7 +195,13 @@ Result> FileStoreCommit::Create( // Use catalog credentials unless the caller supplied a file system. std::shared_ptr specific_fs = ctx->GetSpecificFileSystem(); if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) { - specific_fs = ctx->GetCatalog()->GetFileSystem(); + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // Use the credentials of this table, which a catalog issuing per-table temporary + // ones only hands out through GetTableFileSystem. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index 10b7c7aa..a7f39628 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -39,6 +39,7 @@ #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_increment.h" #include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" @@ -46,6 +47,7 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" +#include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" @@ -103,6 +105,35 @@ TEST(FileStoreCommitTest, TestCreateWithCatalogRequiresIdentifier) { "a catalog commit requires a table identifier"); } +TEST(FileStoreCommitTest, TestCatalogCommitUsesPerTableFileSystem) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + const Identifier identifier("foo", "bar"); + const auto logical_schema = arrow::schema( + {arrow::field("id", arrow::int64(), false), arrow::field("value", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, logical_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {{Options::FILE_FORMAT, "parquet"}})); + auto catalog = std::make_shared(); + catalog->SetTableSchema(schema); + // The catalog-wide file system is a different instance, so a commit that reached for it + // instead of the table's own would not be served the credentials the table needs. + catalog->SetFileSystem(std::make_shared()); + catalog->SetTableFileSystem(dir->GetFileSystem()); + + CommitContextBuilder builder(table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + builder.WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + + // The file system was resolved per table, not from the catalog-wide one. + const std::vector& requests = catalog->TableFileSystemRequests(); + ASSERT_FALSE(requests.empty()); + ASSERT_EQ(requests.back().GetFullName(), identifier.GetFullName()); +} + TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { auto string_field = arrow::field("f0", arrow::utf8()); auto int_field = arrow::field("f1", arrow::int32()); diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index 02121e6b..48f1c1dc 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -157,7 +157,13 @@ Result> FileStoreWrite::Create(std::unique_ptr specific_fs = ctx->GetSpecificFileSystem(); if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && ctx->GetCatalog() != nullptr) { - specific_fs = ctx->GetCatalog()->GetFileSystem(); + if (!ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx->GetOptions(), specific_fs, diff --git a/src/paimon/core/operation/file_store_write_test.cpp b/src/paimon/core/operation/file_store_write_test.cpp index 31356a0c..3893d857 100644 --- a/src/paimon/core/operation/file_store_write_test.cpp +++ b/src/paimon/core/operation/file_store_write_test.cpp @@ -222,6 +222,37 @@ TEST(FileStoreWriteTest, TestCatalogWriteAsksTheCatalogOnlyForTheMainBranchSchem ASSERT_EQ(catalog->LoadTableSchemaIdentifiers().size(), 1u); } +TEST(FileStoreWriteTest, TestCatalogWriteUsesPerTableFileSystem) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + const std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + const Identifier identifier("foo", "bar"); + const auto logical_schema = arrow::schema( + {arrow::field("id", arrow::int64(), false), arrow::field("value", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr schema, + TableSchema::Create(0, logical_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {{Options::FILE_FORMAT, "parquet"}})); + auto catalog = std::make_shared(); + catalog->SetTableSchema(schema); + // The catalog-wide file system is a different instance, so a write that reached for it + // instead of the table's own would not be served the credentials the table needs. + catalog->SetFileSystem(std::make_shared()); + catalog->SetTableFileSystem(dir->GetFileSystem()); + + WriteContextBuilder builder(table_path, "writer"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, + builder.WithCatalog(catalog, identifier).Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + FileStoreWrite::Create(std::move(ctx))); + ASSERT_OK(writer->Close()); + + // The file system was resolved per table, not from the catalog-wide one. + const std::vector& requests = catalog->TableFileSystemRequests(); + ASSERT_FALSE(requests.empty()); + ASSERT_EQ(requests.back().GetFullName(), identifier.GetFullName()); +} + TEST(FileStoreWriteTest, TestCreateWriterForLoadedMapBlobTable) { auto dir = UniqueTestDirectory::Create(); std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 3118cfa7..8f5a84c1 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -227,6 +227,27 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { std::shared_ptr GetFileSystem() const override { return file_system_; } + + /// Records what was asked and serves the per-table file system when one was set, so a + /// test can tell it apart from the catalog-wide one; falls back to `GetFileSystem()`, + /// matching the base default, when none was set. + Result> GetTableFileSystem( + const Identifier& identifier) const override { + table_file_system_requests_.push_back(identifier); + if (table_file_system_ != nullptr) { + return table_file_system_; + } + return GetFileSystem(); + } + + void SetTableFileSystem(const std::shared_ptr& file_system) { + table_file_system_ = file_system; + } + + const std::vector& TableFileSystemRequests() const { + return table_file_system_requests_; + } + const std::map& GetOptions() const override { return options_; } @@ -244,6 +265,8 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { std::string table_uuid_; std::shared_ptr table_schema_; std::shared_ptr file_system_; + std::shared_ptr table_file_system_; + mutable std::vector table_file_system_requests_; bool supports_version_management_ = true; std::function on_commit_; bool check_table_uuid_ = false; From ec7fc3f36d3f60ae68afabf5e57e9e35ae9ebc42 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 17 Sep 2026 17:35:03 +0800 Subject: [PATCH 05/16] fix --- docs/source/user_guide/catalog.rst | 126 +++++++++ include/paimon/catalog/catalog.h | 4 +- include/paimon/fs/credential_provider.h | 54 ++++ .../paimon/fs/credential_provider_factory.h | 65 +++++ include/paimon/read_context.h | 37 ++- include/paimon/scan_context.h | 37 ++- include/paimon/type_fwd.h | 1 + src/paimon/CMakeLists.txt | 4 + .../common/fs/credential_provider_factory.cpp | 61 +++++ .../fs/credential_provider_factory_test.cpp | 140 ++++++++++ src/paimon/common/fs/file_system_factory.cpp | 17 +- src/paimon/core/operation/read_context.cpp | 68 ++++- .../core/operation/read_context_test.cpp | 109 ++++++++ src/paimon/core/operation/scan_context.cpp | 91 +++++-- .../core/operation/scan_context_test.cpp | 99 +++++++ .../core/table/format/format_table_test.cpp | 18 ++ src/paimon/core/table/source/table_read.cpp | 5 + .../core/table/source/table_read_test.cpp | 32 +++ src/paimon/core/table/source/table_scan.cpp | 24 ++ .../core/table/source/table_scan_test.cpp | 50 ++++ src/paimon/fs/oss/oss_file_system_factory.cpp | 14 +- src/paimon/rest/rest_catalog.cpp | 1 + src/paimon/rest/rest_credential_provider.cpp | 109 ++++++++ src/paimon/rest/rest_credential_provider.h | 115 ++++++++ .../rest/rest_credential_provider_test.cpp | 252 ++++++++++++++++++ src/paimon/rest/rest_token_file_system.cpp | 73 +---- src/paimon/rest/rest_token_file_system.h | 43 +-- .../rest/rest_token_file_system_test.cpp | 1 + 28 files changed, 1507 insertions(+), 143 deletions(-) create mode 100644 include/paimon/fs/credential_provider.h create mode 100644 include/paimon/fs/credential_provider_factory.h create mode 100644 src/paimon/common/fs/credential_provider_factory.cpp create mode 100644 src/paimon/common/fs/credential_provider_factory_test.cpp create mode 100644 src/paimon/rest/rest_credential_provider.cpp create mode 100644 src/paimon/rest/rest_credential_provider.h create mode 100644 src/paimon/rest/rest_credential_provider_test.cpp diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index ef2eb897..fd4ef454 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -169,6 +169,23 @@ it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` instead. +A file system passed to ``WithFileSystem`` is used exactly as it is, so a custom +one that signs each request itself stays authenticated by drawing the table's +temporary credentials from ``Catalog::GetTableCredentialProvider``: hold the provider +and call ``CredentialProvider::GetCredentials`` at each access, which returns +credentials that are still valid and reloads them before they expire. A catalog +that issues no per-table credentials returns a null provider, and the custom file +system's own configuration authenticates the access. + +.. code-block:: cpp + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr provider, + shared_catalog->GetTableCredentialProvider(paimon::Identifier("db", "tbl"))); + // MyFileSystem merges provider->GetCredentials() over its own options per access. + auto my_fs = std::make_shared(provider); + write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) + .WithFileSystem(my_fs); + The snapshot being committed carries a uuid generated on the client, and the commit names the snapshot it is based on by that snapshot's uuid, so the server can tell a commit that raced another. A commit which lost such a race is @@ -235,6 +252,52 @@ so its restored file references are visible to the expiration operation. A branch the catalog holds without that directory, or one created while expiration runs, is not found: do not expire a table with such a branch, and serialize branch creation and expiration through the upstream coordinator. +======= +Reading through the catalog +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Pass the catalog and table identifier to ``ReadContextBuilder::WithCatalog`` or +``ScanContextBuilder::WithCatalog`` to read a native table the same way, in one +call: the table's schema comes from the catalog, its data is read through the +table's file system, and a scan reads the latest snapshot from the catalog: + +.. code-block:: cpp + + // Readers and scanners share ownership of the catalog. + std::shared_ptr shared_catalog(std::move(catalog)); + + paimon::ScanContextBuilder scan_builder(table_path); + scan_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + paimon::TableScan::Create(std::move(scan_context))); + + paimon::ReadContextBuilder read_builder(table_path); + read_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + paimon::TableRead::Create(std::move(read_context))); + +A scan takes only its latest snapshot from the catalog; the manifests that +snapshot points at and any historical snapshots still come from the table path. +Both builders read the table's schema from the catalog when ``Finish()`` builds +the context, which is one catalog request; call ``SetTableSchema`` with a schema +you already hold to skip it. The table's file system, including the per-table +temporary credentials a catalog serves through ``Catalog::GetTableFileSystem``, +is used for both. ``WithFileSystem`` overrides it, and a custom file system +that signs its own requests can draw the table's credentials from +``Catalog::GetTableCredentialProvider`` as shown above; the read builder also allows +``WithFileSystemSchemeToIdentifierMap``. + +Only the main branch is supported. Both builders reject other branches in the +identifier or ``branch`` option; the read builder also checks ``WithBranch``. +Explicit ``tbl$branch_main`` and ``branch=main`` are accepted. For a format +table, use ``ReadContextBuilder(FormatTable)`` or ``ScanContextBuilder(FormatTable)`` +instead; ``WithCatalog`` is rejected. A system table path such as ``tbl$snapshots`` +reads snapshots from under the table path, which a version-managed catalog +publishes nowhere but in itself, so it is rejected with ``NotImplemented``; scan +such a table with ``WithFileSystem(Catalog::GetTableFileSystem(...))`` instead. The C++ REST catalog covers the database, table, snapshot and commit operations of the ``Catalog`` API. The parts of the Java REST catalog that have no C++ @@ -298,3 +361,66 @@ identifier. The request body carries the table id but no table name and no branch, so the branch has to appear in the URL the caller sends that body to: the commit endpoint of ``tbl$branch_dev``, not the one of ``tbl``. A body sent to the bare table's URL publishes the branch's snapshot on the main branch. +======= +tags, branch management and consumers — are not supported. + +Authenticating with credentials of your own +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Credentials that a catalog does not issue — from a token service of your own, or +from a catalog Paimon has no client for — reach the built-in file systems through +a ``CredentialProvider``. Implement it together with a +``CredentialProviderFactory`` that builds it, register the factory with +``REGISTER_PAIMON_FACTORY``, and name it in the ``fs.credential.provider`` +option: the file system then builds the provider once and asks it for credentials +as it signs, so credentials are reloaded as they expire without the file system +being rebuilt. + +.. code-block:: cpp + + class MyCredentialProvider : public paimon::CredentialProvider { + public: + // Called as each access is signed; reload the credentials before they expire. + paimon::Result> GetCredentials() const override { + return std::map{ + {"fs.oss.accessKeyId", ...}, {"fs.oss.accessKeySecret", ...}, + {"fs.oss.securityToken", ...}}; + } + }; + + class MyCredentialProviderFactory : public paimon::CredentialProviderFactory { + public: + const char* Identifier() const override { + return "my-token-service"; + } + + paimon::Result> Create( + const std::string& path, + const std::map& options) const override { + return std::make_shared(path, options); + } + }; + + REGISTER_PAIMON_FACTORY(MyCredentialProviderFactory); + +The credentials are file-system options, e.g. ``fs.oss.securityToken``, and are +merged over the options the file system was configured with, so a provider serves +only what rotates and the endpoint and the rest stay in the options. The options +the factory is called with are those of the accesses to authenticate, which is +where an implementation reads its own configuration, such as the address of the +token service, from. + +.. code-block:: none + + fs.credential.provider=my-token-service + +Only the OSS file system consults a provider today; the others report +``NotImplemented`` when the option names one, rather than signing with the +credentials of the moment they were built and failing once those expire. All +factories share one identifier space, so an identifier a file system factory +already takes — ``oss``, ``s3``, ``local``, ``jindo`` — would replace it; name the +provider after where its credentials come from instead. + +This is the other of the two ways a provider is reached. A custom file system +passed to ``WithFileSystem`` signs its own requests and pulls from the provider +itself, which is what ``Catalog::GetTableCredentialProvider`` serves; a built-in +file system signs on your behalf and is pointed at a provider by this option. diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 40f52527..7f92dd0e 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -192,7 +192,9 @@ class PAIMON_EXPORT Catalog { /// @note A catalog that hands out per-table temporary credentials returns a file /// system that refreshes them, so the returned instance must be used for the /// table it was requested for. Pass it to `ReadContextBuilder::WithFileSystem`, - /// `ScanContextBuilder::WithFileSystem` or `WriteContextBuilder::WithFileSystem`. + /// `ScanContextBuilder::WithFileSystem` or `WriteContextBuilder::WithFileSystem`, + /// or let a builder request it for you through `ReadContextBuilder::WithCatalog`, + /// `ScanContextBuilder::WithCatalog` or `WriteContextBuilder::WithCatalog`. /// /// @param identifier The identifier (database and table name) of the table. /// @return A shared pointer to the file system instance; the catalog-level file system diff --git a/include/paimon/fs/credential_provider.h b/include/paimon/fs/credential_provider.h new file mode 100644 index 00000000..27db2f81 --- /dev/null +++ b/include/paimon/fs/credential_provider.h @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// The source of the credentials a file system authenticates its accesses with. +/// +/// Implement this to hand out credentials that expire: the provider is asked again at +/// every access, so it can reload them before they expire without the file system being +/// torn down and built again. +/// +/// A provider is for a caller that brings its own `FileSystem`. Build one with +/// `CredentialProviderFactory::Get`, consult it from every access of your file system, and +/// pass that file system in through `Catalog::Create`, `ReadContextBuilder::WithFileSystem`, +/// `ScanContextBuilder::WithFileSystem` or `WriteContextBuilder::WithFileSystem`. A file +/// system passed this way is used as-is; the built-in file systems are not involved and +/// authenticate with the static credentials of their own options. +/// +/// An implementation is consulted from every thread that accesses the file system and +/// has to be thread-safe. +class PAIMON_EXPORT CredentialProvider { + public: + virtual ~CredentialProvider() = default; + + /// Returns the credentials to sign an access with, reloading them when they are about + /// to expire. The keys are file system options, e.g. "fs.oss.accessKeyId" or + /// "fs.oss.securityToken", so a caller merges them over its own file system options. + virtual Result> GetCredentials() const = 0; +}; + +} // namespace paimon diff --git a/include/paimon/fs/credential_provider_factory.h b/include/paimon/fs/credential_provider_factory.h new file mode 100644 index 00000000..ff373e46 --- /dev/null +++ b/include/paimon/fs/credential_provider_factory.h @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/factories/factory.h" +#include "paimon/fs/credential_provider.h" +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// A factory for creating `CredentialProvider` instances. +/// +/// Register an implementation with `REGISTER_PAIMON_FACTORY` to authenticate accesses +/// with credentials of your own making: the built-in file systems build the provider +/// named by their `fs.credential.provider` option and consult it whenever they sign. +/// +/// @note All factories share one identifier space, so an identifier that a file system +/// factory already takes - "oss", "s3", "local", "jindo" - would replace it. Name the +/// provider after where its credentials come from instead. +class PAIMON_EXPORT CredentialProviderFactory : public Factory { + public: + /// The option naming the factory a file system builds its provider with. + static const char CREDENTIAL_PROVIDER_OPTION[]; + + /// Create a `CredentialProvider` of current factory for the accesses below a path. + /// + /// The options are the file system options of the accesses to authenticate, so an + /// implementation reads its own configuration out of them. + virtual Result> Create( + const std::string& path, const std::map& options) const = 0; + + /// Get a `CredentialProvider` corresponding to identifier for the accesses below a path. + /// @pre Factory is already registered. + static Result> Get( + const std::string& identifier, const std::string& path, + const std::map& fs_options); + + /// Get the `CredentialProvider` the options ask for, or a null provider when they name + /// none, in which case the file system options themselves authenticate the accesses. + static Result> GetIfConfigured( + const std::string& path, const std::map& fs_options); +}; + +} // namespace paimon diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index e24a0e7d..f2fff8cd 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -27,6 +27,7 @@ #include "arrow/c/abi.h" #include "paimon/cache/cache.h" +#include "paimon/catalog/identifier.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -34,6 +35,7 @@ #include "paimon/visibility.h" namespace paimon { +class Catalog; class Executor; class FormatTable; class MemoryPool; @@ -62,7 +64,9 @@ class PAIMON_EXPORT ReadContext { const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level); + const std::shared_ptr& format_table, WarmupLevel warmup_level, + const std::shared_ptr& catalog, + const std::optional& identifier); ~ReadContext(); @@ -115,6 +119,8 @@ class PAIMON_EXPORT ReadContext { uint32_t GetRowToBatchThreadNumber() const { return row_to_batch_thread_number_; } + /// The schema this context reads with, or null when it has to be read from the table path. + /// `WithCatalog()` resolves it to the catalog's schema when `SetTableSchema()` did not answer. const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -155,6 +161,16 @@ class PAIMON_EXPORT ReadContext { return warmup_level_; } + /// Returns the catalog supplying table metadata, or null if unset. + const std::shared_ptr& GetCatalog() const { + return catalog_; + } + + /// Returns the table identifier supplied with the catalog. + const std::optional& GetIdentifier() const { + return identifier_; + } + /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. bool HasReadSchema() const { return read_schema_ != nullptr && read_schema_->release != nullptr; @@ -196,6 +212,8 @@ class PAIMON_EXPORT ReadContext { std::shared_ptr cache_; std::shared_ptr format_table_; WarmupLevel warmup_level_; + std::shared_ptr catalog_; + std::optional identifier_; // Owns schema resources and releases ArrowSchema::release in destructor. std::unique_ptr read_schema_; }; @@ -210,8 +228,8 @@ class PAIMON_EXPORT ReadContextBuilder { /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way /// to read one whose schema lives in a metastore rather than under its location, such as a /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a - /// branch are refused here rather than ignored. + /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()`, + /// `WithCatalog()` and a branch are refused here rather than ignored. /// /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. explicit ReadContextBuilder(const std::shared_ptr& table); @@ -478,6 +496,19 @@ class PAIMON_EXPORT ReadContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ReadContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Loads a native table's schema from the catalog and reads its data through that table's file + /// system - including the per-table temporary credentials a catalog that issues them hands out + /// through `Catalog::GetTableFileSystem`. + /// + /// Only the main branch is supported. `SetTableSchema()` keeps the caller's schema and spares + /// the catalog request; `WithFileSystem()` and `WithFileSystemSchemeToIdentifierMap()` override + /// the file system. + /// @param catalog Non-null catalog, read when `Finish()` builds the context. + /// @param identifier The native table to read. + /// @return Reference to this builder for method chaining. + ReadContextBuilder& WithCatalog(const std::shared_ptr& catalog, + const Identifier& identifier); + /// Inject a cache for read operations. Passing nullptr disables cache. /// @return Reference to this builder for method chaining. ReadContextBuilder& WithCache(const std::shared_ptr& cache); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 28906253..8122d7dd 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -26,6 +26,7 @@ #include #include "paimon/cache/cache.h" +#include "paimon/catalog/identifier.h" #include "paimon/global_index/global_index_result.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" @@ -34,6 +35,7 @@ namespace paimon { class ScanContextBuilder; class ScanFilter; +class Catalog; class Executor; class FormatTable; class MemoryPool; @@ -56,7 +58,9 @@ class PAIMON_EXPORT ScanContext { const std::optional& table_schema, const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table); + const std::shared_ptr& format_table, + const std::shared_ptr& catalog, + const std::optional& identifier); ~ScanContext(); @@ -99,6 +103,8 @@ class PAIMON_EXPORT ScanContext { return specific_file_system_; } + /// The schema this context scans with, or null when it has to be read from the table path. + /// `WithCatalog()` resolves it to the catalog's schema when `SetTableSchema()` did not answer. const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -113,6 +119,16 @@ class PAIMON_EXPORT ScanContext { return format_table_; } + /// Returns the catalog supplying table metadata, or null if unset. + const std::shared_ptr& GetCatalog() const { + return catalog_; + } + + /// Returns the table identifier supplied with the catalog. + const std::optional& GetIdentifier() const { + return identifier_; + } + private: std::string path_; bool is_streaming_mode_; @@ -127,6 +143,8 @@ class PAIMON_EXPORT ScanContext { std::map options_; std::shared_ptr cache_; std::shared_ptr format_table_; + std::shared_ptr catalog_; + std::optional identifier_; }; /// Filter configuration for table scan operations @@ -165,7 +183,8 @@ class PAIMON_EXPORT ScanContextBuilder { /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way /// to scan one whose schema lives in a metastore rather than under its location, such as a /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. + /// `SetTableSchema()`, `WithFileSystem()` and `WithCatalog()` are refused here rather than + /// ignored. /// /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. explicit ScanContextBuilder(const std::shared_ptr& table); @@ -222,6 +241,20 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Loads a native table's schema from the catalog and plans its data through that table's file + /// system - including the per-table temporary credentials a catalog that issues them hands out + /// through `Catalog::GetTableFileSystem`. A catalog which publishes snapshots through catalog + /// commits is also where the latest snapshot is read from. + /// + /// Only the main branch is supported. `SetTableSchema()` keeps the caller's schema and spares + /// the catalog request; `WithFileSystem()` overrides the file system. + /// @param catalog Non-null catalog, read when `Finish()` builds the context and kept alive + /// while the scan uses its snapshot loader. + /// @param identifier The native table to scan. + /// @return Reference to this builder for method chaining. + ScanContextBuilder& WithCatalog(const std::shared_ptr& catalog, + const Identifier& identifier); + /// Set the table schema as a string to avoid schema loading I/O operations. /// /// This optimization allows the scanner to use a pre-loaded schema instead of diff --git a/include/paimon/type_fwd.h b/include/paimon/type_fwd.h index 6656ff0c..e469934c 100644 --- a/include/paimon/type_fwd.h +++ b/include/paimon/type_fwd.h @@ -54,6 +54,7 @@ class OutputStream; class InputStream; class FileStatus; class BasicFileStatus; +class CredentialProvider; class RecordBatch; class RealtimeContext; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 35b613fa..0f3b2d9a 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -77,6 +77,7 @@ set(PAIMON_COMMON_SRCS common/format/file_format_factory.cpp common/fs/file_system.cpp common/fs/resolving_file_system.cpp + common/fs/credential_provider_factory.cpp common/fs/file_system_factory.cpp common/global_index/union_global_index_reader.cpp common/global_index/offset_global_index_reader.cpp @@ -489,6 +490,7 @@ if(PAIMON_ENABLE_REST) rest/rest_catalog.cpp rest/rest_messages.cpp rest/rest_token_file_system.cpp + rest/rest_credential_provider.cpp rest/rest_util.cpp) endif() @@ -1010,6 +1012,7 @@ if(PAIMON_BUILD_TESTS) add_paimon_test(fs_test SOURCES common/fs/file_system_test.cpp + common/fs/credential_provider_factory_test.cpp common/fs/resolving_file_system_test.cpp ${PAIMON_OBJECT_STORE_FS_TEST_SOURCES} fs/local/local_file_test.cpp @@ -1033,6 +1036,7 @@ if(PAIMON_BUILD_TESTS) rest/rest_catalog_test.cpp rest/rest_messages_test.cpp rest/rest_token_file_system_test.cpp + rest/rest_credential_provider_test.cpp rest/rest_util_test.cpp STATIC_LINK_LIBS paimon_shared diff --git a/src/paimon/common/fs/credential_provider_factory.cpp b/src/paimon/common/fs/credential_provider_factory.cpp new file mode 100644 index 00000000..dd2c3e0b --- /dev/null +++ b/src/paimon/common/fs/credential_provider_factory.cpp @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/fs/credential_provider_factory.h" + +#include "fmt/format.h" +#include "paimon/factories/factory_creator.h" +#include "paimon/status.h" + +namespace paimon { + +const char CredentialProviderFactory::CREDENTIAL_PROVIDER_OPTION[] = "fs.credential.provider"; + +Result> CredentialProviderFactory::Get( + const std::string& identifier, const std::string& path, + const std::map& fs_options) { + auto factory_creator = FactoryCreator::GetInstance(); + auto factory = factory_creator->Create(identifier); + if (factory == nullptr) { + return Status::Invalid( + fmt::format("Create factory failed with identifier '{}'.", identifier)); + } + auto credential_provider_factory = dynamic_cast(factory); + if (credential_provider_factory == nullptr) { + return Status::Invalid(fmt::format( + "Failed to cast credential provider factory with identifier '{}'.", identifier)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr provider, + credential_provider_factory->Create(path, fs_options)); + if (provider == nullptr) { + return Status::Invalid( + fmt::format("Credential provider factory '{}' created a null provider.", identifier)); + } + return provider; +} + +Result> CredentialProviderFactory::GetIfConfigured( + const std::string& path, const std::map& fs_options) { + auto option = fs_options.find(CREDENTIAL_PROVIDER_OPTION); + if (option == fs_options.end() || option->second.empty()) { + return std::shared_ptr(nullptr); + } + return Get(option->second, path, fs_options); +} + +} // namespace paimon diff --git a/src/paimon/common/fs/credential_provider_factory_test.cpp b/src/paimon/common/fs/credential_provider_factory_test.cpp new file mode 100644 index 00000000..21bccba7 --- /dev/null +++ b/src/paimon/common/fs/credential_provider_factory_test.cpp @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/fs/credential_provider_factory.h" + +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/fs/local/local_file_system_factory.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +/// Credentials read straight out of the options of the path they authenticate, which is +/// enough to tell whether the factory handed them the path and options it was asked with. +class PathCredentialProvider : public CredentialProvider { + public: + PathCredentialProvider(std::string path, std::string secret) + : path_(std::move(path)), secret_(std::move(secret)) {} + + Result> GetCredentials() const override { + return std::map{{"path", path_}, {"secret", secret_}}; + } + + private: + std::string path_; + std::string secret_; +}; + +class TestCredentialProviderFactory : public CredentialProviderFactory { + public: + static const char IDENTIFIER[]; + + const char* Identifier() const override { + return IDENTIFIER; + } + + Result> Create( + const std::string& path, const std::map& options) const override { + auto secret = options.find("test.secret"); + if (secret == options.end()) { + return Status::Invalid("option 'test.secret' is required"); + } + if (secret->second == "none") { + return std::shared_ptr(nullptr); + } + return std::make_shared(path, secret->second); + } +}; + +const char TestCredentialProviderFactory::IDENTIFIER[] = "test-credential-provider"; + +} // namespace + +REGISTER_PAIMON_FACTORY(TestCredentialProviderFactory); + +TEST(CredentialProviderFactoryTest, TestGet) { + std::map options{{"test.secret", "token"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr provider, + CredentialProviderFactory::Get(TestCredentialProviderFactory::IDENTIFIER, + "oss://bucket/table", options)); + ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + ASSERT_EQ(credentials["path"], "oss://bucket/table"); + ASSERT_EQ(credentials["secret"], "token"); +} + +TEST(CredentialProviderFactoryTest, TestGetUnregisteredIdentifier) { + ASSERT_NOK_WITH_MSG( + CredentialProviderFactory::Get("no-such-provider", "oss://bucket/table", {}).status(), + "Create factory failed with identifier 'no-such-provider'"); +} + +TEST(CredentialProviderFactoryTest, TestGetIdentifierOfAnotherFactoryKind) { + // All factories share one identifier space, so asking for one that is not a credential + // provider factory has to be reported rather than mistaken for one. + ASSERT_NOK_WITH_MSG( + CredentialProviderFactory::Get(LocalFileSystemFactory::IDENTIFIER, "/tmp/table", {}) + .status(), + "Failed to cast credential provider factory"); +} + +TEST(CredentialProviderFactoryTest, TestGetPropagatesFactoryFailure) { + ASSERT_NOK_WITH_MSG(CredentialProviderFactory::Get(TestCredentialProviderFactory::IDENTIFIER, + "oss://bucket/table", {}) + .status(), + "option 'test.secret' is required"); +} + +TEST(CredentialProviderFactoryTest, TestGetRejectsNullProvider) { + std::map options{{"test.secret", "none"}}; + ASSERT_NOK_WITH_MSG(CredentialProviderFactory::Get(TestCredentialProviderFactory::IDENTIFIER, + "oss://bucket/table", options) + .status(), + "created a null provider"); +} + +TEST(CredentialProviderFactoryTest, TestGetIfConfigured) { + std::map options{ + {CredentialProviderFactory::CREDENTIAL_PROVIDER_OPTION, + TestCredentialProviderFactory::IDENTIFIER}, + {"test.secret", "token"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr provider, + CredentialProviderFactory::GetIfConfigured("oss://bucket/table", options)); + ASSERT_NE(provider, nullptr); + ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); + ASSERT_EQ(credentials["secret"], "token"); +} + +TEST(CredentialProviderFactoryTest, TestGetIfConfiguredWithoutOption) { + // No provider named means the file system options authenticate the accesses themselves. + ASSERT_OK_AND_ASSIGN(std::shared_ptr provider, + CredentialProviderFactory::GetIfConfigured("/tmp/table", {})); + ASSERT_EQ(provider, nullptr); + + std::map empty_option{ + {CredentialProviderFactory::CREDENTIAL_PROVIDER_OPTION, ""}}; + ASSERT_OK_AND_ASSIGN(provider, + CredentialProviderFactory::GetIfConfigured("/tmp/table", empty_option)); + ASSERT_EQ(provider, nullptr); +} + +} // namespace paimon::test diff --git a/src/paimon/common/fs/file_system_factory.cpp b/src/paimon/common/fs/file_system_factory.cpp index 5c8da3e0..40c1d73b 100644 --- a/src/paimon/common/fs/file_system_factory.cpp +++ b/src/paimon/common/fs/file_system_factory.cpp @@ -24,9 +24,9 @@ namespace paimon { -Result> FileSystemFactory::Get( - const std::string& identifier, const std::string& path, - const std::map& fs_options) { +namespace { + +Result LookUpFactory(const std::string& identifier) { auto factory_creator = FactoryCreator::GetInstance(); auto factory = factory_creator->Create(identifier); if (factory == nullptr) { @@ -38,7 +38,16 @@ Result> FileSystemFactory::Get( return Status::Invalid( fmt::format("Failed to cast file system factory with identifier '{}'.", identifier)); } - return file_system_factory->Create(path, fs_options); + return file_system_factory; +} + +} // namespace + +Result> FileSystemFactory::Get( + const std::string& identifier, const std::string& path, + const std::map& fs_options) { + PAIMON_ASSIGN_OR_RAISE(FileSystemFactory * factory, LookUpFactory(identifier)); + return factory->Create(path, fs_options); } } // namespace paimon diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index ec6d65b8..6f685898 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -22,10 +22,15 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" +#include "paimon/schema/schema.h" #include "paimon/status.h" #include "paimon/table/format/format_table.h" @@ -45,7 +50,8 @@ ReadContext::ReadContext( const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level) + const std::shared_ptr& format_table, WarmupLevel warmup_level, + const std::shared_ptr& catalog, const std::optional& identifier) : path_(path), branch_(branch), read_field_names_(read_field_names), @@ -69,7 +75,9 @@ ReadContext::ReadContext( cache_config_(cache_config), cache_(cache), format_table_(format_table), - warmup_level_(warmup_level) {} + warmup_level_(warmup_level), + catalog_(catalog), + identifier_(identifier) {} ReadContext::~ReadContext() { if (read_schema_ && read_schema_->release) { @@ -116,6 +124,8 @@ class ReadContextBuilder::Impl { cache_config_ = CacheConfig(); cache_.reset(); warmup_level_ = WarmupLevel::RAW; + catalog_.reset(); + identifier_.reset(); } private: @@ -149,6 +159,8 @@ class ReadContextBuilder::Impl { CacheConfig cache_config_; std::shared_ptr cache_; WarmupLevel warmup_level_ = WarmupLevel::RAW; + std::shared_ptr catalog_; + std::optional identifier_; }; ReadContextBuilder::ReadContextBuilder(const std::string& path) @@ -279,6 +291,13 @@ ReadContextBuilder& ReadContextBuilder::WithFileSystem( return *this; } +ReadContextBuilder& ReadContextBuilder::WithCatalog(const std::shared_ptr& catalog, + const Identifier& identifier) { + impl_->catalog_ = catalog; + impl_->identifier_.emplace(identifier); + return *this; +} + ReadContextBuilder& ReadContextBuilder::SetReadAheadCacheEnabled(bool enabled) { impl_->read_ahead_cache_enabled_ = enabled; return *this; @@ -306,6 +325,11 @@ Result> ReadContextBuilder::Finish() { if (impl_->format_table_ != nullptr) { // The table already answers each of these, and from a source this cannot see behind, so a // second answer is refused rather than silently dropped. + if (impl_->catalog_ != nullptr) { + return Status::Invalid( + "a format table carries its own schema and the file system it was loaded through, " + "so WithCatalog() cannot be used with one; use ReadContextBuilder(FormatTable)"); + } if (impl_->table_schema_) { return Status::Invalid( "a format table carries its own schema, so SetTableSchema() cannot be used with " @@ -354,6 +378,44 @@ Result> ReadContextBuilder::Finish() { if (impl_->enable_multi_thread_row_to_batch_ && impl_->row_to_batch_thread_number_ <= 0) { return Status::Invalid("row to batch thread number should be greater than 0"); } + if (impl_->catalog_ == nullptr && impl_->identifier_) { + return Status::Invalid("cannot read through a null catalog"); + } + if (impl_->catalog_ != nullptr) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(impl_->catalog_)); + PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, + impl_->identifier_->GetBranchName()); + auto branch_option = impl_->options_.find(Options::BRANCH); + std::optional option_branch = + branch_option == impl_->options_.end() + ? std::nullopt + : std::optional(branch_option->second); + for (const std::optional& branch : + {identifier_branch, option_branch, std::optional(impl_->branch_)}) { + if (branch && + !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(branch.value()))) { + return Status::Invalid(fmt::format( + "a read through a catalog reads the main branch of the table, so it cannot be " + "aimed at branch '{}'", + branch.value())); + } + } + if (impl_->specific_file_system_ == nullptr && + impl_->fs_scheme_to_identifier_map_.empty()) { + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the read uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, impl_->identifier_->IsSystemTable()); + // A system table derives its schema from the table it reads rather than keeping one in the + // catalog, so that schema is left to be read from under the table path. + if (!impl_->table_schema_ && !is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + impl_->catalog_->LoadTableSchema(impl_->identifier_.value())); + PAIMON_ASSIGN_OR_RAISE(impl_->table_schema_, schema->GetJsonSchema()); + } + } auto ctx = std::make_unique( impl_->path_, branch, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, impl_->enable_predicate_filter_, impl_->enable_prefetch_, impl_->enable_late_materializing_, @@ -362,7 +424,7 @@ Result> ReadContextBuilder::Finish() { impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, impl_->format_table_, - impl_->warmup_level_); + impl_->warmup_level_, impl_->catalog_, impl_->identifier_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index 62e09423..b00f2a37 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -24,16 +24,123 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/common/io/cache/lru_cache.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/status.h" +#include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { + +/// A catalog serving one table schema and a file system of its own, so a test can tell the +/// per-table answers apart from the catalog-wide ones. +std::shared_ptr CatalogServing( + const std::shared_ptr& table_file_system, std::string* schema_json) { + auto logical_schema = arrow::schema( + {arrow::field("id", arrow::int64(), false), arrow::field("value", arrow::utf8())}); + EXPECT_OK_AND_ASSIGN(std::shared_ptr schema, + TableSchema::Create(0, logical_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {})); + EXPECT_OK_AND_ASSIGN(*schema_json, schema->GetJsonSchema()); + auto catalog = std::make_shared(); + catalog->SetTableSchema(schema); + catalog->SetTableFileSystem(table_file_system); + return catalog; +} + +} // namespace + +TEST(ReadContextTest, TestWithCatalog) { + auto table_fs = std::make_shared(); + std::string schema_json; + std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + + ReadContextBuilder builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); + ASSERT_EQ(ctx->GetCatalog(), catalog); + ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); + // The file system a catalog issuing per-table credentials hands out for this table, and the + // schema the metastore keeps, are both resolved when the context is built. + ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_EQ(ctx->GetSpecificTableSchema(), std::optional(schema_json)); + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 1U); + + // Finish() resets the builder, so the next context reads without the catalog. + ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); + ASSERT_FALSE(next_ctx->GetCatalog()); + ASSERT_FALSE(next_ctx->GetIdentifier()); + ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); + ASSERT_FALSE(next_ctx->GetSpecificTableSchema()); + + ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), + "cannot read through a null catalog"); +} + +TEST(ReadContextTest, TestCatalogAnswersAreOverridable) { + auto table_fs = std::make_shared(); + std::string schema_json; + std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + auto given_fs = std::make_shared(); + + // A file system the caller gave is the one that is used, and the catalog is not asked for one. + ReadContextBuilder fs_builder("table_root_path"); + ASSERT_OK_AND_ASSIGN( + auto fs_ctx, + fs_builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); + ASSERT_EQ(fs_ctx->GetSpecificFileSystem(), given_fs); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); + + // So is a scheme map, which names file systems the same way for the paths it covers. + ReadContextBuilder map_builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto map_ctx, map_builder.WithCatalog(catalog, Identifier("db1", "t1")) + .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) + .Finish()); + ASSERT_FALSE(map_ctx->GetSpecificFileSystem()); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); + + // A schema the caller gave spares the catalog request entirely. A fresh catalog keeps the + // count clean of the requests the schema-less builders above already made. + std::string schema_catalog_json; + std::shared_ptr schema_catalog = + CatalogServing(table_fs, &schema_catalog_json); + ReadContextBuilder schema_builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto schema_ctx, + schema_builder.WithCatalog(schema_catalog, Identifier("db1", "t1")) + .SetTableSchema("table-schema-json") + .Finish()); + ASSERT_EQ(schema_ctx->GetSpecificTableSchema(), + std::optional("table-schema-json")); + ASSERT_EQ(schema_catalog->LoadTableSchemaCalls(), 0U); +} + +TEST(ReadContextTest, TestCatalogReadsTheMainBranchOnly) { + auto table_fs = std::make_shared(); + std::string schema_json; + std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + for (int32_t branch_source = 0; branch_source < 3; ++branch_source) { + SCOPED_TRACE(branch_source); + ReadContextBuilder builder("table_root_path"); + builder.WithCatalog(catalog, Identifier("db1", branch_source == 0 ? "t1$branch_dev" : "t1")) + .WithBranch(branch_source == 1 ? "dev" : "main") + .AddOption(Options::BRANCH, branch_source == 2 ? "dev" : "main"); + ASSERT_NOK_WITH_MSG(builder.Finish(), "it cannot be aimed at branch 'dev'"); + } + + ReadContextBuilder main_builder("table_root_path"); + ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) + .WithBranch("") + .AddOption(Options::BRANCH, "main") + .Finish()); +} + TEST(ReadContextTest, TestDefaultValue) { ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -55,6 +162,8 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_EQ("main", ctx->GetBranch()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_FALSE(ctx->GetCatalog()); + ASSERT_FALSE(ctx->GetIdentifier()); } TEST(ReadContextTest, TestSetContent) { diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 6bc80fe7..88c58c66 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -20,27 +20,32 @@ #include +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" +#include "paimon/schema/schema.h" #include "paimon/status.h" #include "paimon/table/format/format_table.h" namespace paimon { class Predicate; -ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, - std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, - const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, - const std::shared_ptr& cache, - const std::shared_ptr& format_table) +ScanContext::ScanContext( + const std::string& path, bool is_streaming_mode, std::optional limit, + const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, + const std::shared_ptr& realtime_context, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::optional& table_schema, + const std::map& options, const std::shared_ptr& cache, + const std::shared_ptr& format_table, const std::shared_ptr& catalog, + const std::optional& identifier) : path_(path), is_streaming_mode_(is_streaming_mode), limit_(limit), @@ -53,7 +58,9 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, table_schema_(table_schema), options_(options), cache_(cache), - format_table_(format_table) {} + format_table_(format_table), + catalog_(catalog), + identifier_(identifier) {} ScanContext::~ScanContext() = default; @@ -75,6 +82,8 @@ class ScanContextBuilder::Impl { table_schema_ = std::nullopt; options_.clear(); cache_.reset(); + catalog_.reset(); + identifier_.reset(); } private: @@ -97,6 +106,8 @@ class ScanContextBuilder::Impl { std::optional table_schema_; std::map options_; std::shared_ptr cache_; + std::shared_ptr catalog_; + std::optional identifier_; }; ScanContextBuilder::ScanContextBuilder(const std::string& path) @@ -181,6 +192,13 @@ ScanContextBuilder& ScanContextBuilder::WithFileSystem( return *this; } +ScanContextBuilder& ScanContextBuilder::WithCatalog(const std::shared_ptr& catalog, + const Identifier& identifier) { + impl_->catalog_ = catalog; + impl_->identifier_.emplace(identifier); + return *this; +} + ScanContextBuilder& ScanContextBuilder::SetTableSchema(const std::string& table_schema) { impl_->table_schema_ = table_schema; return *this; @@ -196,8 +214,13 @@ Result> ScanContextBuilder::Finish() { return Status::Invalid("cannot scan with null format table"); } if (impl_->format_table_ != nullptr) { - // The table already answers both, and from a source this cannot see behind, so a second - // answer is refused rather than silently dropped. + // The table already answers each of these, and from a source this cannot see behind, so a + // second answer is refused rather than silently dropped. + if (impl_->catalog_ != nullptr) { + return Status::Invalid( + "a format table carries its own schema and the file system it was loaded through, " + "so WithCatalog() cannot be used with one; use ScanContextBuilder(FormatTable)"); + } if (impl_->table_schema_) { return Status::Invalid( "a format table carries its own schema, so SetTableSchema() cannot be used with " @@ -215,13 +238,49 @@ Result> ScanContextBuilder::Finish() { } std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); + if (impl_->catalog_ == nullptr && impl_->identifier_) { + return Status::Invalid("cannot scan through a null catalog"); + } + if (impl_->catalog_ != nullptr) { + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(impl_->catalog_)); + PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, + impl_->identifier_->GetBranchName()); + auto branch_option = impl_->options_.find(Options::BRANCH); + std::optional option_branch = + branch_option == impl_->options_.end() + ? std::nullopt + : std::optional(branch_option->second); + for (const std::optional& branch : {identifier_branch, option_branch}) { + if (branch && + !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(branch.value()))) { + return Status::Invalid(fmt::format( + "a scan through a catalog reads the main branch of the table, so it cannot be " + "aimed at branch '{}'", + branch.value())); + } + } + if (impl_->specific_file_system_ == nullptr) { + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the scan uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, impl_->identifier_->IsSystemTable()); + // A system table derives its schema from the table it reads rather than keeping one in the + // catalog, so that schema is left to be read from under the table path. + if (!impl_->table_schema_ && !is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + impl_->catalog_->LoadTableSchema(impl_->identifier_.value())); + PAIMON_ASSIGN_OR_RAISE(impl_->table_schema_, schema->GetJsonSchema()); + } + } auto ctx = std::make_unique( impl_->path_, impl_->is_streaming_mode_, impl_->limit_, std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->format_table_); + impl_->format_table_, impl_->catalog_, impl_->identifier_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 3b965889..3e8dd4ca 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -20,16 +20,113 @@ #include "gtest/gtest.h" #include "paimon/common/io/cache/lru_cache.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/status.h" +#include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/mock/mock_file_system.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { + +/// A catalog serving one table schema and a file system of its own, so a test can tell the +/// per-table answers apart from the catalog-wide ones. +std::shared_ptr CatalogServing( + const std::shared_ptr& table_file_system, std::string* schema_json) { + auto logical_schema = arrow::schema( + {arrow::field("id", arrow::int64(), false), arrow::field("value", arrow::utf8())}); + EXPECT_OK_AND_ASSIGN(std::shared_ptr schema, + TableSchema::Create(0, logical_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, {})); + EXPECT_OK_AND_ASSIGN(*schema_json, schema->GetJsonSchema()); + auto catalog = std::make_shared(); + catalog->SetTableSchema(schema); + catalog->SetTableFileSystem(table_file_system); + return catalog; +} + +} // namespace + +TEST(ScanContextTest, TestWithCatalog) { + auto table_fs = std::make_shared(); + std::string schema_json; + std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + + ScanContextBuilder builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); + ASSERT_EQ(ctx->GetCatalog(), catalog); + ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); + // The file system a catalog issuing per-table credentials hands out for this table, and the + // schema the metastore keeps, are both resolved when the context is built. + ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); + ASSERT_EQ(ctx->GetSpecificTableSchema(), std::optional(schema_json)); + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 1U); + + // Finish() resets the builder, so the next context scans without the catalog. + ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); + ASSERT_FALSE(next_ctx->GetCatalog()); + ASSERT_FALSE(next_ctx->GetIdentifier()); + ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); + ASSERT_FALSE(next_ctx->GetSpecificTableSchema()); + + ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), + "cannot scan through a null catalog"); +} + +TEST(ScanContextTest, TestCatalogAnswersAreOverridable) { + auto table_fs = std::make_shared(); + std::string schema_json; + std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + + // A file system the caller gave is the one that is used, and the catalog is not asked for one. + auto given_fs = std::make_shared(); + ScanContextBuilder fs_builder("table_root_path"); + ASSERT_OK_AND_ASSIGN( + auto fs_ctx, + fs_builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); + ASSERT_EQ(fs_ctx->GetSpecificFileSystem(), given_fs); + ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); + + // A schema the caller gave spares the catalog request entirely. A fresh catalog keeps the + // count clean of the request the file-system-override builder above already made. + std::string schema_catalog_json; + std::shared_ptr schema_catalog = + CatalogServing(table_fs, &schema_catalog_json); + ScanContextBuilder schema_builder("table_root_path"); + ASSERT_OK_AND_ASSIGN(auto schema_ctx, + schema_builder.WithCatalog(schema_catalog, Identifier("db1", "t1")) + .SetTableSchema("table-schema-json") + .Finish()); + ASSERT_EQ(schema_ctx->GetSpecificTableSchema(), + std::optional("table-schema-json")); + ASSERT_EQ(schema_catalog->LoadTableSchemaCalls(), 0U); +} + +TEST(ScanContextTest, TestCatalogScansTheMainBranchOnly) { + auto table_fs = std::make_shared(); + std::string schema_json; + std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + for (int32_t branch_source = 0; branch_source < 2; ++branch_source) { + SCOPED_TRACE(branch_source); + ScanContextBuilder builder("table_root_path"); + builder.WithCatalog(catalog, Identifier("db1", branch_source == 0 ? "t1$branch_dev" : "t1")) + .AddOption(Options::BRANCH, branch_source == 1 ? "dev" : "main"); + ASSERT_NOK_WITH_MSG(builder.Finish(), "it cannot be aimed at branch 'dev'"); + } + + ScanContextBuilder main_builder("table_root_path"); + ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) + .AddOption(Options::BRANCH, "main") + .Finish()); +} + TEST(ScanContextTest, TestDefaultValue) { ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -45,6 +142,8 @@ TEST(ScanContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetGlobalIndexResult()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_FALSE(ctx->GetCatalog()); + ASSERT_FALSE(ctx->GetIdentifier()); } TEST(ScanContextTest, TestSetContent) { diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index c3e11895..cb786b5b 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -2870,6 +2870,12 @@ TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { ASSERT_NOK_WITH_MSG( write_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), "WithCatalog() requires a native table"); + ReadContextBuilder read_catalog_builder(table); + ASSERT_NOK_WITH_MSG(read_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), + "WithCatalog() cannot be used with one"); + ScanContextBuilder scan_catalog_builder(table); + ASSERT_NOK_WITH_MSG(scan_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish(), + "WithCatalog() cannot be used with one"); catalog->SetTableSchema(latest.value()); WriteContextBuilder path_catalog_builder(dir->Str(), "test-user"); ASSERT_OK_AND_ASSIGN( @@ -2877,6 +2883,18 @@ TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { path_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(path_context)), "use WriteContextBuilder(FormatTable)"); + // A path names a table whose kind is only known once its schema is read, so the same answer + // comes from the read and scan paths once the schema turns out to be a format table's. + ReadContextBuilder read_path_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(auto read_path_context, + read_path_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_path_context)), + "use ReadContextBuilder(FormatTable)"); + ScanContextBuilder scan_path_builder(dir->Str()); + ASSERT_OK_AND_ASSIGN(auto scan_path_context, + scan_path_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_path_context)), + "use ScanContextBuilder(FormatTable)"); CommitContextBuilder commit_fs_builder(table); commit_fs_builder.WithFileSystem(dir->GetFileSystem()); diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index 557f2fd5..a60b96b8 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -162,6 +162,11 @@ Result> NewDataTableRead( /// refuses by name what a format table cannot honour. Result> NewFormatTableRead(const std::shared_ptr& table, const std::shared_ptr& context) { + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; use ReadContextBuilder(FormatTable) for a " + "format table"); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, FormatTableRead::Create(table, context)); return std::unique_ptr(std::move(read)); diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index f2a8644e..dcf58040 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -27,16 +27,21 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" #include "paimon/core/core_options.h" #include "paimon/core/operation/abstract_split_read.h" #include "paimon/core/operation/split_read.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/key_value_table_read.h" #include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/read_context.h" #include "paimon/status.h" +#include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -137,6 +142,33 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { ASSERT_TRUE(append_only_table_read); } +TEST(TableReadTest, TestCatalogReadUsesPerTableFileSystem) { + std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + // A catalog that issues per-table credentials hands out a file system for this table; + // pointing it at a real local file system lets the read reach the table's files, while a + // request count tells that this per-table answer, not the catalog-wide one, was used. + auto table_fs = std::make_shared(); + SchemaManager schema_manager(table_fs, path); + ASSERT_OK_AND_ASSIGN(std::optional> latest, + schema_manager.Latest()); + ASSERT_TRUE(latest.has_value()); + + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); + catalog->SetTableSchema(latest.value()); + + ReadContextBuilder context_builder(path); + context_builder.WithCatalog(catalog, Identifier("append_09.db", "append_09")); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); + ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_TRUE(dynamic_cast(table_read.get())); + + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); + ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("append_09.db", "append_09")); + ASSERT_EQ(catalog->LoadTableSchemaCalls(), 1U); +} + TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 7ede285f..6045d662 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -27,11 +27,13 @@ #include #include "fmt/format.h" +#include "paimon/catalog/catalog.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" @@ -97,6 +99,15 @@ class TableScanImpl { auto manifest_file_format = core_options.GetManifestFormat(); std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); + if (VersionManagedCatalog* versioned = AsVersionManaged(context->GetCatalog())) { + std::shared_ptr catalog = context->GetCatalog(); + Identifier identifier = context->GetIdentifier().value(); + // Keep the catalog alive while the scan uses its snapshot loader. + snapshot_manager->SetSnapshotLoader( + [catalog, versioned, identifier]() -> Result> { + return versioned->LoadSnapshot(identifier); + }); + } // TODO(liancheng.lsz): support fallback branch in scan auto schema_manager = std::make_shared(fs, context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE( @@ -203,6 +214,11 @@ Result> NewFormatTableScan(const std::shared_ptr
& context) { // Anything a `ScanContext` carries that a format table cannot honour is refused rather than // silently dropped. + if (context->GetCatalog() != nullptr) { + return Status::Invalid( + "WithCatalog() requires a native table; use ScanContextBuilder(FormatTable) for a " + "format table"); + } if (context->IsStreamingMode()) { return Status::NotImplemented( "a format table has no snapshots, so there is nothing for a streaming scan to follow"); @@ -278,6 +294,14 @@ Result> TableScan::Create(std::unique_ptr system_table_path, SystemTableLoader::TryParsePath(shared_context->GetPath())); if (system_table_path) { + // A system table reads the snapshots under the table path, and a catalog which manages the + // versions of its tables keeps them nowhere else but in itself. + if (AsVersionManaged(shared_context->GetCatalog()) != nullptr) { + return Status::NotImplemented( + "a system table reads the snapshots under the table path, but this catalog manages " + "the versions of its tables and publishes no snapshot there; scan the system table " + "with WithFileSystem(Catalog::GetTableFileSystem()) instead of WithCatalog()"); + } PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index dbc5b992..940a4f13 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -25,10 +25,17 @@ #include #include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" #include "paimon/scan_context.h" #include "paimon/status.h" +#include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -122,4 +129,47 @@ TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { "key table"); } +TEST(TableScanTest, TestCatalogScanReadsSnapshotFromCatalog) { + std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + auto table_fs = std::make_shared(); + SchemaManager schema_manager(table_fs, path); + ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, + schema_manager.Latest()); + ASSERT_TRUE(latest_schema.has_value()); + // A version-managed catalog keeps the latest snapshot in itself, not under the table path. + // Read the table's real snapshot and hand it back through the catalog, so the plan must go + // through the catalog's loader while the manifests it points at stay on the file system. + SnapshotManager snapshot_manager(table_fs, path); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshot_manager.LatestSnapshot()); + ASSERT_TRUE(snapshot.has_value()); + + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); + catalog->SetTableSchema(latest_schema.value()); + catalog->SetHeldSnapshot(snapshot.value()); + + ScanContextBuilder builder(path); + builder.WithCatalog(catalog, Identifier("append_09.db", "append_09")); + builder.AddOption(Options::FILE_FORMAT, "orc"); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); + ASSERT_TRUE(plan->SnapshotId()); + ASSERT_FALSE(plan->Splits().empty()); + ASSERT_GT(catalog->LoadSnapshotCalls(), 0U); +} + +TEST(TableScanTest, TestCatalogScanRejectsSystemTablePath) { + std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09$snapshots"; + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(std::make_shared()); + ScanContextBuilder builder(path); + builder.WithCatalog(catalog, Identifier("append_09.db", "append_09$snapshots")); + builder.AddOption(Options::FILE_FORMAT, "orc"); + ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), + "this catalog manages the versions of its tables and publishes no snapshot " + "there"); +} + } // namespace paimon::test diff --git a/src/paimon/fs/oss/oss_file_system_factory.cpp b/src/paimon/fs/oss/oss_file_system_factory.cpp index 327586d0..6c9ecc56 100644 --- a/src/paimon/fs/oss/oss_file_system_factory.cpp +++ b/src/paimon/fs/oss/oss_file_system_factory.cpp @@ -180,6 +180,13 @@ Result> OssFileSystemFactory::Create( GetRequiredOption(options, bucket, kOssAccessKeyIdOption)); PAIMON_ASSIGN_OR_RAISE(std::string access_key_secret, GetRequiredOption(options, bucket, kOssAccessKeySecretOption)); + std::string security_token = GetOption(options, bucket, kOssSecurityTokenOption); + if (security_token.empty()) { + security_token = GetOption(options, bucket, kOssSessionTokenOption); + } + std::shared_ptr credentials_provider = + std::make_shared( + std::move(access_key_id), std::move(access_key_secret), std::move(security_token)); std::string endpoint = GetOption(options, bucket, kOssEndpointOption); std::string region = GetOption(options, bucket, kOssRegionOption); if (region.empty()) { @@ -197,10 +204,6 @@ Result> OssFileSystemFactory::Create( return Status::Invalid( "OSS region must be configured when the endpoint does not identify a region"); } - std::string security_token = GetOption(options, bucket, kOssSecurityTokenOption); - if (security_token.empty()) { - security_token = GetOption(options, bucket, kOssSessionTokenOption); - } oss2::ClientConfiguration config = oss2::ClientConfiguration::loadDefault(); if (!endpoint.empty()) { @@ -213,8 +216,7 @@ Result> OssFileSystemFactory::Create( config.signatureVersion = signature_version; } config.userAgent = "paimon-cpp"; - config.credentialsProvider = std::make_shared( - std::move(access_key_id), std::move(access_key_secret), std::move(security_token)); + config.credentialsProvider = std::move(credentials_provider); std::string path_style = GetOption(options, bucket, kOssUsePathStyleOption); if (!path_style.empty()) { std::optional value = StringUtils::StringToValue(path_style); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 7102aad1..e6aec3b1 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -39,6 +39,7 @@ #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" +#include "paimon/rest/rest_credential_provider.h" #include "paimon/rest/rest_token_file_system.h" #include "paimon/rest/rest_util.h" #include "paimon/table/format/format_table.h" diff --git a/src/paimon/rest/rest_credential_provider.cpp b/src/paimon/rest/rest_credential_provider.cpp new file mode 100644 index 00000000..ba29e90e --- /dev/null +++ b/src/paimon/rest/rest_credential_provider.cpp @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_credential_provider.h" + +#include +#include +#include +#include + +#include "paimon/catalog_options.h" + +namespace paimon { + +namespace { +/// Declared here rather than taken from the OSS file system, which is an optional build +/// component this module must not depend on. +constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; +} // namespace + +size_t RestToken::Hash::operator()(const RestToken& rest_token) const { + size_t result = std::hash()(rest_token.expires_at_millis); + for (const auto& [key, value] : rest_token.token) { + result = result * 31 + std::hash()(key); + result = result * 31 + std::hash()(value); + } + return result; +} + +RestCredentialProvider::RestCredentialProvider( + const std::shared_ptr& api, const std::map& catalog_options, + const Identifier& identifier, Clock clock) + : api_(api), + catalog_options_(catalog_options), + identifier_(identifier), + clock_(std::move(clock)), + logger_(Logger::GetLogger("RestCredentialProvider")) {} + +bool RestCredentialProvider::ShouldRefresh() const { + if (!token_) { + return true; + } + int64_t now_millis = + std::chrono::duration_cast(clock_().time_since_epoch()).count(); + return token_->expires_at_millis - now_millis < RestApi::kTokenExpirationSafeTimeMillis; +} + +std::map RestCredentialProvider::MergeTokenOptions( + const std::map& token) const { + std::map merged = token; + // The DLF OSS endpoint overrides the standard one, since the credentials are issued + // for the DLF endpoint rather than for the endpoint the catalog was configured with. + auto dlf_oss_endpoint = catalog_options_.find(CatalogOptions::DLF_OSS_ENDPOINT); + if (dlf_oss_endpoint != catalog_options_.end() && !dlf_oss_endpoint->second.empty()) { + merged[kOssEndpointOption] = dlf_oss_endpoint->second; + } + return merged; +} + +Status RestCredentialProvider::RefreshToken() const { + PAIMON_LOG_INFO(logger_, "begin refresh data token for identifier [%s]", + identifier_.ToString().c_str()); + PAIMON_ASSIGN_OR_RAISE(GetTableTokenResponse response, api_->LoadTableToken(identifier_)); + PAIMON_LOG_INFO(logger_, "end refresh data token for identifier [%s] expiresAtMillis [%ld]", + identifier_.ToString().c_str(), + static_cast(response.GetExpiresAtMillis())); + + token_ = RestToken{MergeTokenOptions(response.GetToken()), response.GetExpiresAtMillis()}; + return Status::OK(); +} + +Result RestCredentialProvider::ValidToken() const { + { + std::shared_lock read_lock(mutex_); + if (!ShouldRefresh()) { + return token_.value(); + } + } + + std::unique_lock write_lock(mutex_); + // Double-check, another thread may have refreshed while this one waited for the lock. + if (!ShouldRefresh()) { + return token_.value(); + } + PAIMON_RETURN_NOT_OK(RefreshToken()); + return token_.value(); +} + +Result> RestCredentialProvider::GetCredentials() const { + PAIMON_ASSIGN_OR_RAISE(RestToken token, ValidToken()); + return token.token; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_credential_provider.h b/src/paimon/rest/rest_credential_provider.h new file mode 100644 index 00000000..aafc8352 --- /dev/null +++ b/src/paimon/rest/rest_credential_provider.h @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/fs/credential_provider.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_api.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// The temporary credentials of one table, as issued by the REST catalog. +struct RestToken { + /// File system options, e.g. "fs.oss.accessKeyId". + std::map token; + int64_t expires_at_millis = 0; + + /// Credentials are interchangeable only when they grant the same access until the same + /// point in time, which is what makes them a file system cache key. + bool operator==(const RestToken& other) const { + return expires_at_millis == other.expires_at_millis && token == other.token; + } + + struct Hash { + size_t operator()(const RestToken& rest_token) const; + }; +}; + +/// The default `CredentialProvider`: it loads the temporary credentials the REST catalog issues +/// for the data of one table and reloads them before they expire, so every access draws +/// credentials that are still valid without the caller reaching the server or tracking the +/// expiry itself. +/// +/// The credentials are cached, and a refresh happens only once they are absent or expire +/// within `RestApi::kTokenExpirationSafeTimeMillis`, so repeated accesses within the valid +/// window ask the server nothing. +class RestCredentialProvider : public CredentialProvider { + public: + using Clock = std::function; + + /// @param api Client of the catalog that issues the credentials. Shared because a + /// provider commonly outlives the catalog it was obtained from. + /// @param catalog_options Options the credentials are merged over. + /// @param identifier The table the credentials are requested for. + /// @param clock Source of the current time, overridable for tests. + RestCredentialProvider(const std::shared_ptr& api, + const std::map& catalog_options, + const Identifier& identifier, + Clock clock = std::chrono::system_clock::now); + + ~RestCredentialProvider() override = default; + + /// Returns credentials that are not about to expire, reloading them when needed. The + /// `RestToken` also carries the expiration, which is what a file system cache keys the + /// delegates it builds from these credentials by. + Result ValidToken() const; + + /// The credentials to sign an access with, reloaded before they expire. This is the + /// `CredentialProvider` side of the same credentials `ValidToken()` serves, so the file + /// systems built from them and the callers handed these credentials draw from one + /// source. + Result> GetCredentials() const override; + + private: + /// Reloads the credentials from the server. Called with the write lock of `mutex_` + /// held. + Status RefreshToken() const; + + /// Whether `token_` is absent or expires within the safe time. + bool ShouldRefresh() const; + + /// `catalog_options_` with `token` merged over it. + std::map MergeTokenOptions( + const std::map& token) const; + + std::shared_ptr api_; + std::map catalog_options_; + Identifier identifier_; + Clock clock_; + std::shared_ptr logger_; + + /// Guards `token_`. + mutable std::shared_mutex mutex_; + mutable std::optional token_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_credential_provider_test.cpp b/src/paimon/rest/rest_credential_provider_test.cpp new file mode 100644 index 00000000..35548db3 --- /dev/null +++ b/src/paimon/rest/rest_credential_provider_test.cpp @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_credential_provider.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/catalog_options.h" +#include "paimon/fs/credential_provider.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/rest/rest_api.h" +#include "paimon/rest/rest_messages.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +constexpr const char kToken[] = "test-token"; +constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; + +using Credentials = std::map; + +// The credentials the mock server hands out, plus the number of times it was asked for +// them. +struct MockTokenState { + Credentials token = {{"fs.oss.accessKeyId", "ak-1"}}; + int64_t expires_at_millis = 0; + // when set, the token endpoint fails with this http code + std::optional force_error_code; + // guards the fields above: the handler runs on the server's accept thread while tests + // seed and inspect the state + std::mutex mutex; + std::atomic request_count{0}; +}; + +MockRestServer::Response HandleTokenRequest(MockTokenState* state, + const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request.path != "/v1/databases/db1/tables/t1/token") { + ErrorResponse error("", "", "unknown path " + request.path, 404); + response.code = 404; + response.body = error.ToJsonString().value(); + return response; + } + state->request_count++; + + std::lock_guard lock(state->mutex); + if (state->force_error_code) { + ErrorResponse error(ErrorResponse::kResourceTypeTable, "t1", "no permission", + state->force_error_code.value()); + response.code = state->force_error_code.value(); + response.body = error.ToJsonString().value(); + return response; + } + GetTableTokenResponse token(state->token, state->expires_at_millis); + response.body = token.ToJsonString().value(); + return response; +} + +} // namespace + +class RestCredentialProviderTest : public ::testing::Test { + protected: + void SetUp() override { + state_ = std::make_shared(); + // credentials that are valid well beyond the safe time, so nothing refreshes + // unless a test moves the clock + state_->expires_at_millis = kExpiresAtMillis; + now_millis_ = kNowMillis; + ASSERT_OK_AND_ASSIGN( + server_, MockRestServer::Start([state = state_](const MockRestServer::Request& req) { + return HandleTokenRequest(state.get(), req); + })); + + catalog_options_ = { + {CatalogOptions::URI, server_->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + } + + void TearDown() override { + if (server_) { + server_->Stop(); + } + } + + std::shared_ptr CreateProvider() { + Result> api = + RestApi::Create(catalog_options_, "", /*config_required=*/false); + if (!api.ok()) { + return nullptr; + } + std::shared_ptr shared_api(std::move(api).value()); + return std::make_shared( + shared_api, catalog_options_, Identifier("db1", "t1"), [this] { + return std::chrono::system_clock::time_point( + std::chrono::milliseconds(now_millis_.load())); + }); + } + + // Epoch millis the injected clock starts at; an arbitrary point far enough from 0 + // that subtracting the safe time stays positive. + static constexpr int64_t kNowMillis = 1700000000000; + // Expiration the mock server reports, far beyond the safe time of `kNowMillis`. + static constexpr int64_t kExpiresAtMillis = + kNowMillis + 10 * RestApi::kTokenExpirationSafeTimeMillis; + + std::shared_ptr state_; + std::unique_ptr server_; + std::map catalog_options_; + std::atomic now_millis_{kNowMillis}; +}; + +TEST_F(RestCredentialProviderTest, LoadsCredentialsOnceAndReusesThemWithinTheSafeTime) { + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + // the provider is seen through the interface a caller that brings its own file system + // holds, so the credentials are pulled from it rather than from a delegated file system + std::shared_ptr credential_provider = provider; + ASSERT_OK_AND_ASSIGN(Credentials credentials, credential_provider->GetCredentials()); + ASSERT_EQ("ak-1", credentials.at("fs.oss.accessKeyId")); + ASSERT_EQ(1, state_->request_count.load()); + + // one millisecond before the safe time the credentials are still served as they are + now_millis_ = kExpiresAtMillis - RestApi::kTokenExpirationSafeTimeMillis - 1; + ASSERT_OK_AND_ASSIGN(Credentials reused, credential_provider->GetCredentials()); + ASSERT_EQ("ak-1", reused.at("fs.oss.accessKeyId")); + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST_F(RestCredentialProviderTest, RefreshesWithinTheSafeTime) { + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + ASSERT_OK_AND_ASSIGN(RestToken first, provider->ValidToken()); + ASSERT_EQ("ak-1", first.token.at("fs.oss.accessKeyId")); + ASSERT_EQ(1, state_->request_count.load()); + + int64_t next_expiration = kExpiresAtMillis + RestApi::kTokenExpirationSafeTimeMillis; + { + std::lock_guard lock(state_->mutex); + state_->token = {{"fs.oss.accessKeyId", "ak-2"}}; + state_->expires_at_millis = next_expiration; + } + now_millis_ = kExpiresAtMillis - 1; + ASSERT_OK_AND_ASSIGN(RestToken second, provider->ValidToken()); + ASSERT_EQ("ak-2", second.token.at("fs.oss.accessKeyId")); + ASSERT_EQ(next_expiration, second.expires_at_millis); + ASSERT_EQ(2, state_->request_count.load()); + + // the refreshed credentials are reused + ASSERT_OK_AND_ASSIGN(Credentials credentials, provider->GetCredentials()); + ASSERT_EQ("ak-2", credentials.at("fs.oss.accessKeyId")); + ASSERT_EQ(2, state_->request_count.load()); +} + +TEST_F(RestCredentialProviderTest, ExpiredTokenReloadsOnEveryCall) { + // an expiration the server did not report makes the credentials expire immediately + { + std::lock_guard lock(state_->mutex); + state_->expires_at_millis = 0; + } + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + ASSERT_OK(provider->ValidToken().status()); + ASSERT_OK(provider->ValidToken().status()); + ASSERT_EQ(2, state_->request_count.load()); +} + +TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpoint) { + catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = "dlf-endpoint"; + { + std::lock_guard lock(state_->mutex); + state_->token = {{"fs.oss.accessKeyId", "ak-1"}, {kOssEndpointOption, "server-endpoint"}}; + } + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + ASSERT_OK_AND_ASSIGN(RestToken token, provider->ValidToken()); + ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); + // the endpoint the credentials were issued for wins over the one the server reported + ASSERT_EQ("dlf-endpoint", token.token.at(kOssEndpointOption)); + ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); + // the catalog options are not part of the token, so its secrets stay private + ASSERT_EQ(0u, token.token.count(CatalogOptions::TOKEN)); + ASSERT_EQ(2u, token.token.size()); +} + +TEST_F(RestCredentialProviderTest, EmptyDlfOssEndpointIsNotApplied) { + catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = ""; + { + std::lock_guard lock(state_->mutex); + state_->token = {{kOssEndpointOption, "server-endpoint"}}; + } + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + // an unset dlf endpoint leaves the endpoint the server reported alone + ASSERT_OK_AND_ASSIGN(RestToken token, provider->ValidToken()); + ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); + ASSERT_EQ(1u, token.token.size()); +} + +TEST_F(RestCredentialProviderTest, ForbiddenIsReportedToTheCaller) { + { + std::lock_guard lock(state_->mutex); + state_->force_error_code = 403; + } + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Status status = provider->GetCredentials().status(); + ASSERT_NOK(status); + ASSERT_NOK_WITH_MSG(status, "no permission"); + + // a later success is not blocked by the earlier failure + { + std::lock_guard lock(state_->mutex); + state_->force_error_code.reset(); + } + ASSERT_OK_AND_ASSIGN(Credentials credentials, provider->GetCredentials()); + ASSERT_EQ("ak-1", credentials.at("fs.oss.accessKeyId")); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_token_file_system.cpp b/src/paimon/rest/rest_token_file_system.cpp index baa2d7ad..43d59fad 100644 --- a/src/paimon/rest/rest_token_file_system.cpp +++ b/src/paimon/rest/rest_token_file_system.cpp @@ -18,30 +18,12 @@ #include "paimon/rest/rest_token_file_system.h" -#include -#include #include -#include "paimon/catalog_options.h" #include "paimon/core/core_options.h" namespace paimon { -namespace { -/// Declared here rather than taken from the OSS file system, which is an optional build -/// component this module must not depend on. -constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; -} // namespace - -size_t RestToken::Hash::operator()(const RestToken& rest_token) const { - size_t result = std::hash()(rest_token.expires_at_millis); - for (const auto& [key, value] : rest_token.token) { - result = result * 31 + std::hash()(key); - result = result * 31 + std::hash()(value); - } - return result; -} - std::shared_ptr RestTokenFileSystem::CreateFileSystemCache() { RestTokenFileSystemCache::Options cache_options; cache_options.max_weight = kMaxCachedFileSystems; @@ -54,45 +36,11 @@ RestTokenFileSystem::RestTokenFileSystem(const std::shared_ptr& api, const Identifier& identifier, std::shared_ptr fs_cache, Clock clock) - : api_(api), - catalog_options_(catalog_options), + : catalog_options_(catalog_options), identifier_(identifier), fs_cache_(fs_cache != nullptr ? std::move(fs_cache) : CreateFileSystemCache()), - clock_(std::move(clock)), - logger_(Logger::GetLogger("RestTokenFileSystem")) {} - -bool RestTokenFileSystem::ShouldRefresh() const { - if (!token_) { - return true; - } - int64_t now_millis = - std::chrono::duration_cast(clock_().time_since_epoch()).count(); - return token_->expires_at_millis - now_millis < RestApi::kTokenExpirationSafeTimeMillis; -} - -std::map RestTokenFileSystem::MergeTokenOptions( - const std::map& token) const { - std::map merged = token; - // The DLF OSS endpoint overrides the standard one, since the credentials are issued - // for the DLF endpoint rather than for the endpoint the catalog was configured with. - auto dlf_oss_endpoint = catalog_options_.find(CatalogOptions::DLF_OSS_ENDPOINT); - if (dlf_oss_endpoint != catalog_options_.end() && !dlf_oss_endpoint->second.empty()) { - merged[kOssEndpointOption] = dlf_oss_endpoint->second; - } - return merged; -} - -Status RestTokenFileSystem::RefreshToken() const { - PAIMON_LOG_INFO(logger_, "begin refresh data token for identifier [%s]", - identifier_.ToString().c_str()); - PAIMON_ASSIGN_OR_RAISE(GetTableTokenResponse response, api_->LoadTableToken(identifier_)); - PAIMON_LOG_INFO(logger_, "end refresh data token for identifier [%s] expiresAtMillis [%ld]", - identifier_.ToString().c_str(), - static_cast(response.GetExpiresAtMillis())); - - token_ = RestToken{MergeTokenOptions(response.GetToken()), response.GetExpiresAtMillis()}; - return Status::OK(); -} + provider_(std::make_shared(api, catalog_options, identifier, + std::move(clock))) {} Result> RestTokenFileSystem::BuildFileSystem( const RestToken& token) const { @@ -117,20 +65,7 @@ Result> RestTokenFileSystem::Delegate() const { } Result RestTokenFileSystem::ValidToken() const { - { - std::shared_lock read_lock(mutex_); - if (!ShouldRefresh()) { - return token_.value(); - } - } - - std::unique_lock write_lock(mutex_); - // Double-check, another thread may have refreshed while this one waited for the lock. - if (!ShouldRefresh()) { - return token_.value(); - } - PAIMON_RETURN_NOT_OK(RefreshToken()); - return token_.value(); + return provider_->ValidToken(); } Result> RestTokenFileSystem::Open(const std::string& path) const { diff --git a/src/paimon/rest/rest_token_file_system.h b/src/paimon/rest/rest_token_file_system.h index b1272682..0ac63326 100644 --- a/src/paimon/rest/rest_token_file_system.h +++ b/src/paimon/rest/rest_token_file_system.h @@ -19,43 +19,23 @@ #pragma once #include -#include #include #include #include #include -#include -#include #include #include #include "paimon/catalog/identifier.h" #include "paimon/common/utils/generic_lru_cache.h" #include "paimon/fs/file_system.h" -#include "paimon/logging.h" #include "paimon/rest/rest_api.h" +#include "paimon/rest/rest_credential_provider.h" #include "paimon/result.h" #include "paimon/status.h" namespace paimon { -/// The temporary credentials of one table, as issued by the REST catalog. -struct RestToken { - /// File system options, e.g. "fs.oss.accessKeyId". - std::map token; - int64_t expires_at_millis = 0; - - /// Credentials are interchangeable only when they grant the same access until the same - /// point in time, which is what makes them a file system cache key. - bool operator==(const RestToken& other) const { - return expires_at_millis == other.expires_at_millis && token == other.token; - } - - struct Hash { - size_t operator()(const RestToken& rest_token) const; - }; -}; - /// File systems keyed by the credentials they were built from, so that the tables the /// server issues the same credentials for share one file system. Sharing this cache /// between the `RestTokenFileSystem` instances of many tables keeps a rotation of one @@ -69,7 +49,7 @@ using RestTokenFileSystemCache = /// options, so the schemes configured for the catalog keep working. class RestTokenFileSystem : public FileSystem { public: - using Clock = std::function; + using Clock = RestCredentialProvider::Clock; /// Bounds of the file system cache, matching the Java client: the file system of /// credentials that were not used for this long is dropped, which also keeps a stream @@ -121,31 +101,16 @@ class RestTokenFileSystem : public FileSystem { /// expire in less than `RestApi::kTokenExpirationSafeTimeMillis`. Result> Delegate() const; - /// Reloads the credentials from the server. Called with the write lock of `mutex_` - /// held. - Status RefreshToken() const; - - /// Whether `token_` is absent or expires within the safe time. - bool ShouldRefresh() const; - /// Builds the file system of `token`: the credentials are the only file system options /// that change, so it is built from the catalog options with them merged over. Result> BuildFileSystem(const RestToken& token) const; - /// `catalog_options_` with `token` merged over it. - std::map MergeTokenOptions( - const std::map& token) const; - - std::shared_ptr api_; std::map catalog_options_; Identifier identifier_; std::shared_ptr fs_cache_; - Clock clock_; - std::shared_ptr logger_; - /// Guards `token_`. - mutable std::shared_mutex mutex_; - mutable std::optional token_; + /// The credentials this file system delegates with, reloaded before they expire. + std::shared_ptr provider_; }; } // namespace paimon diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index f52029a8..f81d1fde 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -37,6 +37,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/defs.h" +#include "paimon/fs/credential_provider.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" From 0dbacec66dd3c5df4d7c8ef53557f0997261c3e0 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 17 Sep 2026 17:38:28 +0800 Subject: [PATCH 06/16] fix --- docs/source/user_guide/catalog.rst | 55 ++++++++----------- .../rest/rest_token_file_system_test.cpp | 1 - 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index fd4ef454..1520f93f 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -169,18 +169,18 @@ it; writers also allow ``WithFileSystemSchemeToIdentifierMap`` to override file-system selection. For format tables, use ``WriteContextBuilder(FormatTable)`` instead. -A file system passed to ``WithFileSystem`` is used exactly as it is, so a custom -one that signs each request itself stays authenticated by drawing the table's -temporary credentials from ``Catalog::GetTableCredentialProvider``: hold the provider -and call ``CredentialProvider::GetCredentials`` at each access, which returns -credentials that are still valid and reloads them before they expire. A catalog -that issues no per-table credentials returns a null provider, and the custom file -system's own configuration authenticates the access. +A file system passed to ``WithFileSystem`` is used exactly as it is: neither the +built-in file systems nor the table's own credentials are involved, so a custom +one has to authenticate its own accesses. Draw credentials that expire from a +``CredentialProvider`` you build with ``CredentialProviderFactory``: hold the +provider and call ``CredentialProvider::GetCredentials`` at each access, which +returns credentials that are still valid and reloads them before they expire. .. code-block:: cpp PAIMON_ASSIGN_OR_RAISE(std::shared_ptr provider, - shared_catalog->GetTableCredentialProvider(paimon::Identifier("db", "tbl"))); + paimon::CredentialProviderFactory::Get("my-token-service", + "oss://bucket/tbl", options)); // MyFileSystem merges provider->GetCredentials() over its own options per access. auto my_fs = std::make_shared(provider); write_builder.WithCatalog(shared_catalog, paimon::Identifier("db", "tbl")) @@ -285,9 +285,8 @@ Both builders read the table's schema from the catalog when ``Finish()`` builds the context, which is one catalog request; call ``SetTableSchema`` with a schema you already hold to skip it. The table's file system, including the per-table temporary credentials a catalog serves through ``Catalog::GetTableFileSystem``, -is used for both. ``WithFileSystem`` overrides it, and a custom file system -that signs its own requests can draw the table's credentials from -``Catalog::GetTableCredentialProvider`` as shown above; the read builder also allows +is used for both. ``WithFileSystem`` overrides it with a file system that is used +as-is and authenticates its own accesses, and the read builder also allows ``WithFileSystemSchemeToIdentifierMap``. Only the main branch is supported. Both builders reject other branches in the @@ -367,13 +366,19 @@ tags, branch management and consumers — are not supported. Authenticating with credentials of your own ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Credentials that a catalog does not issue — from a token service of your own, or -from a catalog Paimon has no client for — reach the built-in file systems through -a ``CredentialProvider``. Implement it together with a -``CredentialProviderFactory`` that builds it, register the factory with -``REGISTER_PAIMON_FACTORY``, and name it in the ``fs.credential.provider`` -option: the file system then builds the provider once and asks it for credentials -as it signs, so credentials are reloaded as they expire without the file system -being rebuilt. +from a store Paimon has no client for — reach a file system through a +``CredentialProvider`` that you consult yourself. Implement it together with a +``CredentialProviderFactory`` that builds it and register the factory with +``REGISTER_PAIMON_FACTORY``. Build a provider with +``CredentialProviderFactory::Get``, wrap it in a ``FileSystem`` of your own that +calls ``CredentialProvider::GetCredentials`` as it signs each access, and pass +that file system in through ``Catalog::Create`` or a builder's ``WithFileSystem``. +A file system passed this way is used as-is, so the credentials are reloaded as +they expire without it being rebuilt. + +The built-in file systems — ``oss``, ``s3``, ``local``, ``jindo`` — do not consult +a provider: they sign with the static credentials of their own options. Bring a +file system of your own when the credentials expire. .. code-block:: cpp @@ -409,18 +414,6 @@ the factory is called with are those of the accesses to authenticate, which is where an implementation reads its own configuration, such as the address of the token service, from. -.. code-block:: none - - fs.credential.provider=my-token-service - -Only the OSS file system consults a provider today; the others report -``NotImplemented`` when the option names one, rather than signing with the -credentials of the moment they were built and failing once those expire. All -factories share one identifier space, so an identifier a file system factory +All factories share one identifier space, so an identifier a file system factory already takes — ``oss``, ``s3``, ``local``, ``jindo`` — would replace it; name the provider after where its credentials come from instead. - -This is the other of the two ways a provider is reached. A custom file system -passed to ``WithFileSystem`` signs its own requests and pulls from the provider -itself, which is what ``Catalog::GetTableCredentialProvider`` serves; a built-in -file system signs on your behalf and is pointed at a provider by this option. diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index f81d1fde..f52029a8 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -37,7 +37,6 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/defs.h" -#include "paimon/fs/credential_provider.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" From 862c7b7b20ab5615165b196f2b6249a1e020be9d Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Thu, 17 Sep 2026 21:24:59 +0800 Subject: [PATCH 07/16] fix --- include/paimon/catalog/catalog.h | 5 +- .../paimon/fs/credential_provider_factory.h | 16 ++---- .../common/fs/credential_provider_factory.cpp | 11 ---- .../fs/credential_provider_factory_test.cpp | 25 -------- .../core/catalog/file_system_catalog_test.cpp | 4 +- .../core/operation/file_store_commit.cpp | 7 ++- .../core/operation/file_store_write.cpp | 7 ++- src/paimon/core/operation/read_context.cpp | 8 ++- src/paimon/core/operation/scan_context.cpp | 8 ++- .../table/system/global_system_tables.cpp | 2 +- .../core/table/system/system_table_test.cpp | 3 +- src/paimon/rest/rest_catalog.cpp | 26 +++++++-- src/paimon/rest/rest_catalog.h | 8 ++- src/paimon/rest/rest_catalog_test.cpp | 57 ++++++++++++++----- src/paimon/testing/mock/mock_catalog.h | 3 +- 15 files changed, 103 insertions(+), 87 deletions(-) diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 7f92dd0e..cadd5220 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -197,10 +197,13 @@ class PAIMON_EXPORT Catalog { /// `ScanContextBuilder::WithCatalog` or `WriteContextBuilder::WithCatalog`. /// /// @param identifier The identifier (database and table name) of the table. + /// @param fs_options File system options that override the ones the file system is + /// built from. The per-table temporary credentials a catalog issues + /// still take precedence over them. /// @return A shared pointer to the file system instance; the catalog-level file system /// by default. virtual Result> GetTableFileSystem( - const Identifier& identifier) const { + const Identifier& identifier, const std::map& fs_options) const { return GetFileSystem(); } diff --git a/include/paimon/fs/credential_provider_factory.h b/include/paimon/fs/credential_provider_factory.h index ff373e46..8a7b4dd0 100644 --- a/include/paimon/fs/credential_provider_factory.h +++ b/include/paimon/fs/credential_provider_factory.h @@ -31,18 +31,17 @@ namespace paimon { /// A factory for creating `CredentialProvider` instances. /// -/// Register an implementation with `REGISTER_PAIMON_FACTORY` to authenticate accesses -/// with credentials of your own making: the built-in file systems build the provider -/// named by their `fs.credential.provider` option and consult it whenever they sign. +/// Register an implementation with `REGISTER_PAIMON_FACTORY` to build a provider that a +/// file system of your own consults: get it with `Get`, call `GetCredentials` as you sign +/// each access, and pass that file system in through `Catalog::Create` or a builder's +/// `WithFileSystem`. The built-in file systems do not consult a provider; they sign with +/// the static credentials of their own options. /// /// @note All factories share one identifier space, so an identifier that a file system /// factory already takes - "oss", "s3", "local", "jindo" - would replace it. Name the /// provider after where its credentials come from instead. class PAIMON_EXPORT CredentialProviderFactory : public Factory { public: - /// The option naming the factory a file system builds its provider with. - static const char CREDENTIAL_PROVIDER_OPTION[]; - /// Create a `CredentialProvider` of current factory for the accesses below a path. /// /// The options are the file system options of the accesses to authenticate, so an @@ -55,11 +54,6 @@ class PAIMON_EXPORT CredentialProviderFactory : public Factory { static Result> Get( const std::string& identifier, const std::string& path, const std::map& fs_options); - - /// Get the `CredentialProvider` the options ask for, or a null provider when they name - /// none, in which case the file system options themselves authenticate the accesses. - static Result> GetIfConfigured( - const std::string& path, const std::map& fs_options); }; } // namespace paimon diff --git a/src/paimon/common/fs/credential_provider_factory.cpp b/src/paimon/common/fs/credential_provider_factory.cpp index dd2c3e0b..5172185a 100644 --- a/src/paimon/common/fs/credential_provider_factory.cpp +++ b/src/paimon/common/fs/credential_provider_factory.cpp @@ -24,8 +24,6 @@ namespace paimon { -const char CredentialProviderFactory::CREDENTIAL_PROVIDER_OPTION[] = "fs.credential.provider"; - Result> CredentialProviderFactory::Get( const std::string& identifier, const std::string& path, const std::map& fs_options) { @@ -49,13 +47,4 @@ Result> CredentialProviderFactory::Get( return provider; } -Result> CredentialProviderFactory::GetIfConfigured( - const std::string& path, const std::map& fs_options) { - auto option = fs_options.find(CREDENTIAL_PROVIDER_OPTION); - if (option == fs_options.end() || option->second.empty()) { - return std::shared_ptr(nullptr); - } - return Get(option->second, path, fs_options); -} - } // namespace paimon diff --git a/src/paimon/common/fs/credential_provider_factory_test.cpp b/src/paimon/common/fs/credential_provider_factory_test.cpp index 21bccba7..427feb45 100644 --- a/src/paimon/common/fs/credential_provider_factory_test.cpp +++ b/src/paimon/common/fs/credential_provider_factory_test.cpp @@ -112,29 +112,4 @@ TEST(CredentialProviderFactoryTest, TestGetRejectsNullProvider) { "created a null provider"); } -TEST(CredentialProviderFactoryTest, TestGetIfConfigured) { - std::map options{ - {CredentialProviderFactory::CREDENTIAL_PROVIDER_OPTION, - TestCredentialProviderFactory::IDENTIFIER}, - {"test.secret", "token"}}; - ASSERT_OK_AND_ASSIGN(std::shared_ptr provider, - CredentialProviderFactory::GetIfConfigured("oss://bucket/table", options)); - ASSERT_NE(provider, nullptr); - ASSERT_OK_AND_ASSIGN(auto credentials, provider->GetCredentials()); - ASSERT_EQ(credentials["secret"], "token"); -} - -TEST(CredentialProviderFactoryTest, TestGetIfConfiguredWithoutOption) { - // No provider named means the file system options authenticate the accesses themselves. - ASSERT_OK_AND_ASSIGN(std::shared_ptr provider, - CredentialProviderFactory::GetIfConfigured("/tmp/table", {})); - ASSERT_EQ(provider, nullptr); - - std::map empty_option{ - {CredentialProviderFactory::CREDENTIAL_PROVIDER_OPTION, ""}}; - ASSERT_OK_AND_ASSIGN(provider, - CredentialProviderFactory::GetIfConfigured("/tmp/table", empty_option)); - ASSERT_EQ(provider, nullptr); -} - } // namespace paimon::test diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 18793419..8a050ae1 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -55,9 +55,9 @@ TEST(FileSystemCatalogTest, TestGetTableFileSystem) { // a catalog without per-table credentials serves the table data with the catalog wide // file system, whatever the table is - ASSERT_OK_AND_ASSIGN(auto table_fs, catalog.GetTableFileSystem(Identifier("db1", "t1"))); + ASSERT_OK_AND_ASSIGN(auto table_fs, catalog.GetTableFileSystem(Identifier("db1", "t1"), {})); ASSERT_EQ(catalog.GetFileSystem(), table_fs); - ASSERT_OK_AND_ASSIGN(auto other_fs, catalog.GetTableFileSystem(Identifier("db2", "t2"))); + ASSERT_OK_AND_ASSIGN(auto other_fs, catalog.GetTableFileSystem(Identifier("db2", "t2"), {})); ASSERT_EQ(catalog.GetFileSystem(), other_fs); } diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index db9cee5c..36d04833 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -199,9 +199,10 @@ Result> FileStoreCommit::Create( return Status::Invalid("a catalog commit requires a table identifier"); } // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. - PAIMON_ASSIGN_OR_RAISE(specific_fs, - ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + // ones only hands out through GetTableFileSystem. The context options override the + // catalog ones the file system is built from. + PAIMON_ASSIGN_OR_RAISE(specific_fs, ctx->GetCatalog()->GetTableFileSystem( + ctx->GetIdentifier().value(), ctx->GetOptions())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index 48f1c1dc..d3b9bd61 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -161,9 +161,10 @@ Result> FileStoreWrite::Create(std::unique_ptrGetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); + // GetTableFileSystem, so the write uses the credentials of this table. The context + // options override the catalog ones the file system is built from. + PAIMON_ASSIGN_OR_RAISE(specific_fs, ctx->GetCatalog()->GetTableFileSystem( + ctx->GetIdentifier().value(), ctx->GetOptions())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx->GetOptions(), specific_fs, diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index 6f685898..caa75f60 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -403,9 +403,11 @@ Result> ReadContextBuilder::Finish() { if (impl_->specific_file_system_ == nullptr && impl_->fs_scheme_to_identifier_map_.empty()) { // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the read uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); + // GetTableFileSystem, so the read uses the credentials of this table. The context + // options override the catalog ones the file system is built from. + PAIMON_ASSIGN_OR_RAISE( + impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value(), impl_->options_)); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, impl_->identifier_->IsSystemTable()); // A system table derives its schema from the table it reads rather than keeping one in the diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 88c58c66..baa14e69 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -261,9 +261,11 @@ Result> ScanContextBuilder::Finish() { } if (impl_->specific_file_system_ == nullptr) { // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the scan uses the credentials of this table. - PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); + // GetTableFileSystem, so the scan uses the credentials of this table. The context + // options override the catalog ones the file system is built from. + PAIMON_ASSIGN_OR_RAISE( + impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value(), impl_->options_)); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, impl_->identifier_->IsSystemTable()); // A system table derives its schema from the table it reads rather than keeping one in the diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 647da76c..b176982d 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -518,7 +518,7 @@ Result> PartitionsSystemTable::BuildRows() const { // The files of each table are served with the credentials of that table, which a // catalog issuing temporary ones per table only hands out through the catalog. Result> table_fs_result = - context_.catalog->GetTableFileSystem(id); + context_.catalog->GetTableFileSystem(id, /*fs_options=*/{}); if (!table_fs_result.ok()) { if (table_fs_result.status().IsNotExist()) { continue; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 7591e53b..32ab1bd7 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -298,7 +298,8 @@ class PerTableFileSystemCatalog : public FileSystemCatalog { : FileSystemCatalog(fs, warehouse, options) {} Result> GetTableFileSystem( - const Identifier& identifier) const override { + const Identifier& identifier, + const std::map& /*fs_options*/) const override { requested.push_back(identifier.GetFullName()); if (failure) { return failure.value(); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index e6aec3b1..6b1c645d 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -454,20 +454,34 @@ std::shared_ptr RestCatalog::GetFileSystem() const { } Result> RestCatalog::GetTableFileSystem( - const Identifier& identifier) const { + const Identifier& identifier, const std::map& fs_options) const { if (!data_token_enabled_) { return fs_; } // The credentials are issued for the data table, so a system table shares those of // the table it belongs to. PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + // The caller's options override the catalog ones; the temporary credentials are merged + // over both when the delegate is built, so they always win. + std::map options = api_->GetMergedOptions(); + for (const auto& [key, value] : fs_options) { + options[key] = value; + } // Building the file system asks the server for nothing, the credentials are loaded on // the first access, so nothing is keyed by the table here: the file systems built from - // the credentials are what `token_fs_cache_` reuses and bounds. Keying an instance by - // its table would keep serving the credentials of a dropped table to a table recreated - // at another location. - return std::make_shared(api_, api_->GetMergedOptions(), load_identifier, - token_fs_cache_); + // the credentials are what a cache reuses and bounds. Keying an instance by its table + // would keep serving the credentials of a dropped table to a table recreated at another + // location. + // + // The shared cache is keyed by the credentials alone and holds the file systems built + // from the catalog options, which every table that overrides nothing agrees on. A call + // that does override them builds a different file system, so it must neither read nor + // write that cache: reading it would hand back a file system built from other options + // and silently drop the override. Such a file system keeps a private cache of its own, + // still keyed by the credentials so a rotation rebuilds it. + std::shared_ptr fs_cache = + fs_options.empty() ? token_fs_cache_ : nullptr; + return std::make_shared(api_, options, load_identifier, fs_cache); } Result> RestCatalog::ListSnapshots(const Identifier& identifier, diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 1af74938..f988c7b0 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -79,9 +79,13 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// catalog-level file system otherwise. Every call returns an instance bound to the /// table it was asked for, whose credentials are loaded on the first access; the file /// systems built from them are shared through a bounded cache, so the tables the - /// server issues the same credentials for share one file system. + /// server issues the same credentials for share one file system. `fs_options` override + /// the catalog options the file system is built from, the credentials still win over + /// them; a call that overrides anything builds a file system of its own rather than + /// reading the shared cache, so its override is never dropped for a cache hit. Result> GetTableFileSystem( - const Identifier& identifier) const override; + const Identifier& identifier, + const std::map& fs_options) const override; Result> GetTable(const Identifier& identifier) const override; Result> ListSnapshots(const Identifier& identifier, const std::string& branch) const override; diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 4c5581a1..420917f2 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -548,7 +548,7 @@ TEST_F(RestCatalogTest, CreateMergesServerConfig) { TEST_F(RestCatalogTest, TableFileSystemWithoutDataToken) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, - catalog->GetTableFileSystem(Identifier("db1", "t1"))); + catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); // without the data token the catalog wide credentials are used for the data as well ASSERT_EQ(catalog->GetFileSystem(), fs); } @@ -557,7 +557,7 @@ TEST_F(RestCatalogTest, TableFileSystemWithDataToken) { options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, - catalog->GetTableFileSystem(Identifier("db1", "t1"))); + catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); ASSERT_NE(nullptr, fs); ASSERT_NE(catalog->GetFileSystem(), fs); @@ -571,10 +571,10 @@ TEST_F(RestCatalogTest, TableFileSystemWithDataToken) { // a system table reads the files of the table it belongs to, so it is served the // credentials of that table ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1$snapshots"))); + catalog->GetTableFileSystem(Identifier("db1", "t1$snapshots"), {})); ASSERT_OK(system_table_fs->Exists("/no-such-file").status()); ASSERT_OK_AND_ASSIGN(std::shared_ptr other_table_fs, - catalog->GetTableFileSystem(Identifier("db1", "t2"))); + catalog->GetTableFileSystem(Identifier("db1", "t2"), {})); ASSERT_OK(other_table_fs->Exists("/no-such-file").status()); ASSERT_EQ(std::vector( {TokenPath("db1", "t1"), TokenPath("db1", "t1"), TokenPath("db1", "t2")}), @@ -588,10 +588,10 @@ TEST_F(RestCatalogTest, TableFileSystemIsBoundToTheTableItWasAskedFor) { // the database and the table are addressed separately, so identifiers that print the // same must not be served the credentials of one another ASSERT_OK_AND_ASSIGN(std::shared_ptr dotted_database, - catalog->GetTableFileSystem(Identifier("db1.a", "t1"))); + catalog->GetTableFileSystem(Identifier("db1.a", "t1"), {})); ASSERT_OK(dotted_database->Exists("/no-such-file").status()); ASSERT_OK_AND_ASSIGN(std::shared_ptr dotted_table, - catalog->GetTableFileSystem(Identifier("db1", "a.t1"))); + catalog->GetTableFileSystem(Identifier("db1", "a.t1"), {})); ASSERT_OK(dotted_table->Exists("/no-such-file").status()); ASSERT_EQ(std::vector({TokenPath("db1.a", "t1"), TokenPath("db1", "a.t1")}), TokenRequests()); @@ -603,16 +603,17 @@ TEST_F(RestCatalogTest, TableFileSystemNormalizesTheBranch) { // the main branch is the table itself, so it shares the credentials ASSERT_OK_AND_ASSIGN(std::shared_ptr main_branch_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1$branch_main"))); + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_main"), {})); ASSERT_OK(main_branch_fs->Exists("/no-such-file").status()); // another branch is addressed as its own object on the server, so it gets its own // credentials ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1"))); + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1"), {})); ASSERT_OK(branch_fs->Exists("/no-such-file").status()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_system_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1$snapshots"))); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr branch_system_fs, + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1$snapshots"), {})); ASSERT_OK(branch_system_fs->Exists("/no-such-file").status()); ASSERT_EQ(std::vector({TokenPath("db1", "t1"), TokenPath("db1", "t1$branch_b1"), TokenPath("db1", "t1$branch_b1")}), @@ -627,9 +628,9 @@ TEST_F(RestCatalogTest, TableFileSystemIsNotRetainedPerTable) { // another location is never served the credentials of the dropped one. What is cached // and bounded are the file systems built from the credentials, keyed by them. ASSERT_OK_AND_ASSIGN(std::shared_ptr first, - catalog->GetTableFileSystem(Identifier("db1", "t1"))); + catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); ASSERT_OK_AND_ASSIGN(std::shared_ptr second, - catalog->GetTableFileSystem(Identifier("db1", "t1"))); + catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); ASSERT_NE(first, second); ASSERT_OK(first->Exists("/no-such-file").status()); @@ -638,6 +639,34 @@ TEST_F(RestCatalogTest, TableFileSystemIsNotRetainedPerTable) { TokenRequests()); } +TEST_F(RestCatalogTest, TableFileSystemOptionsOverrideIsNotDroppedByTheSharedCache) { + options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + Identifier identifier("db1", "t1"); + + // A call that overrides nothing builds its delegate from the catalog options and puts it + // in the cache the tables of this catalog share, keyed by the credentials. + ASSERT_OK_AND_ASSIGN(std::shared_ptr plain, + catalog->GetTableFileSystem(identifier, {})); + ASSERT_OK(plain->Exists("/no-such-file").status()); + + // The same table with an override must not reuse that cached delegate: it builds a file + // system of its own, so the overridden scheme reaches the delegate and building it fails + // with the scheme it was overridden to instead of silently keeping the cached local one. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr overridden, + catalog->GetTableFileSystem(identifier, {{Options::FILE_SYSTEM, "no-such-file-system"}})); + Status status = overridden->Exists("/no-such-file").status(); + ASSERT_NOK(status); + ASSERT_NOK_WITH_MSG(status, "no-such-file-system"); + + // The override never entered the shared cache, so a following default call is still + // served a delegate built from the catalog options. + ASSERT_OK_AND_ASSIGN(std::shared_ptr plain_again, + catalog->GetTableFileSystem(identifier, {})); + ASSERT_OK(plain_again->Exists("/no-such-file").status()); +} + TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) { options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); @@ -646,7 +675,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); ASSERT_OK_AND_ASSIGN(std::string old_location, catalog->GetTableLocation(identifier)); ASSERT_OK_AND_ASSIGN(std::shared_ptr first, - catalog->GetTableFileSystem(identifier)); + catalog->GetTableFileSystem(identifier, {})); std::shared_ptr first_token_fs = std::dynamic_pointer_cast(first); ASSERT_NE(nullptr, first_token_fs); @@ -669,7 +698,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(identifier)); ASSERT_EQ(new_location, location); ASSERT_OK_AND_ASSIGN(std::shared_ptr second, - catalog->GetTableFileSystem(identifier)); + catalog->GetTableFileSystem(identifier, {})); ASSERT_NE(first, second); std::shared_ptr second_token_fs = std::dynamic_pointer_cast(second); diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index 8f5a84c1..cf230bd0 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -232,7 +232,8 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { /// test can tell it apart from the catalog-wide one; falls back to `GetFileSystem()`, /// matching the base default, when none was set. Result> GetTableFileSystem( - const Identifier& identifier) const override { + const Identifier& identifier, + const std::map& /*fs_options*/) const override { table_file_system_requests_.push_back(identifier); if (table_file_system_ != nullptr) { return table_file_system_; From d665babb9f21cbf90fba14101ae86de2a91c306f Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 18 Sep 2026 10:56:30 +0800 Subject: [PATCH 08/16] fix --- docs/source/user_guide/catalog.rst | 4 +- .../source/snapshot/full_starting_scanner.h | 29 +++++++---- .../core/table/source/table_scan_test.cpp | 23 +++++++-- src/paimon/rest/rest_catalog.cpp | 16 ++++-- src/paimon/rest/rest_catalog.h | 23 ++++++--- src/paimon/rest/rest_catalog_test.cpp | 27 ++++++++++ src/paimon/rest/rest_token_file_system.cpp | 50 +++++++++++++++++-- src/paimon/rest/rest_token_file_system.h | 9 ++++ .../rest/rest_token_file_system_test.cpp | 36 +++++++++++++ 9 files changed, 184 insertions(+), 33 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 1520f93f..26893e95 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -252,7 +252,7 @@ so its restored file references are visible to the expiration operation. A branch the catalog holds without that directory, or one created while expiration runs, is not found: do not expire a table with such a branch, and serialize branch creation and expiration through the upstream coordinator. -======= + Reading through the catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pass the catalog and table identifier to ``ReadContextBuilder::WithCatalog`` or @@ -360,8 +360,6 @@ identifier. The request body carries the table id but no table name and no branch, so the branch has to appear in the URL the caller sends that body to: the commit endpoint of ``tbl$branch_dev``, not the one of ``tbl``. A body sent to the bare table's URL publishes the branch's snapshot on the main branch. -======= -tags, branch management and consumers — are not supported. Authenticating with credentials of your own ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/paimon/core/table/source/snapshot/full_starting_scanner.h b/src/paimon/core/table/source/snapshot/full_starting_scanner.h index 28a4872b..c2f99a6f 100644 --- a/src/paimon/core/table/source/snapshot/full_starting_scanner.h +++ b/src/paimon/core/table/source/snapshot/full_starting_scanner.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include "paimon/core/table/source/snapshot/starting_scanner.h" @@ -32,19 +33,29 @@ class FullStartingScanner : public StartingScanner { Result> Scan( const std::shared_ptr& snapshot_reader) override { - if (starting_snapshot_id_ == std::nullopt) { - // try to get first snapshot - PAIMON_ASSIGN_OR_RAISE(starting_snapshot_id_, snapshot_manager_->LatestSnapshotId()); + if (starting_snapshot_ == std::nullopt) { + // Resolve the latest snapshot together with its body: a version-managed catalog + // can hold a snapshot it never published to the table path, so reading only its + // id and then reloading the body from the path would fail with NotExist. Whether + // the body came from the catalog or from the path, it is the one to plan from. + PAIMON_ASSIGN_OR_RAISE(SnapshotManager::LatestSnapshotResult latest, + snapshot_manager_->LatestSnapshotWithSource()); + if (latest.snapshot == std::nullopt) { + return std::make_shared(); + } + starting_snapshot_ = latest.snapshot; + starting_snapshot_id_ = latest.snapshot->Id(); } - if (starting_snapshot_id_ == std::nullopt) { - return std::make_shared(); - } - PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, - snapshot_manager_->LoadSnapshot(starting_snapshot_id_.value())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr plan, - snapshot_reader->WithMode(ScanMode::ALL)->WithSnapshot(snapshot)->Read()); + snapshot_reader->WithMode(ScanMode::ALL)->WithSnapshot(*starting_snapshot_)->Read()); return std::make_shared(plan); } + + private: + /// The body of the snapshot this scan starts from, resolved once so that a catalog-held + /// snapshot is planned from the body the catalog returned instead of being re-read from a + /// path it was never published to. + std::optional starting_snapshot_; }; } // namespace paimon diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 940a4f13..863d4e72 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -130,19 +130,32 @@ TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { } TEST(TableScanTest, TestCatalogScanReadsSnapshotFromCatalog) { - std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - auto table_fs = std::make_shared(); + std::string fixture = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + // Copy the table so its snapshot file can be removed without touching the shared fixture. + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, dir); + std::string path = dir->Str(); + ASSERT_TRUE(TestUtil::CopyDirectory(fixture, path)); + auto table_fs = dir->GetFileSystem(); + ASSERT_NE(nullptr, table_fs); + SchemaManager schema_manager(table_fs, path); ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, schema_manager.Latest()); ASSERT_TRUE(latest_schema.has_value()); - // A version-managed catalog keeps the latest snapshot in itself, not under the table path. - // Read the table's real snapshot and hand it back through the catalog, so the plan must go - // through the catalog's loader while the manifests it points at stay on the file system. SnapshotManager snapshot_manager(table_fs, path); ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshot_manager.LatestSnapshot()); ASSERT_TRUE(snapshot.has_value()); + // A version-managed catalog can hold a snapshot it never published to the table path. Remove + // the snapshot file so that re-reading the body from the path would fail with NotExist: the + // plan must be built from the body the catalog returns, while the manifests it points at stay + // on the file system. + ASSERT_OK(table_fs->Delete(snapshot_manager.SnapshotDirectory(), /*recursive=*/true)); + ASSERT_OK_AND_ASSIGN(bool snapshot_file_exists, + table_fs->Exists(snapshot_manager.SnapshotPath(snapshot.value().Id()))); + ASSERT_FALSE(snapshot_file_exists); + auto catalog = std::make_shared(); catalog->SetTableFileSystem(table_fs); catalog->SetTableSchema(latest_schema.value()); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 6b1c645d..344fcd34 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -76,11 +76,13 @@ Result ToLoadIdentifier(const Identifier& identifier) { } // namespace RestCatalog::RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, - const std::string& warehouse, bool data_token_enabled) + const std::string& warehouse, bool data_token_enabled, + bool fs_explicitly_supplied) : api_(std::move(api)), fs_(fs), warehouse_(warehouse), data_token_enabled_(data_token_enabled), + fs_explicitly_supplied_(fs_explicitly_supplied), table_default_options_(RestUtil::ExtractPrefixMap( api_->GetMergedOptions(), CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX)), logger_(Logger::GetLogger("RestCatalog")) { @@ -100,8 +102,9 @@ Result> RestCatalog::Create( PAIMON_ASSIGN_OR_RAISE(bool data_token_enabled, OptionsUtils::GetValueFromMap( api->GetMergedOptions(), CatalogOptions::DATA_TOKEN_ENABLED, false)); - return std::unique_ptr(new RestCatalog( - std::move(api), core_options.GetFileSystem(), warehouse, data_token_enabled)); + return std::unique_ptr( + new RestCatalog(std::move(api), core_options.GetFileSystem(), warehouse, data_token_enabled, + /*fs_explicitly_supplied=*/file_system != nullptr)); } const std::map& RestCatalog::GetOptions() const { @@ -455,7 +458,12 @@ std::shared_ptr RestCatalog::GetFileSystem() const { Result> RestCatalog::GetTableFileSystem( const Identifier& identifier, const std::map& fs_options) const { - if (!data_token_enabled_) { + // A file system the caller supplied to `Catalog::Create` authenticates its own accesses + // and is used as-is, so it is handed out even when the server issues data tokens: + // rebuilding a delegate from the options would discard it together with the + // `CredentialProvider` it signs with, and could send a custom scheme to the default + // backend. Without data tokens the catalog-wide file system serves the data as well. + if (!data_token_enabled_ || fs_explicitly_supplied_) { return fs_; } // The credentials are issued for the data table, so a system table shares those of diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index f988c7b0..13d4927b 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -76,13 +76,16 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { std::shared_ptr GetFileSystem() const override; /// Returns a file system that refreshes the temporary credentials the server issues /// for the table when `CatalogOptions::DATA_TOKEN_ENABLED` is set, and the - /// catalog-level file system otherwise. Every call returns an instance bound to the - /// table it was asked for, whose credentials are loaded on the first access; the file - /// systems built from them are shared through a bounded cache, so the tables the - /// server issues the same credentials for share one file system. `fs_options` override - /// the catalog options the file system is built from, the credentials still win over - /// them; a call that overrides anything builds a file system of its own rather than - /// reading the shared cache, so its override is never dropped for a cache hit. + /// catalog-level file system otherwise. A file system the caller supplied to + /// `Catalog::Create` is that catalog-level file system and is returned as-is, data + /// tokens or not, so its own `CredentialProvider` keeps signing its accesses. Every + /// call returns an instance bound to the table it was asked for, whose credentials are + /// loaded on the first access; the file systems built from them are shared through a + /// bounded cache, so the tables the server issues the same credentials for share one + /// file system. `fs_options` override the catalog options the file system is built + /// from, the credentials still win over them; a call that overrides anything builds a + /// file system of its own rather than reading the shared cache, so its override is + /// never dropped for a cache hit. Result> GetTableFileSystem( const Identifier& identifier, const std::map& fs_options) const override; @@ -113,7 +116,7 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { private: RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, - const std::string& warehouse, bool data_token_enabled); + const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied); /// Loads the schema and catalog table ID from the same response. Result> LoadTableSchema(const Identifier& identifier, @@ -133,6 +136,10 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { std::string warehouse_; /// Whether table data is accessed with the credentials the server issues per table. bool data_token_enabled_ = false; + /// Whether `fs_` was supplied by the caller of `Catalog::Create` rather than built from + /// the merged options. A supplied file system authenticates its own accesses and is used + /// as-is, so it is handed out even when the server issues data tokens. + bool fs_explicitly_supplied_ = false; /// The "table-default." options of the merged config, applied to `CreateTable` /// options when absent. std::map table_default_options_; diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 420917f2..9de451d0 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -553,6 +553,33 @@ TEST_F(RestCatalogTest, TableFileSystemWithoutDataToken) { ASSERT_EQ(catalog->GetFileSystem(), fs); } +TEST_F(RestCatalogTest, TableFileSystemKeepsAnExplicitlySuppliedFileSystem) { + options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; + std::unique_ptr dir = UniqueTestDirectory::Create(); + ASSERT_NE(nullptr, dir); + std::shared_ptr custom_fs = dir->GetFileSystem(); + ASSERT_NE(nullptr, custom_fs); + + // A file system supplied to Create authenticates its own accesses and is used as-is, so + // it is handed out for the table data even though the server issues data tokens: + // rebuilding a delegate from the options would discard it and its CredentialProvider. + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, + RestCatalog::Create(kWarehouse, options_, custom_fs)); + ASSERT_EQ(custom_fs, catalog->GetFileSystem()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_fs, + catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); + ASSERT_EQ(custom_fs, table_fs); + // Handing out the supplied file system asks the server for no data token. + ASSERT_TRUE(TokenRequests().empty()); + + // The same options without a supplied file system do build a token file system, so the + // identity above comes from the supplied file system, not from data tokens being off. + ASSERT_OK_AND_ASSIGN(std::unique_ptr token_catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr token_fs, + token_catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); + ASSERT_NE(custom_fs, token_fs); +} + TEST_F(RestCatalogTest, TableFileSystemWithDataToken) { options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); diff --git a/src/paimon/rest/rest_token_file_system.cpp b/src/paimon/rest/rest_token_file_system.cpp index 43d59fad..c1296e5b 100644 --- a/src/paimon/rest/rest_token_file_system.cpp +++ b/src/paimon/rest/rest_token_file_system.cpp @@ -18,11 +18,27 @@ #include "paimon/rest/rest_token_file_system.h" +#include #include +#include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" namespace paimon { +namespace { + +// Length of the "fs.." prefix of a file system option key, or 0 when `key` is not +// scheme-qualified. Only a scheme-qualified key such as "fs.oss.accessKeyId" can have a +// bucket-scoped variant such as "fs.oss.bucket..accessKeyId". +size_t FileSystemOptionPrefixLength(const std::string& key) { + if (!StringUtils::StartsWith(key, "fs.")) { + return 0; + } + size_t scheme_end = key.find('.', 3); + return scheme_end == std::string::npos ? 0 : scheme_end + 1; +} + +} // namespace std::shared_ptr RestTokenFileSystem::CreateFileSystemCache() { RestTokenFileSystemCache::Options cache_options; @@ -42,14 +58,40 @@ RestTokenFileSystem::RestTokenFileSystem(const std::shared_ptr& api, provider_(std::make_shared(api, catalog_options, identifier, std::move(clock))) {} -Result> RestTokenFileSystem::BuildFileSystem( - const RestToken& token) const { - std::map fs_options = catalog_options_; +std::map RestTokenFileSystem::MergeTokenOptions( + const std::map& catalog_options, const RestToken& token) { + std::map fs_options = catalog_options; for (const auto& [key, value] : token.token) { + // A file system resolves a bucket-scoped option such as + // "fs.oss.bucket..accessKeyId" ahead of the flat "fs.oss.accessKeyId" the + // token carries. Keeping a catalog's bucket-scoped credential would sign an access + // with that stale key pair together with the token's security token, an invalid + // combination, so drop the bucket-scoped variant of every option the token sets: the + // issued credentials then win whichever bucket the table's data lives in. + size_t prefix_length = FileSystemOptionPrefixLength(key); + if (prefix_length != 0) { + std::string bucket_prefix = key.substr(0, prefix_length) + "bucket."; + std::string bucket_suffix = "." + key.substr(prefix_length); + for (auto it = fs_options.begin(); it != fs_options.end();) { + if (it->first.size() > bucket_prefix.size() + bucket_suffix.size() && + StringUtils::StartsWith(it->first, bucket_prefix) && + StringUtils::EndsWith(it->first, bucket_suffix)) { + it = fs_options.erase(it); + } else { + ++it; + } + } + } fs_options[key] = value; } + return fs_options; +} + +Result> RestTokenFileSystem::BuildFileSystem( + const RestToken& token) const { PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(fs_options, /*specified_file_system=*/nullptr)); + CoreOptions::FromMap(MergeTokenOptions(catalog_options_, token), + /*specified_file_system=*/nullptr)); std::shared_ptr fs = core_options.GetFileSystem(); if (fs == nullptr) { return Status::Invalid("failed to build the file system of the data token of ", diff --git a/src/paimon/rest/rest_token_file_system.h b/src/paimon/rest/rest_token_file_system.h index 0ac63326..fa5e5134 100644 --- a/src/paimon/rest/rest_token_file_system.h +++ b/src/paimon/rest/rest_token_file_system.h @@ -96,6 +96,15 @@ class RestTokenFileSystem : public FileSystem { /// credentials are requested with. Result ValidToken() const; + /// Merges the issued credentials over the catalog options the delegate is built from. A + /// bucket-scoped variant of an option the credentials set, such as + /// "fs.oss.bucket..accessKeyId" for the credentials' "fs.oss.accessKeyId", is + /// dropped: a file system resolves the bucket-scoped option first, so keeping a catalog's + /// one would sign an access with a stale key pair and the issued security token, an invalid + /// combination. Exposed for tests. + static std::map MergeTokenOptions( + const std::map& catalog_options, const RestToken& token); + private: /// Returns the file system of the current credentials, reloading them when they /// expire in less than `RestApi::kTokenExpirationSafeTimeMillis`. diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index f52029a8..9e65a1bd 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -377,6 +377,42 @@ TEST_F(RestTokenFileSystemTest, TokenOverridesTheCatalogFileSystemOptions) { ASSERT_EQ(1, state_->request_count.load()); } +TEST(RestTokenFileSystemMergeTokenOptions, IssuedCredentialsWinOverBucketScopedCatalogOnes) { + // The catalog is configured with bucket-scoped credentials, which a file system resolves + // ahead of the flat options, so they must not shadow the credentials the server issues. + std::map catalog_options = { + {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, + {"fs.oss.bucket.b.accessKeySecret", "catalog-bucket-sk"}, + {"fs.oss.bucket.other.accessKeyId", "catalog-other-ak"}, + {"fs.oss.accessKeyId", "catalog-ak"}, + {"fs.oss.endpoint", "catalog-endpoint"}, + {"fs.oss.bucket.b.endpoint", "catalog-bucket-endpoint"}, + {"unrelated", "kept"}, + }; + RestToken token; + token.token = {{"fs.oss.accessKeyId", "token-ak"}, + {"fs.oss.accessKeySecret", "token-sk"}, + {"fs.oss.securityToken", "token-sts"}}; + + std::map merged = + RestTokenFileSystem::MergeTokenOptions(catalog_options, token); + + // The issued credentials are present and every bucket-scoped variant of them is gone, so + // the per-bucket resolution a file system does falls back to the issued flat options + // rather than signing with a stale catalog key pair and the issued security token. + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ("token-sk", merged.at("fs.oss.accessKeySecret")); + ASSERT_EQ("token-sts", merged.at("fs.oss.securityToken")); + ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeyId")); + ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeySecret")); + ASSERT_EQ(0u, merged.count("fs.oss.bucket.other.accessKeyId")); + + // Options the token does not set keep their catalog value, bucket-scoped ones included. + ASSERT_EQ("catalog-endpoint", merged.at("fs.oss.endpoint")); + ASSERT_EQ("catalog-bucket-endpoint", merged.at("fs.oss.bucket.b.endpoint")); + ASSERT_EQ("kept", merged.at("unrelated")); +} + TEST_F(RestTokenFileSystemTest, ConcurrentFirstAccessLoadsTheTokenOnce) { std::string path = WriteFile("data", "paimon"); std::shared_ptr fs = CreateFileSystem(); From a386d232833600de7c477ec108ffc9b89d483b9c Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Fri, 18 Sep 2026 17:40:11 +0800 Subject: [PATCH 09/16] fix --- docs/source/user_guide/catalog.rst | 34 +++--- include/paimon/catalog/catalog.h | 3 +- include/paimon/read_context.h | 33 ++---- include/paimon/scan_context.h | 35 ++---- include/paimon/write_context.h | 9 +- src/paimon/core/operation/read_context.cpp | 68 +++--------- .../core/operation/read_context_test.cpp | 104 +++--------------- src/paimon/core/operation/scan_context.cpp | 87 +++++---------- .../core/operation/scan_context_test.cpp | 96 ++++------------ .../core/table/format/format_table_test.cpp | 12 -- .../source/snapshot/full_starting_scanner.h | 29 ++--- src/paimon/core/table/source/table_read.cpp | 5 - .../core/table/source/table_read_test.cpp | 32 ------ src/paimon/core/table/source/table_scan.cpp | 24 ---- .../core/table/source/table_scan_test.cpp | 63 ----------- src/paimon/rest/rest_catalog.h | 1 - 16 files changed, 123 insertions(+), 512 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 26893e95..7848d7f7 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -255,10 +255,10 @@ so its restored file references are visible to the expiration operation. Reading through the catalog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Pass the catalog and table identifier to ``ReadContextBuilder::WithCatalog`` or -``ScanContextBuilder::WithCatalog`` to read a native table the same way, in one -call: the table's schema comes from the catalog, its data is read through the -table's file system, and a scan reads the latest snapshot from the catalog: +To read a native table with the per-table temporary credentials a catalog issues, +pass the catalog and table identifier to ``ReadContextBuilder::WithCatalog`` or +``ScanContextBuilder::WithCatalog``. This is a shorthand for asking the catalog +for that table's file system and passing it in through ``WithFileSystem``: .. code-block:: cpp @@ -279,24 +279,16 @@ table's file system, and a scan reads the latest snapshot from the catalog: PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, paimon::TableRead::Create(std::move(read_context))); -A scan takes only its latest snapshot from the catalog; the manifests that -snapshot points at and any historical snapshots still come from the table path. -Both builders read the table's schema from the catalog when ``Finish()`` builds -the context, which is one catalog request; call ``SetTableSchema`` with a schema -you already hold to skip it. The table's file system, including the per-table -temporary credentials a catalog serves through ``Catalog::GetTableFileSystem``, -is used for both. ``WithFileSystem`` overrides it with a file system that is used -as-is and authenticates its own accesses, and the read builder also allows -``WithFileSystemSchemeToIdentifierMap``. - -Only the main branch is supported. Both builders reject other branches in the -identifier or ``branch`` option; the read builder also checks ``WithBranch``. -Explicit ``tbl$branch_main`` and ``branch=main`` are accepted. For a format +When ``Finish()`` builds the context, it asks the catalog for the table's file +system through ``Catalog::GetTableFileSystem`` and uses it as-is for the schema, +snapshots, manifests and data under the table path, so it signs every access with +the table's credentials and reloads them as they expire. An explicit +``WithFileSystem`` takes precedence, so the catalog is not asked. A catalog that +issues no per-table credentials returns its catalog-level file system, so this is +also how you read with the catalog's own object-store credentials. For a format table, use ``ReadContextBuilder(FormatTable)`` or ``ScanContextBuilder(FormatTable)`` -instead; ``WithCatalog`` is rejected. A system table path such as ``tbl$snapshots`` -reads snapshots from under the table path, which a version-managed catalog -publishes nowhere but in itself, so it is rejected with ``NotImplemented``; scan -such a table with ``WithFileSystem(Catalog::GetTableFileSystem(...))`` instead. +instead; ``WithCatalog`` is rejected because the table already carries the file +system it was loaded through. The C++ REST catalog covers the database, table, snapshot and commit operations of the ``Catalog`` API. The parts of the Java REST catalog that have no C++ diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index cadd5220..d31dfcb0 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -194,7 +194,8 @@ class PAIMON_EXPORT Catalog { /// table it was requested for. Pass it to `ReadContextBuilder::WithFileSystem`, /// `ScanContextBuilder::WithFileSystem` or `WriteContextBuilder::WithFileSystem`, /// or let a builder request it for you through `ReadContextBuilder::WithCatalog`, - /// `ScanContextBuilder::WithCatalog` or `WriteContextBuilder::WithCatalog`. + /// `ScanContextBuilder::WithCatalog`, `WriteContextBuilder::WithCatalog` or + /// `CommitContextBuilder::WithCatalog`. /// /// @param identifier The identifier (database and table name) of the table. /// @param fs_options File system options that override the ones the file system is diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index f2fff8cd..068e688a 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -64,9 +64,7 @@ class PAIMON_EXPORT ReadContext { const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, - const std::optional& identifier); + const std::shared_ptr& format_table, WarmupLevel warmup_level); ~ReadContext(); @@ -119,8 +117,6 @@ class PAIMON_EXPORT ReadContext { uint32_t GetRowToBatchThreadNumber() const { return row_to_batch_thread_number_; } - /// The schema this context reads with, or null when it has to be read from the table path. - /// `WithCatalog()` resolves it to the catalog's schema when `SetTableSchema()` did not answer. const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -161,16 +157,6 @@ class PAIMON_EXPORT ReadContext { return warmup_level_; } - /// Returns the catalog supplying table metadata, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. bool HasReadSchema() const { return read_schema_ != nullptr && read_schema_->release != nullptr; @@ -212,8 +198,6 @@ class PAIMON_EXPORT ReadContext { std::shared_ptr cache_; std::shared_ptr format_table_; WarmupLevel warmup_level_; - std::shared_ptr catalog_; - std::optional identifier_; // Owns schema resources and releases ArrowSchema::release in destructor. std::unique_ptr read_schema_; }; @@ -228,8 +212,8 @@ class PAIMON_EXPORT ReadContextBuilder { /// Constructs a `ReadContextBuilder` for a format table that is already loaded: the only way /// to read one whose schema lives in a metastore rather than under its location, such as a /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()`, - /// `WithCatalog()` and a branch are refused here rather than ignored. + /// `SetTableSchema()`, `WithFileSystem()`, `WithFileSystemSchemeToIdentifierMap()` and a + /// branch are refused here rather than ignored. /// /// @param table The format table to read, as `Catalog::GetFormatTable()` hands it back. explicit ReadContextBuilder(const std::shared_ptr& table); @@ -496,13 +480,10 @@ class PAIMON_EXPORT ReadContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ReadContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Loads a native table's schema from the catalog and reads its data through that table's file - /// system - including the per-table temporary credentials a catalog that issues them hands out - /// through `Catalog::GetTableFileSystem`. - /// - /// Only the main branch is supported. `SetTableSchema()` keeps the caller's schema and spares - /// the catalog request; `WithFileSystem()` and `WithFileSystemSchemeToIdentifierMap()` override - /// the file system. + /// Reads a native table through its own file system - including the per-table temporary + /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. + /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an + /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked. /// @param catalog Non-null catalog, read when `Finish()` builds the context. /// @param identifier The native table to read. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 8122d7dd..8f87d121 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -58,9 +58,7 @@ class PAIMON_EXPORT ScanContext { const std::optional& table_schema, const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, - const std::shared_ptr& catalog, - const std::optional& identifier); + const std::shared_ptr& format_table); ~ScanContext(); @@ -103,8 +101,6 @@ class PAIMON_EXPORT ScanContext { return specific_file_system_; } - /// The schema this context scans with, or null when it has to be read from the table path. - /// `WithCatalog()` resolves it to the catalog's schema when `SetTableSchema()` did not answer. const std::optional& GetSpecificTableSchema() const { return table_schema_; } @@ -119,16 +115,6 @@ class PAIMON_EXPORT ScanContext { return format_table_; } - /// Returns the catalog supplying table metadata, or null if unset. - const std::shared_ptr& GetCatalog() const { - return catalog_; - } - - /// Returns the table identifier supplied with the catalog. - const std::optional& GetIdentifier() const { - return identifier_; - } - private: std::string path_; bool is_streaming_mode_; @@ -143,8 +129,6 @@ class PAIMON_EXPORT ScanContext { std::map options_; std::shared_ptr cache_; std::shared_ptr format_table_; - std::shared_ptr catalog_; - std::optional identifier_; }; /// Filter configuration for table scan operations @@ -183,8 +167,7 @@ class PAIMON_EXPORT ScanContextBuilder { /// Constructs a `ScanContextBuilder` for a format table that is already loaded: the only way /// to scan one whose schema lives in a metastore rather than under its location, such as a /// table a REST catalog serves. The table carries what such a location does not say, so - /// `SetTableSchema()`, `WithFileSystem()` and `WithCatalog()` are refused here rather than - /// ignored. + /// `SetTableSchema()` and `WithFileSystem()` are refused here rather than ignored. /// /// @param table The format table to scan, as `Catalog::GetFormatTable()` hands it back. explicit ScanContextBuilder(const std::shared_ptr& table); @@ -241,15 +224,11 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); - /// Loads a native table's schema from the catalog and plans its data through that table's file - /// system - including the per-table temporary credentials a catalog that issues them hands out - /// through `Catalog::GetTableFileSystem`. A catalog which publishes snapshots through catalog - /// commits is also where the latest snapshot is read from. - /// - /// Only the main branch is supported. `SetTableSchema()` keeps the caller's schema and spares - /// the catalog request; `WithFileSystem()` overrides the file system. - /// @param catalog Non-null catalog, read when `Finish()` builds the context and kept alive - /// while the scan uses its snapshot loader. + /// Plans a native table through its own file system - including the per-table temporary + /// credentials a catalog that issues them hands out through `Catalog::GetTableFileSystem`. + /// This is a shorthand for `WithFileSystem(catalog->GetTableFileSystem(identifier))`; an + /// explicit `WithFileSystem()` takes precedence, so the catalog is not asked. + /// @param catalog Non-null catalog, read when `Finish()` builds the context. /// @param identifier The native table to scan. /// @return Reference to this builder for method chaining. ScanContextBuilder& WithCatalog(const std::shared_ptr& catalog, diff --git a/include/paimon/write_context.h b/include/paimon/write_context.h index 2cacb325..e0a940eb 100644 --- a/include/paimon/write_context.h +++ b/include/paimon/write_context.h @@ -263,10 +263,11 @@ class PAIMON_EXPORT WriteContextBuilder { /// snapshot. Use the same catalog and identifier as `CommitContextBuilder` to restore and /// refresh real-time progress. /// - /// Only the main branch is supported. Data, manifests and historical metadata remain on the - /// file system. The catalog supplies that table's file system - including the per-table - /// temporary credentials a catalog that issues them hands out through - /// `Catalog::GetTableFileSystem` - unless overridden by `WithFileSystem()` or + /// A branch is addressed by the identifier, as `tbl$branch_dev`, so that the catalog answers + /// for that branch. Its schema is read from the branch directory rather than from the catalog, + /// just as a read of that branch reads it. + /// Data, manifests and historical metadata remain on the file system. The catalog supplies + /// that file system unless overridden by `WithFileSystem()` or /// `WithFileSystemSchemeToIdentifierMap()`. /// @param catalog Non-null catalog, kept alive while the writer uses its snapshot loader. /// @param identifier The native table to write to. diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index caa75f60..38ad37db 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -22,15 +22,11 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" -#include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" -#include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" -#include "paimon/schema/schema.h" #include "paimon/status.h" #include "paimon/table/format/format_table.h" @@ -50,8 +46,7 @@ ReadContext::ReadContext( const std::shared_ptr& realtime_context, const std::map& options, bool read_ahead_cache_enabled, const CacheConfig& cache_config, const std::shared_ptr& cache, - const std::shared_ptr& format_table, WarmupLevel warmup_level, - const std::shared_ptr& catalog, const std::optional& identifier) + const std::shared_ptr& format_table, WarmupLevel warmup_level) : path_(path), branch_(branch), read_field_names_(read_field_names), @@ -75,9 +70,7 @@ ReadContext::ReadContext( cache_config_(cache_config), cache_(cache), format_table_(format_table), - warmup_level_(warmup_level), - catalog_(catalog), - identifier_(identifier) {} + warmup_level_(warmup_level) {} ReadContext::~ReadContext() { if (read_schema_ && read_schema_->release) { @@ -327,8 +320,8 @@ Result> ReadContextBuilder::Finish() { // second answer is refused rather than silently dropped. if (impl_->catalog_ != nullptr) { return Status::Invalid( - "a format table carries its own schema and the file system it was loaded through, " - "so WithCatalog() cannot be used with one; use ReadContextBuilder(FormatTable)"); + "a format table carries the file system it was loaded through, so WithCatalog() " + "cannot be used with one"); } if (impl_->table_schema_) { return Status::Invalid( @@ -351,6 +344,17 @@ Result> ReadContextBuilder::Finish() { if (impl_->path_.empty()) { return Status::Invalid("cannot read with empty table path"); } + if (impl_->catalog_ == nullptr && impl_->identifier_) { + return Status::Invalid("cannot read through a null catalog"); + } + if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the read uses the credentials of this table. The context + // options override the catalog ones the file system is built from. + PAIMON_ASSIGN_OR_RAISE( + impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value(), impl_->options_)); + } PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch(/*identifier=*/std::nullopt, impl_->options_, impl_->branch_, "read")); @@ -378,46 +382,6 @@ Result> ReadContextBuilder::Finish() { if (impl_->enable_multi_thread_row_to_batch_ && impl_->row_to_batch_thread_number_ <= 0) { return Status::Invalid("row to batch thread number should be greater than 0"); } - if (impl_->catalog_ == nullptr && impl_->identifier_) { - return Status::Invalid("cannot read through a null catalog"); - } - if (impl_->catalog_ != nullptr) { - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(impl_->catalog_)); - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_->GetBranchName()); - auto branch_option = impl_->options_.find(Options::BRANCH); - std::optional option_branch = - branch_option == impl_->options_.end() - ? std::nullopt - : std::optional(branch_option->second); - for (const std::optional& branch : - {identifier_branch, option_branch, std::optional(impl_->branch_)}) { - if (branch && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(branch.value()))) { - return Status::Invalid(fmt::format( - "a read through a catalog reads the main branch of the table, so it cannot be " - "aimed at branch '{}'", - branch.value())); - } - } - if (impl_->specific_file_system_ == nullptr && - impl_->fs_scheme_to_identifier_map_.empty()) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the read uses the credentials of this table. The context - // options override the catalog ones the file system is built from. - PAIMON_ASSIGN_OR_RAISE( - impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value(), impl_->options_)); - } - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, impl_->identifier_->IsSystemTable()); - // A system table derives its schema from the table it reads rather than keeping one in the - // catalog, so that schema is left to be read from under the table path. - if (!impl_->table_schema_ && !is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - impl_->catalog_->LoadTableSchema(impl_->identifier_.value())); - PAIMON_ASSIGN_OR_RAISE(impl_->table_schema_, schema->GetJsonSchema()); - } - } auto ctx = std::make_unique( impl_->path_, branch, impl_->read_field_names_, impl_->read_field_ids_, impl_->predicate_, impl_->enable_predicate_filter_, impl_->enable_prefetch_, impl_->enable_late_materializing_, @@ -426,7 +390,7 @@ Result> ReadContextBuilder::Finish() { impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, impl_->realtime_context_, impl_->options_, impl_->read_ahead_cache_enabled_, impl_->cache_config_, impl_->cache_, impl_->format_table_, - impl_->warmup_level_, impl_->catalog_, impl_->identifier_); + impl_->warmup_level_); if (impl_->read_schema_ && impl_->read_schema_->release) { ctx->SetReadSchema(std::move(impl_->read_schema_)); } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index b00f2a37..d4ec0769 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -24,7 +24,6 @@ #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/common/io/cache/lru_cache.h" -#include "paimon/core/schema/table_schema.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" @@ -36,109 +35,44 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -namespace { - -/// A catalog serving one table schema and a file system of its own, so a test can tell the -/// per-table answers apart from the catalog-wide ones. -std::shared_ptr CatalogServing( - const std::shared_ptr& table_file_system, std::string* schema_json) { - auto logical_schema = arrow::schema( - {arrow::field("id", arrow::int64(), false), arrow::field("value", arrow::utf8())}); - EXPECT_OK_AND_ASSIGN(std::shared_ptr schema, - TableSchema::Create(0, logical_schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, {})); - EXPECT_OK_AND_ASSIGN(*schema_json, schema->GetJsonSchema()); - auto catalog = std::make_shared(); - catalog->SetTableSchema(schema); - catalog->SetTableFileSystem(table_file_system); - return catalog; -} - -} // namespace - -TEST(ReadContextTest, TestWithCatalog) { +TEST(ReadContextTest, TestWithCatalogResolvesTableFileSystem) { + // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it + // resolves the per-table file system when Finish() builds the context and sets nothing else. auto table_fs = std::make_shared(); - std::string schema_json; - std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - // The file system a catalog issuing per-table credentials hands out for this table, and the - // schema the metastore keeps, are both resolved when the context is built. ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); - ASSERT_EQ(ctx->GetSpecificTableSchema(), std::optional(schema_json)); ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); - ASSERT_EQ(catalog->LoadTableSchemaCalls(), 1U); // Finish() resets the builder, so the next context reads without the catalog. ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_FALSE(next_ctx->GetCatalog()); - ASSERT_FALSE(next_ctx->GetIdentifier()); ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); - ASSERT_FALSE(next_ctx->GetSpecificTableSchema()); - - ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot read through a null catalog"); + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); } -TEST(ReadContextTest, TestCatalogAnswersAreOverridable) { +TEST(ReadContextTest, TestWithFileSystemOverridesCatalog) { + // A file system the caller gave is the one that is used, and the catalog is not asked for one. auto table_fs = std::make_shared(); - std::string schema_json; - std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); auto given_fs = std::make_shared(); - // A file system the caller gave is the one that is used, and the catalog is not asked for one. - ReadContextBuilder fs_builder("table_root_path"); + ReadContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN( - auto fs_ctx, - fs_builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); - ASSERT_EQ(fs_ctx->GetSpecificFileSystem(), given_fs); - ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); - - // So is a scheme map, which names file systems the same way for the paths it covers. - ReadContextBuilder map_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto map_ctx, map_builder.WithCatalog(catalog, Identifier("db1", "t1")) - .WithFileSystemSchemeToIdentifierMap({{"file", "local"}}) - .Finish()); - ASSERT_FALSE(map_ctx->GetSpecificFileSystem()); + auto ctx, + builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); + ASSERT_EQ(ctx->GetSpecificFileSystem(), given_fs); ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); - - // A schema the caller gave spares the catalog request entirely. A fresh catalog keeps the - // count clean of the requests the schema-less builders above already made. - std::string schema_catalog_json; - std::shared_ptr schema_catalog = - CatalogServing(table_fs, &schema_catalog_json); - ReadContextBuilder schema_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto schema_ctx, - schema_builder.WithCatalog(schema_catalog, Identifier("db1", "t1")) - .SetTableSchema("table-schema-json") - .Finish()); - ASSERT_EQ(schema_ctx->GetSpecificTableSchema(), - std::optional("table-schema-json")); - ASSERT_EQ(schema_catalog->LoadTableSchemaCalls(), 0U); } -TEST(ReadContextTest, TestCatalogReadsTheMainBranchOnly) { - auto table_fs = std::make_shared(); - std::string schema_json; - std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); - for (int32_t branch_source = 0; branch_source < 3; ++branch_source) { - SCOPED_TRACE(branch_source); - ReadContextBuilder builder("table_root_path"); - builder.WithCatalog(catalog, Identifier("db1", branch_source == 0 ? "t1$branch_dev" : "t1")) - .WithBranch(branch_source == 1 ? "dev" : "main") - .AddOption(Options::BRANCH, branch_source == 2 ? "dev" : "main"); - ASSERT_NOK_WITH_MSG(builder.Finish(), "it cannot be aimed at branch 'dev'"); - } - - ReadContextBuilder main_builder("table_root_path"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .WithBranch("") - .AddOption(Options::BRANCH, "main") - .Finish()); +TEST(ReadContextTest, TestWithCatalogNullRejected) { + ReadContextBuilder builder("table_root_path"); + ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), + "cannot read through a null catalog"); } TEST(ReadContextTest, TestDefaultValue) { @@ -162,8 +96,6 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_EQ("main", ctx->GetBranch()); ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_FALSE(ctx->GetCatalog()); - ASSERT_FALSE(ctx->GetIdentifier()); } TEST(ReadContextTest, TestSetContent) { diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index baa14e69..5ca7aa8d 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -20,32 +20,28 @@ #include -#include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/utils/path_util.h" -#include "paimon/core/catalog/version_managed_catalog.h" -#include "paimon/core/utils/branch_manager.h" -#include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" -#include "paimon/schema/schema.h" #include "paimon/status.h" #include "paimon/table/format/format_table.h" namespace paimon { class Predicate; -ScanContext::ScanContext( - const std::string& path, bool is_streaming_mode, std::optional limit, - const std::shared_ptr& scan_filter, - const std::shared_ptr& global_index_result, - const std::shared_ptr& realtime_context, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, - const std::shared_ptr& specific_file_system, - const std::optional& table_schema, - const std::map& options, const std::shared_ptr& cache, - const std::shared_ptr& format_table, const std::shared_ptr& catalog, - const std::optional& identifier) +ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, + std::optional limit, + const std::shared_ptr& scan_filter, + const std::shared_ptr& global_index_result, + const std::shared_ptr& realtime_context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, + const std::shared_ptr& specific_file_system, + const std::optional& table_schema, + const std::map& options, + const std::shared_ptr& cache, + const std::shared_ptr& format_table) : path_(path), is_streaming_mode_(is_streaming_mode), limit_(limit), @@ -58,9 +54,7 @@ ScanContext::ScanContext( table_schema_(table_schema), options_(options), cache_(cache), - format_table_(format_table), - catalog_(catalog), - identifier_(identifier) {} + format_table_(format_table) {} ScanContext::~ScanContext() = default; @@ -214,12 +208,12 @@ Result> ScanContextBuilder::Finish() { return Status::Invalid("cannot scan with null format table"); } if (impl_->format_table_ != nullptr) { - // The table already answers each of these, and from a source this cannot see behind, so a - // second answer is refused rather than silently dropped. + // The table already answers both, and from a source this cannot see behind, so a second + // answer is refused rather than silently dropped. if (impl_->catalog_ != nullptr) { return Status::Invalid( - "a format table carries its own schema and the file system it was loaded through, " - "so WithCatalog() cannot be used with one; use ScanContextBuilder(FormatTable)"); + "a format table carries the file system it was loaded through, so WithCatalog() " + "cannot be used with one"); } if (impl_->table_schema_) { return Status::Invalid( @@ -236,53 +230,26 @@ Result> ScanContextBuilder::Finish() { if (impl_->path_.empty()) { return Status::Invalid("cannot scan with empty table path"); } - std::shared_ptr executor = - impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); if (impl_->catalog_ == nullptr && impl_->identifier_) { return Status::Invalid("cannot scan through a null catalog"); } - if (impl_->catalog_ != nullptr) { - PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(impl_->catalog_)); - PAIMON_ASSIGN_OR_RAISE(std::optional identifier_branch, - impl_->identifier_->GetBranchName()); - auto branch_option = impl_->options_.find(Options::BRANCH); - std::optional option_branch = - branch_option == impl_->options_.end() - ? std::nullopt - : std::optional(branch_option->second); - for (const std::optional& branch : {identifier_branch, option_branch}) { - if (branch && - !BranchManager::IsMainBranch(BranchManager::NormalizeBranch(branch.value()))) { - return Status::Invalid(fmt::format( - "a scan through a catalog reads the main branch of the table, so it cannot be " - "aimed at branch '{}'", - branch.value())); - } - } - if (impl_->specific_file_system_ == nullptr) { - // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the scan uses the credentials of this table. The context - // options override the catalog ones the file system is built from. - PAIMON_ASSIGN_OR_RAISE( - impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value(), impl_->options_)); - } - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, impl_->identifier_->IsSystemTable()); - // A system table derives its schema from the table it reads rather than keeping one in the - // catalog, so that schema is left to be read from under the table path. - if (!impl_->table_schema_ && !is_system_table) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, - impl_->catalog_->LoadTableSchema(impl_->identifier_.value())); - PAIMON_ASSIGN_OR_RAISE(impl_->table_schema_, schema->GetJsonSchema()); - } + if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the scan uses the credentials of this table. The context + // options override the catalog ones the file system is built from. + PAIMON_ASSIGN_OR_RAISE( + impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value(), impl_->options_)); } + std::shared_ptr executor = + impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); auto ctx = std::make_unique( impl_->path_, impl_->is_streaming_mode_, impl_->limit_, std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->realtime_context_, impl_->memory_pool_, executor, impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_, - impl_->format_table_, impl_->catalog_, impl_->identifier_); + impl_->format_table_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 3e8dd4ca..bc08b841 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -20,7 +20,6 @@ #include "gtest/gtest.h" #include "paimon/common/io/cache/lru_cache.h" -#include "paimon/core/schema/table_schema.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/global_index/bitmap_global_index_result.h" @@ -32,99 +31,44 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -namespace { - -/// A catalog serving one table schema and a file system of its own, so a test can tell the -/// per-table answers apart from the catalog-wide ones. -std::shared_ptr CatalogServing( - const std::shared_ptr& table_file_system, std::string* schema_json) { - auto logical_schema = arrow::schema( - {arrow::field("id", arrow::int64(), false), arrow::field("value", arrow::utf8())}); - EXPECT_OK_AND_ASSIGN(std::shared_ptr schema, - TableSchema::Create(0, logical_schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, {})); - EXPECT_OK_AND_ASSIGN(*schema_json, schema->GetJsonSchema()); - auto catalog = std::make_shared(); - catalog->SetTableSchema(schema); - catalog->SetTableFileSystem(table_file_system); - return catalog; -} - -} // namespace - -TEST(ScanContextTest, TestWithCatalog) { +TEST(ScanContextTest, TestWithCatalogResolvesTableFileSystem) { + // WithCatalog is a shorthand for WithFileSystem(catalog->GetTableFileSystem(identifier)); it + // resolves the per-table file system when Finish() builds the context and sets nothing else. auto table_fs = std::make_shared(); - std::string schema_json; - std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN(auto ctx, builder.WithCatalog(catalog, Identifier("db1", "t1")).Finish()); - ASSERT_EQ(ctx->GetCatalog(), catalog); - ASSERT_EQ(ctx->GetIdentifier(), std::optional(Identifier("db1", "t1"))); - // The file system a catalog issuing per-table credentials hands out for this table, and the - // schema the metastore keeps, are both resolved when the context is built. ASSERT_EQ(ctx->GetSpecificFileSystem(), table_fs); - ASSERT_EQ(ctx->GetSpecificTableSchema(), std::optional(schema_json)); ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("db1", "t1")); - ASSERT_EQ(catalog->LoadTableSchemaCalls(), 1U); // Finish() resets the builder, so the next context scans without the catalog. ASSERT_OK_AND_ASSIGN(auto next_ctx, builder.Finish()); - ASSERT_FALSE(next_ctx->GetCatalog()); - ASSERT_FALSE(next_ctx->GetIdentifier()); ASSERT_FALSE(next_ctx->GetSpecificFileSystem()); - ASSERT_FALSE(next_ctx->GetSpecificTableSchema()); - - ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), - "cannot scan through a null catalog"); + ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); } -TEST(ScanContextTest, TestCatalogAnswersAreOverridable) { - auto table_fs = std::make_shared(); - std::string schema_json; - std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); - +TEST(ScanContextTest, TestWithFileSystemOverridesCatalog) { // A file system the caller gave is the one that is used, and the catalog is not asked for one. + auto table_fs = std::make_shared(); + auto catalog = std::make_shared(); + catalog->SetTableFileSystem(table_fs); auto given_fs = std::make_shared(); - ScanContextBuilder fs_builder("table_root_path"); + + ScanContextBuilder builder("table_root_path"); ASSERT_OK_AND_ASSIGN( - auto fs_ctx, - fs_builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); - ASSERT_EQ(fs_ctx->GetSpecificFileSystem(), given_fs); + auto ctx, + builder.WithCatalog(catalog, Identifier("db1", "t1")).WithFileSystem(given_fs).Finish()); + ASSERT_EQ(ctx->GetSpecificFileSystem(), given_fs); ASSERT_TRUE(catalog->TableFileSystemRequests().empty()); - - // A schema the caller gave spares the catalog request entirely. A fresh catalog keeps the - // count clean of the request the file-system-override builder above already made. - std::string schema_catalog_json; - std::shared_ptr schema_catalog = - CatalogServing(table_fs, &schema_catalog_json); - ScanContextBuilder schema_builder("table_root_path"); - ASSERT_OK_AND_ASSIGN(auto schema_ctx, - schema_builder.WithCatalog(schema_catalog, Identifier("db1", "t1")) - .SetTableSchema("table-schema-json") - .Finish()); - ASSERT_EQ(schema_ctx->GetSpecificTableSchema(), - std::optional("table-schema-json")); - ASSERT_EQ(schema_catalog->LoadTableSchemaCalls(), 0U); } -TEST(ScanContextTest, TestCatalogScansTheMainBranchOnly) { - auto table_fs = std::make_shared(); - std::string schema_json; - std::shared_ptr catalog = CatalogServing(table_fs, &schema_json); - for (int32_t branch_source = 0; branch_source < 2; ++branch_source) { - SCOPED_TRACE(branch_source); - ScanContextBuilder builder("table_root_path"); - builder.WithCatalog(catalog, Identifier("db1", branch_source == 0 ? "t1$branch_dev" : "t1")) - .AddOption(Options::BRANCH, branch_source == 1 ? "dev" : "main"); - ASSERT_NOK_WITH_MSG(builder.Finish(), "it cannot be aimed at branch 'dev'"); - } - - ScanContextBuilder main_builder("table_root_path"); - ASSERT_OK(main_builder.WithCatalog(catalog, Identifier("db1", "t1$branch_main")) - .AddOption(Options::BRANCH, "main") - .Finish()); +TEST(ScanContextTest, TestWithCatalogNullRejected) { + ScanContextBuilder builder("table_root_path"); + ASSERT_NOK_WITH_MSG(builder.WithCatalog(nullptr, Identifier("db1", "t1")).Finish(), + "cannot scan through a null catalog"); } TEST(ScanContextTest, TestDefaultValue) { @@ -142,8 +86,6 @@ TEST(ScanContextTest, TestDefaultValue) { ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetGlobalIndexResult()); ASSERT_FALSE(ctx->GetSpecificFileSystem()); - ASSERT_FALSE(ctx->GetCatalog()); - ASSERT_FALSE(ctx->GetIdentifier()); } TEST(ScanContextTest, TestSetContent) { diff --git a/src/paimon/core/table/format/format_table_test.cpp b/src/paimon/core/table/format/format_table_test.cpp index cb786b5b..6071a749 100644 --- a/src/paimon/core/table/format/format_table_test.cpp +++ b/src/paimon/core/table/format/format_table_test.cpp @@ -2883,18 +2883,6 @@ TEST(FormatTableTest, TestAContextBuiltFromAFormatTableRefusesASecondAnswer) { path_catalog_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); ASSERT_NOK_WITH_MSG(FileStoreWrite::Create(std::move(path_context)), "use WriteContextBuilder(FormatTable)"); - // A path names a table whose kind is only known once its schema is read, so the same answer - // comes from the read and scan paths once the schema turns out to be a format table's. - ReadContextBuilder read_path_builder(dir->Str()); - ASSERT_OK_AND_ASSIGN(auto read_path_context, - read_path_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_path_context)), - "use ReadContextBuilder(FormatTable)"); - ScanContextBuilder scan_path_builder(dir->Str()); - ASSERT_OK_AND_ASSIGN(auto scan_path_context, - scan_path_builder.WithCatalog(catalog, Identifier("db", "tbl")).Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_path_context)), - "use ScanContextBuilder(FormatTable)"); CommitContextBuilder commit_fs_builder(table); commit_fs_builder.WithFileSystem(dir->GetFileSystem()); diff --git a/src/paimon/core/table/source/snapshot/full_starting_scanner.h b/src/paimon/core/table/source/snapshot/full_starting_scanner.h index c2f99a6f..28a4872b 100644 --- a/src/paimon/core/table/source/snapshot/full_starting_scanner.h +++ b/src/paimon/core/table/source/snapshot/full_starting_scanner.h @@ -20,7 +20,6 @@ #pragma once #include -#include #include "paimon/core/table/source/snapshot/starting_scanner.h" @@ -33,29 +32,19 @@ class FullStartingScanner : public StartingScanner { Result> Scan( const std::shared_ptr& snapshot_reader) override { - if (starting_snapshot_ == std::nullopt) { - // Resolve the latest snapshot together with its body: a version-managed catalog - // can hold a snapshot it never published to the table path, so reading only its - // id and then reloading the body from the path would fail with NotExist. Whether - // the body came from the catalog or from the path, it is the one to plan from. - PAIMON_ASSIGN_OR_RAISE(SnapshotManager::LatestSnapshotResult latest, - snapshot_manager_->LatestSnapshotWithSource()); - if (latest.snapshot == std::nullopt) { - return std::make_shared(); - } - starting_snapshot_ = latest.snapshot; - starting_snapshot_id_ = latest.snapshot->Id(); + if (starting_snapshot_id_ == std::nullopt) { + // try to get first snapshot + PAIMON_ASSIGN_OR_RAISE(starting_snapshot_id_, snapshot_manager_->LatestSnapshotId()); } + if (starting_snapshot_id_ == std::nullopt) { + return std::make_shared(); + } + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, + snapshot_manager_->LoadSnapshot(starting_snapshot_id_.value())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr plan, - snapshot_reader->WithMode(ScanMode::ALL)->WithSnapshot(*starting_snapshot_)->Read()); + snapshot_reader->WithMode(ScanMode::ALL)->WithSnapshot(snapshot)->Read()); return std::make_shared(plan); } - - private: - /// The body of the snapshot this scan starts from, resolved once so that a catalog-held - /// snapshot is planned from the body the catalog returned instead of being re-read from a - /// path it was never published to. - std::optional starting_snapshot_; }; } // namespace paimon diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index a60b96b8..557f2fd5 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -162,11 +162,6 @@ Result> NewDataTableRead( /// refuses by name what a format table cannot honour. Result> NewFormatTableRead(const std::shared_ptr& table, const std::shared_ptr& context) { - if (context->GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use ReadContextBuilder(FormatTable) for a " - "format table"); - } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, FormatTableRead::Create(table, context)); return std::unique_ptr(std::move(read)); diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index dcf58040..f2a8644e 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -27,21 +27,16 @@ #include #include "gtest/gtest.h" -#include "paimon/catalog/identifier.h" #include "paimon/core/core_options.h" #include "paimon/core/operation/abstract_split_read.h" #include "paimon/core/operation/split_read.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/append_only_table_read.h" #include "paimon/core/table/source/key_value_table_read.h" #include "paimon/defs.h" -#include "paimon/fs/local/local_file_system.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/read_context.h" #include "paimon/status.h" -#include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -142,33 +137,6 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { ASSERT_TRUE(append_only_table_read); } -TEST(TableReadTest, TestCatalogReadUsesPerTableFileSystem) { - std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - // A catalog that issues per-table credentials hands out a file system for this table; - // pointing it at a real local file system lets the read reach the table's files, while a - // request count tells that this per-table answer, not the catalog-wide one, was used. - auto table_fs = std::make_shared(); - SchemaManager schema_manager(table_fs, path); - ASSERT_OK_AND_ASSIGN(std::optional> latest, - schema_manager.Latest()); - ASSERT_TRUE(latest.has_value()); - - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - catalog->SetTableSchema(latest.value()); - - ReadContextBuilder context_builder(path); - context_builder.WithCatalog(catalog, Identifier("append_09.db", "append_09")); - context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); - ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_TRUE(dynamic_cast(table_read.get())); - - ASSERT_EQ(catalog->TableFileSystemRequests().size(), 1U); - ASSERT_EQ(catalog->TableFileSystemRequests().front(), Identifier("append_09.db", "append_09")); - ASSERT_EQ(catalog->LoadTableSchemaCalls(), 1U); -} - TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 6045d662..7ede285f 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -27,13 +27,11 @@ #include #include "fmt/format.h" -#include "paimon/catalog/catalog.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/options_utils.h" -#include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/core/core_options.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/index/pk/primary_key_index_definitions.h" @@ -99,15 +97,6 @@ class TableScanImpl { auto manifest_file_format = core_options.GetManifestFormat(); std::string branch = BranchManager::NormalizeBranch(core_options.GetBranch()); auto snapshot_manager = std::make_shared(fs, context->GetPath(), branch); - if (VersionManagedCatalog* versioned = AsVersionManaged(context->GetCatalog())) { - std::shared_ptr catalog = context->GetCatalog(); - Identifier identifier = context->GetIdentifier().value(); - // Keep the catalog alive while the scan uses its snapshot loader. - snapshot_manager->SetSnapshotLoader( - [catalog, versioned, identifier]() -> Result> { - return versioned->LoadSnapshot(identifier); - }); - } // TODO(liancheng.lsz): support fallback branch in scan auto schema_manager = std::make_shared(fs, context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE( @@ -214,11 +203,6 @@ Result> NewFormatTableScan(const std::shared_ptr& context) { // Anything a `ScanContext` carries that a format table cannot honour is refused rather than // silently dropped. - if (context->GetCatalog() != nullptr) { - return Status::Invalid( - "WithCatalog() requires a native table; use ScanContextBuilder(FormatTable) for a " - "format table"); - } if (context->IsStreamingMode()) { return Status::NotImplemented( "a format table has no snapshots, so there is nothing for a streaming scan to follow"); @@ -294,14 +278,6 @@ Result> TableScan::Create(std::unique_ptr system_table_path, SystemTableLoader::TryParsePath(shared_context->GetPath())); if (system_table_path) { - // A system table reads the snapshots under the table path, and a catalog which manages the - // versions of its tables keeps them nowhere else but in itself. - if (AsVersionManaged(shared_context->GetCatalog()) != nullptr) { - return Status::NotImplemented( - "a system table reads the snapshots under the table path, but this catalog manages " - "the versions of its tables and publishes no snapshot there; scan the system table " - "with WithFileSystem(Catalog::GetTableFileSystem()) instead of WithCatalog()"); - } PAIMON_ASSIGN_OR_RAISE( std::shared_ptr system_table, SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), shared_context->GetPath(), diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 863d4e72..dbc5b992 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -25,17 +25,10 @@ #include #include "gtest/gtest.h" -#include "paimon/catalog/identifier.h" -#include "paimon/core/schema/schema_manager.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/snapshot.h" -#include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" -#include "paimon/fs/local/local_file_system.h" #include "paimon/metrics.h" #include "paimon/scan_context.h" #include "paimon/status.h" -#include "paimon/testing/mock/mock_catalog.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -129,60 +122,4 @@ TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { "key table"); } -TEST(TableScanTest, TestCatalogScanReadsSnapshotFromCatalog) { - std::string fixture = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; - // Copy the table so its snapshot file can be removed without touching the shared fixture. - std::unique_ptr dir = UniqueTestDirectory::Create(); - ASSERT_NE(nullptr, dir); - std::string path = dir->Str(); - ASSERT_TRUE(TestUtil::CopyDirectory(fixture, path)); - auto table_fs = dir->GetFileSystem(); - ASSERT_NE(nullptr, table_fs); - - SchemaManager schema_manager(table_fs, path); - ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, - schema_manager.Latest()); - ASSERT_TRUE(latest_schema.has_value()); - SnapshotManager snapshot_manager(table_fs, path); - ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshot_manager.LatestSnapshot()); - ASSERT_TRUE(snapshot.has_value()); - - // A version-managed catalog can hold a snapshot it never published to the table path. Remove - // the snapshot file so that re-reading the body from the path would fail with NotExist: the - // plan must be built from the body the catalog returns, while the manifests it points at stay - // on the file system. - ASSERT_OK(table_fs->Delete(snapshot_manager.SnapshotDirectory(), /*recursive=*/true)); - ASSERT_OK_AND_ASSIGN(bool snapshot_file_exists, - table_fs->Exists(snapshot_manager.SnapshotPath(snapshot.value().Id()))); - ASSERT_FALSE(snapshot_file_exists); - - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(table_fs); - catalog->SetTableSchema(latest_schema.value()); - catalog->SetHeldSnapshot(snapshot.value()); - - ScanContextBuilder builder(path); - builder.WithCatalog(catalog, Identifier("append_09.db", "append_09")); - builder.AddOption(Options::FILE_FORMAT, "orc"); - ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); - ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); - ASSERT_TRUE(plan->SnapshotId()); - ASSERT_FALSE(plan->Splits().empty()); - ASSERT_GT(catalog->LoadSnapshotCalls(), 0U); -} - -TEST(TableScanTest, TestCatalogScanRejectsSystemTablePath) { - std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09$snapshots"; - auto catalog = std::make_shared(); - catalog->SetTableFileSystem(std::make_shared()); - ScanContextBuilder builder(path); - builder.WithCatalog(catalog, Identifier("append_09.db", "append_09$snapshots")); - builder.AddOption(Options::FILE_FORMAT, "orc"); - ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), - "this catalog manages the versions of its tables and publishes no snapshot " - "there"); -} - } // namespace paimon::test diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 13d4927b..5a8638bf 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -25,7 +25,6 @@ #include #include "paimon/catalog/catalog.h" -#include "paimon/common/utils/generic_lru_cache.h" #include "paimon/core/catalog/version_managed_catalog.h" #include "paimon/logging.h" #include "paimon/rest/rest_api.h" From 6adb73c996e0f18b2954fad06e05ca193cba9bb3 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 21 Sep 2026 10:43:22 +0800 Subject: [PATCH 10/16] fix --- include/paimon/catalog/catalog.h | 5 +- src/paimon/common/fs/file_system_factory.cpp | 17 ++---- .../core/catalog/file_system_catalog_test.cpp | 4 +- .../core/operation/file_store_commit.cpp | 7 +-- .../core/operation/file_store_write.cpp | 7 +-- src/paimon/core/operation/read_context.cpp | 8 +-- src/paimon/core/operation/scan_context.cpp | 8 +-- .../table/system/global_system_tables.cpp | 2 +- .../core/table/system/system_table_test.cpp | 3 +- src/paimon/fs/oss/oss_file_system_factory.cpp | 14 ++--- src/paimon/rest/rest_catalog.cpp | 19 ++---- src/paimon/rest/rest_catalog.h | 8 +-- src/paimon/rest/rest_catalog_test.cpp | 61 +++++-------------- src/paimon/testing/mock/mock_catalog.h | 3 +- 14 files changed, 50 insertions(+), 116 deletions(-) diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index d31dfcb0..18f25277 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -198,13 +198,10 @@ class PAIMON_EXPORT Catalog { /// `CommitContextBuilder::WithCatalog`. /// /// @param identifier The identifier (database and table name) of the table. - /// @param fs_options File system options that override the ones the file system is - /// built from. The per-table temporary credentials a catalog issues - /// still take precedence over them. /// @return A shared pointer to the file system instance; the catalog-level file system /// by default. virtual Result> GetTableFileSystem( - const Identifier& identifier, const std::map& fs_options) const { + const Identifier& identifier) const { return GetFileSystem(); } diff --git a/src/paimon/common/fs/file_system_factory.cpp b/src/paimon/common/fs/file_system_factory.cpp index 40c1d73b..5c8da3e0 100644 --- a/src/paimon/common/fs/file_system_factory.cpp +++ b/src/paimon/common/fs/file_system_factory.cpp @@ -24,9 +24,9 @@ namespace paimon { -namespace { - -Result LookUpFactory(const std::string& identifier) { +Result> FileSystemFactory::Get( + const std::string& identifier, const std::string& path, + const std::map& fs_options) { auto factory_creator = FactoryCreator::GetInstance(); auto factory = factory_creator->Create(identifier); if (factory == nullptr) { @@ -38,16 +38,7 @@ Result LookUpFactory(const std::string& identifier) { return Status::Invalid( fmt::format("Failed to cast file system factory with identifier '{}'.", identifier)); } - return file_system_factory; -} - -} // namespace - -Result> FileSystemFactory::Get( - const std::string& identifier, const std::string& path, - const std::map& fs_options) { - PAIMON_ASSIGN_OR_RAISE(FileSystemFactory * factory, LookUpFactory(identifier)); - return factory->Create(path, fs_options); + return file_system_factory->Create(path, fs_options); } } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 8a050ae1..18793419 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -55,9 +55,9 @@ TEST(FileSystemCatalogTest, TestGetTableFileSystem) { // a catalog without per-table credentials serves the table data with the catalog wide // file system, whatever the table is - ASSERT_OK_AND_ASSIGN(auto table_fs, catalog.GetTableFileSystem(Identifier("db1", "t1"), {})); + ASSERT_OK_AND_ASSIGN(auto table_fs, catalog.GetTableFileSystem(Identifier("db1", "t1"))); ASSERT_EQ(catalog.GetFileSystem(), table_fs); - ASSERT_OK_AND_ASSIGN(auto other_fs, catalog.GetTableFileSystem(Identifier("db2", "t2"), {})); + ASSERT_OK_AND_ASSIGN(auto other_fs, catalog.GetTableFileSystem(Identifier("db2", "t2"))); ASSERT_EQ(catalog.GetFileSystem(), other_fs); } diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 36d04833..db9cee5c 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -199,10 +199,9 @@ Result> FileStoreCommit::Create( return Status::Invalid("a catalog commit requires a table identifier"); } // Use the credentials of this table, which a catalog issuing per-table temporary - // ones only hands out through GetTableFileSystem. The context options override the - // catalog ones the file system is built from. - PAIMON_ASSIGN_OR_RAISE(specific_fs, ctx->GetCatalog()->GetTableFileSystem( - ctx->GetIdentifier().value(), ctx->GetOptions())); + // ones only hands out through GetTableFileSystem. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx->GetOptions(), specific_fs)); diff --git a/src/paimon/core/operation/file_store_write.cpp b/src/paimon/core/operation/file_store_write.cpp index d3b9bd61..48f1c1dc 100644 --- a/src/paimon/core/operation/file_store_write.cpp +++ b/src/paimon/core/operation/file_store_write.cpp @@ -161,10 +161,9 @@ Result> FileStoreWrite::Create(std::unique_ptrGetCatalog()->GetTableFileSystem( - ctx->GetIdentifier().value(), ctx->GetOptions())); + // GetTableFileSystem, so the write uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value())); } PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(ctx->GetOptions(), specific_fs, diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index 38ad37db..6eca55d9 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -349,11 +349,9 @@ Result> ReadContextBuilder::Finish() { } if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the read uses the credentials of this table. The context - // options override the catalog ones the file system is built from. - PAIMON_ASSIGN_OR_RAISE( - impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value(), impl_->options_)); + // GetTableFileSystem, so the read uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch(/*identifier=*/std::nullopt, diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 5ca7aa8d..bdef9732 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -235,11 +235,9 @@ Result> ScanContextBuilder::Finish() { } if (impl_->catalog_ != nullptr && impl_->specific_file_system_ == nullptr) { // A catalog issuing per-table temporary credentials only hands them out through - // GetTableFileSystem, so the scan uses the credentials of this table. The context - // options override the catalog ones the file system is built from. - PAIMON_ASSIGN_OR_RAISE( - impl_->specific_file_system_, - impl_->catalog_->GetTableFileSystem(impl_->identifier_.value(), impl_->options_)); + // GetTableFileSystem, so the scan uses the credentials of this table. + PAIMON_ASSIGN_OR_RAISE(impl_->specific_file_system_, + impl_->catalog_->GetTableFileSystem(impl_->identifier_.value())); } std::shared_ptr executor = impl_->executor_ ? impl_->executor_ : CreateDefaultExecutor(); diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index b176982d..647da76c 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -518,7 +518,7 @@ Result> PartitionsSystemTable::BuildRows() const { // The files of each table are served with the credentials of that table, which a // catalog issuing temporary ones per table only hands out through the catalog. Result> table_fs_result = - context_.catalog->GetTableFileSystem(id, /*fs_options=*/{}); + context_.catalog->GetTableFileSystem(id); if (!table_fs_result.ok()) { if (table_fs_result.status().IsNotExist()) { continue; diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index 32ab1bd7..7591e53b 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -298,8 +298,7 @@ class PerTableFileSystemCatalog : public FileSystemCatalog { : FileSystemCatalog(fs, warehouse, options) {} Result> GetTableFileSystem( - const Identifier& identifier, - const std::map& /*fs_options*/) const override { + const Identifier& identifier) const override { requested.push_back(identifier.GetFullName()); if (failure) { return failure.value(); diff --git a/src/paimon/fs/oss/oss_file_system_factory.cpp b/src/paimon/fs/oss/oss_file_system_factory.cpp index 6c9ecc56..327586d0 100644 --- a/src/paimon/fs/oss/oss_file_system_factory.cpp +++ b/src/paimon/fs/oss/oss_file_system_factory.cpp @@ -180,13 +180,6 @@ Result> OssFileSystemFactory::Create( GetRequiredOption(options, bucket, kOssAccessKeyIdOption)); PAIMON_ASSIGN_OR_RAISE(std::string access_key_secret, GetRequiredOption(options, bucket, kOssAccessKeySecretOption)); - std::string security_token = GetOption(options, bucket, kOssSecurityTokenOption); - if (security_token.empty()) { - security_token = GetOption(options, bucket, kOssSessionTokenOption); - } - std::shared_ptr credentials_provider = - std::make_shared( - std::move(access_key_id), std::move(access_key_secret), std::move(security_token)); std::string endpoint = GetOption(options, bucket, kOssEndpointOption); std::string region = GetOption(options, bucket, kOssRegionOption); if (region.empty()) { @@ -204,6 +197,10 @@ Result> OssFileSystemFactory::Create( return Status::Invalid( "OSS region must be configured when the endpoint does not identify a region"); } + std::string security_token = GetOption(options, bucket, kOssSecurityTokenOption); + if (security_token.empty()) { + security_token = GetOption(options, bucket, kOssSessionTokenOption); + } oss2::ClientConfiguration config = oss2::ClientConfiguration::loadDefault(); if (!endpoint.empty()) { @@ -216,7 +213,8 @@ Result> OssFileSystemFactory::Create( config.signatureVersion = signature_version; } config.userAgent = "paimon-cpp"; - config.credentialsProvider = std::move(credentials_provider); + config.credentialsProvider = std::make_shared( + std::move(access_key_id), std::move(access_key_secret), std::move(security_token)); std::string path_style = GetOption(options, bucket, kOssUsePathStyleOption); if (!path_style.empty()) { std::optional value = StringUtils::StringToValue(path_style); diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 344fcd34..f127697e 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -457,7 +457,7 @@ std::shared_ptr RestCatalog::GetFileSystem() const { } Result> RestCatalog::GetTableFileSystem( - const Identifier& identifier, const std::map& fs_options) const { + const Identifier& identifier) const { // A file system the caller supplied to `Catalog::Create` authenticates its own accesses // and is used as-is, so it is handed out even when the server issues data tokens: // rebuilding a delegate from the options would discard it together with the @@ -469,12 +469,6 @@ Result> RestCatalog::GetTableFileSystem( // The credentials are issued for the data table, so a system table shares those of // the table it belongs to. PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); - // The caller's options override the catalog ones; the temporary credentials are merged - // over both when the delegate is built, so they always win. - std::map options = api_->GetMergedOptions(); - for (const auto& [key, value] : fs_options) { - options[key] = value; - } // Building the file system asks the server for nothing, the credentials are loaded on // the first access, so nothing is keyed by the table here: the file systems built from // the credentials are what a cache reuses and bounds. Keying an instance by its table @@ -482,14 +476,9 @@ Result> RestCatalog::GetTableFileSystem( // location. // // The shared cache is keyed by the credentials alone and holds the file systems built - // from the catalog options, which every table that overrides nothing agrees on. A call - // that does override them builds a different file system, so it must neither read nor - // write that cache: reading it would hand back a file system built from other options - // and silently drop the override. Such a file system keeps a private cache of its own, - // still keyed by the credentials so a rotation rebuilds it. - std::shared_ptr fs_cache = - fs_options.empty() ? token_fs_cache_ : nullptr; - return std::make_shared(api_, options, load_identifier, fs_cache); + // from the catalog options, which every table agrees on, so a rotation rebuilds them. + return std::make_shared(api_, api_->GetMergedOptions(), load_identifier, + token_fs_cache_); } Result> RestCatalog::ListSnapshots(const Identifier& identifier, diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index 5a8638bf..ef3049a7 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -81,13 +81,9 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// call returns an instance bound to the table it was asked for, whose credentials are /// loaded on the first access; the file systems built from them are shared through a /// bounded cache, so the tables the server issues the same credentials for share one - /// file system. `fs_options` override the catalog options the file system is built - /// from, the credentials still win over them; a call that overrides anything builds a - /// file system of its own rather than reading the shared cache, so its override is - /// never dropped for a cache hit. + /// file system. Result> GetTableFileSystem( - const Identifier& identifier, - const std::map& fs_options) const override; + const Identifier& identifier) const override; Result> GetTable(const Identifier& identifier) const override; Result> ListSnapshots(const Identifier& identifier, const std::string& branch) const override; diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index 9de451d0..e3abdc1a 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -548,7 +548,7 @@ TEST_F(RestCatalogTest, CreateMergesServerConfig) { TEST_F(RestCatalogTest, TableFileSystemWithoutDataToken) { ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, - catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t1"))); // without the data token the catalog wide credentials are used for the data as well ASSERT_EQ(catalog->GetFileSystem(), fs); } @@ -567,7 +567,7 @@ TEST_F(RestCatalogTest, TableFileSystemKeepsAnExplicitlySuppliedFileSystem) { RestCatalog::Create(kWarehouse, options_, custom_fs)); ASSERT_EQ(custom_fs, catalog->GetFileSystem()); ASSERT_OK_AND_ASSIGN(std::shared_ptr table_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t1"))); ASSERT_EQ(custom_fs, table_fs); // Handing out the supplied file system asks the server for no data token. ASSERT_TRUE(TokenRequests().empty()); @@ -576,7 +576,7 @@ TEST_F(RestCatalogTest, TableFileSystemKeepsAnExplicitlySuppliedFileSystem) { // identity above comes from the supplied file system, not from data tokens being off. ASSERT_OK_AND_ASSIGN(std::unique_ptr token_catalog, CreateRestCatalog()); ASSERT_OK_AND_ASSIGN(std::shared_ptr token_fs, - token_catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); + token_catalog->GetTableFileSystem(Identifier("db1", "t1"))); ASSERT_NE(custom_fs, token_fs); } @@ -584,7 +584,7 @@ TEST_F(RestCatalogTest, TableFileSystemWithDataToken) { options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, - catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t1"))); ASSERT_NE(nullptr, fs); ASSERT_NE(catalog->GetFileSystem(), fs); @@ -598,10 +598,10 @@ TEST_F(RestCatalogTest, TableFileSystemWithDataToken) { // a system table reads the files of the table it belongs to, so it is served the // credentials of that table ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1$snapshots"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t1$snapshots"))); ASSERT_OK(system_table_fs->Exists("/no-such-file").status()); ASSERT_OK_AND_ASSIGN(std::shared_ptr other_table_fs, - catalog->GetTableFileSystem(Identifier("db1", "t2"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t2"))); ASSERT_OK(other_table_fs->Exists("/no-such-file").status()); ASSERT_EQ(std::vector( {TokenPath("db1", "t1"), TokenPath("db1", "t1"), TokenPath("db1", "t2")}), @@ -615,10 +615,10 @@ TEST_F(RestCatalogTest, TableFileSystemIsBoundToTheTableItWasAskedFor) { // the database and the table are addressed separately, so identifiers that print the // same must not be served the credentials of one another ASSERT_OK_AND_ASSIGN(std::shared_ptr dotted_database, - catalog->GetTableFileSystem(Identifier("db1.a", "t1"), {})); + catalog->GetTableFileSystem(Identifier("db1.a", "t1"))); ASSERT_OK(dotted_database->Exists("/no-such-file").status()); ASSERT_OK_AND_ASSIGN(std::shared_ptr dotted_table, - catalog->GetTableFileSystem(Identifier("db1", "a.t1"), {})); + catalog->GetTableFileSystem(Identifier("db1", "a.t1"))); ASSERT_OK(dotted_table->Exists("/no-such-file").status()); ASSERT_EQ(std::vector({TokenPath("db1.a", "t1"), TokenPath("db1", "a.t1")}), TokenRequests()); @@ -630,17 +630,16 @@ TEST_F(RestCatalogTest, TableFileSystemNormalizesTheBranch) { // the main branch is the table itself, so it shares the credentials ASSERT_OK_AND_ASSIGN(std::shared_ptr main_branch_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1$branch_main"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_main"))); ASSERT_OK(main_branch_fs->Exists("/no-such-file").status()); // another branch is addressed as its own object on the server, so it gets its own // credentials ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1"))); ASSERT_OK(branch_fs->Exists("/no-such-file").status()); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr branch_system_fs, - catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1$snapshots"), {})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr branch_system_fs, + catalog->GetTableFileSystem(Identifier("db1", "t1$branch_b1$snapshots"))); ASSERT_OK(branch_system_fs->Exists("/no-such-file").status()); ASSERT_EQ(std::vector({TokenPath("db1", "t1"), TokenPath("db1", "t1$branch_b1"), TokenPath("db1", "t1$branch_b1")}), @@ -655,9 +654,9 @@ TEST_F(RestCatalogTest, TableFileSystemIsNotRetainedPerTable) { // another location is never served the credentials of the dropped one. What is cached // and bounded are the file systems built from the credentials, keyed by them. ASSERT_OK_AND_ASSIGN(std::shared_ptr first, - catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t1"))); ASSERT_OK_AND_ASSIGN(std::shared_ptr second, - catalog->GetTableFileSystem(Identifier("db1", "t1"), {})); + catalog->GetTableFileSystem(Identifier("db1", "t1"))); ASSERT_NE(first, second); ASSERT_OK(first->Exists("/no-such-file").status()); @@ -666,34 +665,6 @@ TEST_F(RestCatalogTest, TableFileSystemIsNotRetainedPerTable) { TokenRequests()); } -TEST_F(RestCatalogTest, TableFileSystemOptionsOverrideIsNotDroppedByTheSharedCache) { - options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); - Identifier identifier("db1", "t1"); - - // A call that overrides nothing builds its delegate from the catalog options and puts it - // in the cache the tables of this catalog share, keyed by the credentials. - ASSERT_OK_AND_ASSIGN(std::shared_ptr plain, - catalog->GetTableFileSystem(identifier, {})); - ASSERT_OK(plain->Exists("/no-such-file").status()); - - // The same table with an override must not reuse that cached delegate: it builds a file - // system of its own, so the overridden scheme reaches the delegate and building it fails - // with the scheme it was overridden to instead of silently keeping the cached local one. - ASSERT_OK_AND_ASSIGN( - std::shared_ptr overridden, - catalog->GetTableFileSystem(identifier, {{Options::FILE_SYSTEM, "no-such-file-system"}})); - Status status = overridden->Exists("/no-such-file").status(); - ASSERT_NOK(status); - ASSERT_NOK_WITH_MSG(status, "no-such-file-system"); - - // The override never entered the shared cache, so a following default call is still - // served a delegate built from the catalog options. - ASSERT_OK_AND_ASSIGN(std::shared_ptr plain_again, - catalog->GetTableFileSystem(identifier, {})); - ASSERT_OK(plain_again->Exists("/no-such-file").status()); -} - TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) { options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); @@ -702,7 +673,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); ASSERT_OK_AND_ASSIGN(std::string old_location, catalog->GetTableLocation(identifier)); ASSERT_OK_AND_ASSIGN(std::shared_ptr first, - catalog->GetTableFileSystem(identifier, {})); + catalog->GetTableFileSystem(identifier)); std::shared_ptr first_token_fs = std::dynamic_pointer_cast(first); ASSERT_NE(nullptr, first_token_fs); @@ -725,7 +696,7 @@ TEST_F(RestCatalogTest, RecreatedTableLoadsNewCredentialsBeforeOldTokenExpires) ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(identifier)); ASSERT_EQ(new_location, location); ASSERT_OK_AND_ASSIGN(std::shared_ptr second, - catalog->GetTableFileSystem(identifier, {})); + catalog->GetTableFileSystem(identifier)); ASSERT_NE(first, second); std::shared_ptr second_token_fs = std::dynamic_pointer_cast(second); diff --git a/src/paimon/testing/mock/mock_catalog.h b/src/paimon/testing/mock/mock_catalog.h index cf230bd0..8f5a84c1 100644 --- a/src/paimon/testing/mock/mock_catalog.h +++ b/src/paimon/testing/mock/mock_catalog.h @@ -232,8 +232,7 @@ class MockVersionManagedCatalog : public Catalog, public VersionManagedCatalog { /// test can tell it apart from the catalog-wide one; falls back to `GetFileSystem()`, /// matching the base default, when none was set. Result> GetTableFileSystem( - const Identifier& identifier, - const std::map& /*fs_options*/) const override { + const Identifier& identifier) const override { table_file_system_requests_.push_back(identifier); if (table_file_system_ != nullptr) { return table_file_system_; From 9291ecf3a05433fd9763a1af5c86a4d45aacab3a Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 21 Sep 2026 14:36:02 +0800 Subject: [PATCH 11/16] fix --- src/paimon/rest/rest_catalog.cpp | 5 +- src/paimon/rest/rest_credential_provider.cpp | 5 +- src/paimon/rest/rest_credential_provider.h | 9 +++- src/paimon/rest/rest_token_file_system.cpp | 50 ++----------------- src/paimon/rest/rest_token_file_system.h | 37 ++++++-------- .../rest/rest_token_file_system_test.cpp | 42 +++++++++------- 6 files changed, 56 insertions(+), 92 deletions(-) diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index f127697e..8f1a8ab6 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -477,7 +477,10 @@ Result> RestCatalog::GetTableFileSystem( // // The shared cache is keyed by the credentials alone and holds the file systems built // from the catalog options, which every table agrees on, so a rotation rebuilds them. - return std::make_shared(api_, api_->GetMergedOptions(), load_identifier, + const std::map& catalog_options = api_->GetMergedOptions(); + std::shared_ptr provider = + std::make_shared(api_, catalog_options, load_identifier); + return std::make_shared(std::move(provider), catalog_options, token_fs_cache_); } diff --git a/src/paimon/rest/rest_credential_provider.cpp b/src/paimon/rest/rest_credential_provider.cpp index ba29e90e..8a37c34a 100644 --- a/src/paimon/rest/rest_credential_provider.cpp +++ b/src/paimon/rest/rest_credential_provider.cpp @@ -60,7 +60,7 @@ bool RestCredentialProvider::ShouldRefresh() const { return token_->expires_at_millis - now_millis < RestApi::kTokenExpirationSafeTimeMillis; } -std::map RestCredentialProvider::MergeTokenOptions( +std::map RestCredentialProvider::ApplyDlfEndpointOverride( const std::map& token) const { std::map merged = token; // The DLF OSS endpoint overrides the standard one, since the credentials are issued @@ -80,7 +80,8 @@ Status RestCredentialProvider::RefreshToken() const { identifier_.ToString().c_str(), static_cast(response.GetExpiresAtMillis())); - token_ = RestToken{MergeTokenOptions(response.GetToken()), response.GetExpiresAtMillis()}; + token_ = + RestToken{ApplyDlfEndpointOverride(response.GetToken()), response.GetExpiresAtMillis()}; return Status::OK(); } diff --git a/src/paimon/rest/rest_credential_provider.h b/src/paimon/rest/rest_credential_provider.h index aafc8352..f9243fb9 100644 --- a/src/paimon/rest/rest_credential_provider.h +++ b/src/paimon/rest/rest_credential_provider.h @@ -97,8 +97,13 @@ class RestCredentialProvider : public CredentialProvider { /// Whether `token_` is absent or expires within the safe time. bool ShouldRefresh() const; - /// `catalog_options_` with `token` merged over it. - std::map MergeTokenOptions( + /// The issued `token` with the catalog's DLF OSS endpoint, when set, overriding the endpoint + /// the server reported: the credentials are issued for the DLF endpoint, not the one the + /// catalog was configured with. The token is otherwise left exactly as issued -- it is a file + /// system cache key and what `ValidToken()` serves, so it carries only the credentials, never + /// the whole catalog options; merging those over the token to build a delegate is the file + /// system's `MergeTokenOptions`. + std::map ApplyDlfEndpointOverride( const std::map& token) const; std::shared_ptr api_; diff --git a/src/paimon/rest/rest_token_file_system.cpp b/src/paimon/rest/rest_token_file_system.cpp index c1296e5b..55535ec4 100644 --- a/src/paimon/rest/rest_token_file_system.cpp +++ b/src/paimon/rest/rest_token_file_system.cpp @@ -21,24 +21,9 @@ #include #include -#include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" namespace paimon { -namespace { - -// Length of the "fs.." prefix of a file system option key, or 0 when `key` is not -// scheme-qualified. Only a scheme-qualified key such as "fs.oss.accessKeyId" can have a -// bucket-scoped variant such as "fs.oss.bucket..accessKeyId". -size_t FileSystemOptionPrefixLength(const std::string& key) { - if (!StringUtils::StartsWith(key, "fs.")) { - return 0; - } - size_t scheme_end = key.find('.', 3); - return scheme_end == std::string::npos ? 0 : scheme_end + 1; -} - -} // namespace std::shared_ptr RestTokenFileSystem::CreateFileSystemCache() { RestTokenFileSystemCache::Options cache_options; @@ -47,41 +32,17 @@ std::shared_ptr RestTokenFileSystem::CreateFileSystemC return std::make_shared(std::move(cache_options)); } -RestTokenFileSystem::RestTokenFileSystem(const std::shared_ptr& api, +RestTokenFileSystem::RestTokenFileSystem(std::shared_ptr provider, const std::map& catalog_options, - const Identifier& identifier, - std::shared_ptr fs_cache, - Clock clock) + std::shared_ptr fs_cache) : catalog_options_(catalog_options), - identifier_(identifier), - fs_cache_(fs_cache != nullptr ? std::move(fs_cache) : CreateFileSystemCache()), - provider_(std::make_shared(api, catalog_options, identifier, - std::move(clock))) {} + fs_cache_(std::move(fs_cache)), + provider_(std::move(provider)) {} std::map RestTokenFileSystem::MergeTokenOptions( const std::map& catalog_options, const RestToken& token) { std::map fs_options = catalog_options; for (const auto& [key, value] : token.token) { - // A file system resolves a bucket-scoped option such as - // "fs.oss.bucket..accessKeyId" ahead of the flat "fs.oss.accessKeyId" the - // token carries. Keeping a catalog's bucket-scoped credential would sign an access - // with that stale key pair together with the token's security token, an invalid - // combination, so drop the bucket-scoped variant of every option the token sets: the - // issued credentials then win whichever bucket the table's data lives in. - size_t prefix_length = FileSystemOptionPrefixLength(key); - if (prefix_length != 0) { - std::string bucket_prefix = key.substr(0, prefix_length) + "bucket."; - std::string bucket_suffix = "." + key.substr(prefix_length); - for (auto it = fs_options.begin(); it != fs_options.end();) { - if (it->first.size() > bucket_prefix.size() + bucket_suffix.size() && - StringUtils::StartsWith(it->first, bucket_prefix) && - StringUtils::EndsWith(it->first, bucket_suffix)) { - it = fs_options.erase(it); - } else { - ++it; - } - } - } fs_options[key] = value; } return fs_options; @@ -94,8 +55,7 @@ Result> RestTokenFileSystem::BuildFileSystem( /*specified_file_system=*/nullptr)); std::shared_ptr fs = core_options.GetFileSystem(); if (fs == nullptr) { - return Status::Invalid("failed to build the file system of the data token of ", - identifier_.ToString()); + return Status::Invalid("failed to build the file system from the data token credentials"); } return fs; } diff --git a/src/paimon/rest/rest_token_file_system.h b/src/paimon/rest/rest_token_file_system.h index fa5e5134..d17af9fb 100644 --- a/src/paimon/rest/rest_token_file_system.h +++ b/src/paimon/rest/rest_token_file_system.h @@ -18,18 +18,14 @@ #pragma once -#include #include -#include #include #include #include #include -#include "paimon/catalog/identifier.h" #include "paimon/common/utils/generic_lru_cache.h" #include "paimon/fs/file_system.h" -#include "paimon/rest/rest_api.h" #include "paimon/rest/rest_credential_provider.h" #include "paimon/result.h" #include "paimon/status.h" @@ -49,8 +45,6 @@ using RestTokenFileSystemCache = /// options, so the schemes configured for the catalog keep working. class RestTokenFileSystem : public FileSystem { public: - using Clock = RestCredentialProvider::Clock; - /// Bounds of the file system cache, matching the Java client: the file system of /// credentials that were not used for this long is dropped, which also keeps a stream /// opened just before a rotation from losing the file system it came from. @@ -60,18 +54,17 @@ class RestTokenFileSystem : public FileSystem { /// Creates a cache the file systems of many tables can share. static std::shared_ptr CreateFileSystemCache(); - /// @param api Client of the catalog that issues the credentials. Shared because this - /// file system commonly outlives the catalog it was obtained from. - /// @param catalog_options Options the credentials are merged over. - /// @param identifier The table the credentials are requested for. + /// @param provider Source of this table's credentials, built and owned by the catalog + /// and injected so this file system neither reaches the server nor knows + /// how the credentials are obtained. Shared because this file system + /// commonly outlives the catalog it was obtained from. + /// @param catalog_options Options the credentials are merged over to build a delegate. /// @param fs_cache Cache of the delegates, shared with the file systems of the other - /// tables of the same catalog. A private one is created when null. - /// @param clock Source of the current time, overridable for tests. - RestTokenFileSystem(const std::shared_ptr& api, + /// tables of the same catalog so that tables issued equal credentials + /// reuse one delegate. The catalog owns it and hands it out. + RestTokenFileSystem(std::shared_ptr provider, const std::map& catalog_options, - const Identifier& identifier, - std::shared_ptr fs_cache = nullptr, - Clock clock = std::chrono::system_clock::now); + std::shared_ptr fs_cache); ~RestTokenFileSystem() override = default; @@ -96,12 +89,11 @@ class RestTokenFileSystem : public FileSystem { /// credentials are requested with. Result ValidToken() const; - /// Merges the issued credentials over the catalog options the delegate is built from. A - /// bucket-scoped variant of an option the credentials set, such as - /// "fs.oss.bucket..accessKeyId" for the credentials' "fs.oss.accessKeyId", is - /// dropped: a file system resolves the bucket-scoped option first, so keeping a catalog's - /// one would sign an access with a stale key pair and the issued security token, an invalid - /// combination. Exposed for tests. + /// Merges the issued credentials over the catalog options the delegate is built from. The + /// credentials are themselves file system options, so they are overlaid key by key and win + /// wherever they overlap, mirroring the Java client; how a concrete file system resolves the + /// catalog options the token does not carry is that file system's own concern, so this merge + /// stays scheme-agnostic. Exposed for tests. static std::map MergeTokenOptions( const std::map& catalog_options, const RestToken& token); @@ -115,7 +107,6 @@ class RestTokenFileSystem : public FileSystem { Result> BuildFileSystem(const RestToken& token) const; std::map catalog_options_; - Identifier identifier_; std::shared_ptr fs_cache_; /// The credentials this file system delegates with, reloaded before they expire. diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index 9e65a1bd..13c3615f 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -40,6 +40,7 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" +#include "paimon/rest/rest_credential_provider.h" #include "paimon/rest/rest_messages.h" #include "paimon/testing/utils/testharness.h" @@ -157,11 +158,16 @@ class RestTokenFileSystemTest : public ::testing::Test { return nullptr; } std::shared_ptr shared_api(std::move(api).value()); - return std::make_shared( - shared_api, catalog_options_, Identifier("db1", "t1"), std::move(fs_cache), [this] { + if (fs_cache == nullptr) { + fs_cache = RestTokenFileSystem::CreateFileSystemCache(); + } + std::shared_ptr provider = std::make_shared( + shared_api, catalog_options_, Identifier("db1", "t1"), [this] { return std::chrono::system_clock::time_point( std::chrono::milliseconds(now_millis_.load())); }); + return std::make_shared(std::move(provider), catalog_options_, + std::move(fs_cache)); } // Writes `content` to a file of the temp directory with the local file system and @@ -338,8 +344,11 @@ TEST_F(RestTokenFileSystemTest, DefaultClockIsTheSystemClock) { } ASSERT_OK_AND_ASSIGN(std::unique_ptr api, RestApi::Create(catalog_options_, "", /*config_required=*/false)); - RestTokenFileSystem fs(std::shared_ptr(std::move(api)), catalog_options_, - Identifier("db1", "t1")); + // the provider is built with its default clock, the system clock + std::shared_ptr provider = std::make_shared( + std::shared_ptr(std::move(api)), catalog_options_, Identifier("db1", "t1")); + RestTokenFileSystem fs(provider, catalog_options_, + RestTokenFileSystem::CreateFileSystemCache()); ASSERT_OK(fs.ValidToken().status()); ASSERT_OK(fs.ValidToken().status()); ASSERT_EQ(1, state_->request_count.load()); @@ -377,16 +386,15 @@ TEST_F(RestTokenFileSystemTest, TokenOverridesTheCatalogFileSystemOptions) { ASSERT_EQ(1, state_->request_count.load()); } -TEST(RestTokenFileSystemMergeTokenOptions, IssuedCredentialsWinOverBucketScopedCatalogOnes) { - // The catalog is configured with bucket-scoped credentials, which a file system resolves - // ahead of the flat options, so they must not shadow the credentials the server issues. +TEST(RestTokenFileSystemMergeTokenOptions, IssuedCredentialsOverrideTheCatalogOptions) { + // The issued credentials are file system options merged over the catalog options: they win + // wherever the two overlap, and every catalog option the token does not carry is kept as-is, + // mirroring the Java client. How the file system resolves the kept options -- a per-bucket + // variant or an alias -- is its own concern, so the merge leaves them untouched. std::map catalog_options = { - {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, - {"fs.oss.bucket.b.accessKeySecret", "catalog-bucket-sk"}, - {"fs.oss.bucket.other.accessKeyId", "catalog-other-ak"}, {"fs.oss.accessKeyId", "catalog-ak"}, {"fs.oss.endpoint", "catalog-endpoint"}, - {"fs.oss.bucket.b.endpoint", "catalog-bucket-endpoint"}, + {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, {"unrelated", "kept"}, }; RestToken token; @@ -397,19 +405,15 @@ TEST(RestTokenFileSystemMergeTokenOptions, IssuedCredentialsWinOverBucketScopedC std::map merged = RestTokenFileSystem::MergeTokenOptions(catalog_options, token); - // The issued credentials are present and every bucket-scoped variant of them is gone, so - // the per-bucket resolution a file system does falls back to the issued flat options - // rather than signing with a stale catalog key pair and the issued security token. + // The token value replaces the catalog's for the key they share, and the token-only keys are + // added. ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); ASSERT_EQ("token-sk", merged.at("fs.oss.accessKeySecret")); ASSERT_EQ("token-sts", merged.at("fs.oss.securityToken")); - ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeyId")); - ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeySecret")); - ASSERT_EQ(0u, merged.count("fs.oss.bucket.other.accessKeyId")); - // Options the token does not set keep their catalog value, bucket-scoped ones included. + // Catalog options the token does not set are kept untouched, whatever their form. ASSERT_EQ("catalog-endpoint", merged.at("fs.oss.endpoint")); - ASSERT_EQ("catalog-bucket-endpoint", merged.at("fs.oss.bucket.b.endpoint")); + ASSERT_EQ("catalog-bucket-ak", merged.at("fs.oss.bucket.b.accessKeyId")); ASSERT_EQ("kept", merged.at("unrelated")); } From 4a0c6e29b83a3b5374b9eac01ccebd9f8ca583cb Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 21 Sep 2026 16:24:47 +0800 Subject: [PATCH 12/16] fix --- docs/source/user_guide/catalog.rst | 8 ++ include/paimon/fs/credential_provider.h | 24 +++- src/paimon/rest/rest_catalog.cpp | 2 +- src/paimon/rest/rest_credential_provider.cpp | 23 ++-- src/paimon/rest/rest_credential_provider.h | 24 ++-- .../rest/rest_credential_provider_test.cpp | 22 ++-- src/paimon/rest/rest_token_file_system.cpp | 16 +-- src/paimon/rest/rest_token_file_system.h | 8 -- .../rest/rest_token_file_system_test.cpp | 105 ++++++++++++------ 9 files changed, 142 insertions(+), 90 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 7848d7f7..d881968c 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -404,6 +404,14 @@ the factory is called with are those of the accesses to authenticate, which is where an implementation reads its own configuration, such as the address of the token service, from. +Merge the credentials into your file system's options with +``CredentialProvider::MergeOptionsWithCredentials`` rather than by hand, so they are +shaped the same way the built-in data token file system shapes them. The default +overlays the credentials key by key over the base options, mirroring the Java +client. A provider that knows the file system its credentials are for overrides +``MergeOptionsWithCredentials`` to normalize option aliases or clear the stale +bucket-scoped variants the fresh credentials replace. + All factories share one identifier space, so an identifier a file system factory already takes — ``oss``, ``s3``, ``local``, ``jindo`` — would replace it; name the provider after where its credentials come from instead. diff --git a/include/paimon/fs/credential_provider.h b/include/paimon/fs/credential_provider.h index 27db2f81..8e45cc14 100644 --- a/include/paimon/fs/credential_provider.h +++ b/include/paimon/fs/credential_provider.h @@ -46,9 +46,29 @@ class PAIMON_EXPORT CredentialProvider { virtual ~CredentialProvider() = default; /// Returns the credentials to sign an access with, reloading them when they are about - /// to expire. The keys are file system options, e.g. "fs.oss.accessKeyId" or - /// "fs.oss.securityToken", so a caller merges them over its own file system options. + /// to expire. The keys are file system options, so a caller merges them over its own + /// file system options with `MergeOptionsWithCredentials`. virtual Result> GetCredentials() const = 0; + + /// Merges the issued credentials into the file system options a delegate is built from. + /// This is the canonical way to shape credentials into the options of an access: a + /// caller that brings its own file system calls it so the credentials are applied the + /// same way the built-in data token file system applies them. + /// + /// The default overlays the credentials key by key over `base_options`, so they win + /// wherever they overlap, mirroring the Java client and staying scheme-agnostic. A + /// provider that knows the file system its credentials are for overrides this to + /// normalize option aliases or clear stale bucket-scoped variants the credentials + /// replace. + virtual std::map MergeOptionsWithCredentials( + const std::map& base_options, + const std::map& credentials) const { + std::map merged = base_options; + for (const auto& [key, value] : credentials) { + merged[key] = value; + } + return merged; + } }; } // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index 8f1a8ab6..da88cb24 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -479,7 +479,7 @@ Result> RestCatalog::GetTableFileSystem( // from the catalog options, which every table agrees on, so a rotation rebuilds them. const std::map& catalog_options = api_->GetMergedOptions(); std::shared_ptr provider = - std::make_shared(api_, catalog_options, load_identifier); + std::make_shared(api_, load_identifier); return std::make_shared(std::move(provider), catalog_options, token_fs_cache_); } diff --git a/src/paimon/rest/rest_credential_provider.cpp b/src/paimon/rest/rest_credential_provider.cpp index 8a37c34a..a4538636 100644 --- a/src/paimon/rest/rest_credential_provider.cpp +++ b/src/paimon/rest/rest_credential_provider.cpp @@ -42,11 +42,9 @@ size_t RestToken::Hash::operator()(const RestToken& rest_token) const { return result; } -RestCredentialProvider::RestCredentialProvider( - const std::shared_ptr& api, const std::map& catalog_options, - const Identifier& identifier, Clock clock) +RestCredentialProvider::RestCredentialProvider(const std::shared_ptr& api, + const Identifier& identifier, Clock clock) : api_(api), - catalog_options_(catalog_options), identifier_(identifier), clock_(std::move(clock)), logger_(Logger::GetLogger("RestCredentialProvider")) {} @@ -60,13 +58,17 @@ bool RestCredentialProvider::ShouldRefresh() const { return token_->expires_at_millis - now_millis < RestApi::kTokenExpirationSafeTimeMillis; } -std::map RestCredentialProvider::ApplyDlfEndpointOverride( - const std::map& token) const { - std::map merged = token; +std::map RestCredentialProvider::MergeOptionsWithCredentials( + const std::map& base_options, + const std::map& credentials) const { + std::map merged = base_options; + for (const auto& [key, value] : credentials) { + merged[key] = value; + } // The DLF OSS endpoint overrides the standard one, since the credentials are issued // for the DLF endpoint rather than for the endpoint the catalog was configured with. - auto dlf_oss_endpoint = catalog_options_.find(CatalogOptions::DLF_OSS_ENDPOINT); - if (dlf_oss_endpoint != catalog_options_.end() && !dlf_oss_endpoint->second.empty()) { + auto dlf_oss_endpoint = base_options.find(CatalogOptions::DLF_OSS_ENDPOINT); + if (dlf_oss_endpoint != base_options.end() && !dlf_oss_endpoint->second.empty()) { merged[kOssEndpointOption] = dlf_oss_endpoint->second; } return merged; @@ -80,8 +82,7 @@ Status RestCredentialProvider::RefreshToken() const { identifier_.ToString().c_str(), static_cast(response.GetExpiresAtMillis())); - token_ = - RestToken{ApplyDlfEndpointOverride(response.GetToken()), response.GetExpiresAtMillis()}; + token_ = RestToken{response.GetToken(), response.GetExpiresAtMillis()}; return Status::OK(); } diff --git a/src/paimon/rest/rest_credential_provider.h b/src/paimon/rest/rest_credential_provider.h index f9243fb9..3860ee67 100644 --- a/src/paimon/rest/rest_credential_provider.h +++ b/src/paimon/rest/rest_credential_provider.h @@ -68,12 +68,9 @@ class RestCredentialProvider : public CredentialProvider { /// @param api Client of the catalog that issues the credentials. Shared because a /// provider commonly outlives the catalog it was obtained from. - /// @param catalog_options Options the credentials are merged over. /// @param identifier The table the credentials are requested for. /// @param clock Source of the current time, overridable for tests. - RestCredentialProvider(const std::shared_ptr& api, - const std::map& catalog_options, - const Identifier& identifier, + RestCredentialProvider(const std::shared_ptr& api, const Identifier& identifier, Clock clock = std::chrono::system_clock::now); ~RestCredentialProvider() override = default; @@ -89,6 +86,15 @@ class RestCredentialProvider : public CredentialProvider { /// source. Result> GetCredentials() const override; + /// Merges the issued credentials over `base_options`, then corrects the OSS endpoint: the + /// credentials are issued for the catalog's DLF OSS endpoint, which overrides both the + /// endpoint the catalog was configured with and the one the server reported. The correction + /// lives here rather than in the token so the token stays a minimal cache key that carries + /// only the issued credentials. + std::map MergeOptionsWithCredentials( + const std::map& base_options, + const std::map& credentials) const override; + private: /// Reloads the credentials from the server. Called with the write lock of `mutex_` /// held. @@ -97,17 +103,7 @@ class RestCredentialProvider : public CredentialProvider { /// Whether `token_` is absent or expires within the safe time. bool ShouldRefresh() const; - /// The issued `token` with the catalog's DLF OSS endpoint, when set, overriding the endpoint - /// the server reported: the credentials are issued for the DLF endpoint, not the one the - /// catalog was configured with. The token is otherwise left exactly as issued -- it is a file - /// system cache key and what `ValidToken()` serves, so it carries only the credentials, never - /// the whole catalog options; merging those over the token to build a delegate is the file - /// system's `MergeTokenOptions`. - std::map ApplyDlfEndpointOverride( - const std::map& token) const; - std::shared_ptr api_; - std::map catalog_options_; Identifier identifier_; Clock clock_; std::shared_ptr logger_; diff --git a/src/paimon/rest/rest_credential_provider_test.cpp b/src/paimon/rest/rest_credential_provider_test.cpp index 35548db3..2a2cabeb 100644 --- a/src/paimon/rest/rest_credential_provider_test.cpp +++ b/src/paimon/rest/rest_credential_provider_test.cpp @@ -117,7 +117,7 @@ class RestCredentialProviderTest : public ::testing::Test { } std::shared_ptr shared_api(std::move(api).value()); return std::make_shared( - shared_api, catalog_options_, Identifier("db1", "t1"), [this] { + shared_api, Identifier("db1", "t1"), [this] { return std::chrono::system_clock::time_point( std::chrono::milliseconds(now_millis_.load())); }); @@ -194,7 +194,7 @@ TEST_F(RestCredentialProviderTest, ExpiredTokenReloadsOnEveryCall) { ASSERT_EQ(2, state_->request_count.load()); } -TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpoint) { +TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpointOnMerge) { catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = "dlf-endpoint"; { std::lock_guard lock(state_->mutex); @@ -203,14 +203,20 @@ TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpoint) { std::shared_ptr provider = CreateProvider(); ASSERT_NE(nullptr, provider); + // the token is the cache key and stays exactly as issued, carrying no catalog secrets and + // not the endpoint correction, which is a merge concern ASSERT_OK_AND_ASSIGN(RestToken token, provider->ValidToken()); ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); - // the endpoint the credentials were issued for wins over the one the server reported - ASSERT_EQ("dlf-endpoint", token.token.at(kOssEndpointOption)); + ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); - // the catalog options are not part of the token, so its secrets stay private ASSERT_EQ(0u, token.token.count(CatalogOptions::TOKEN)); ASSERT_EQ(2u, token.token.size()); + + // merging shapes the credentials into the file system options: the endpoint the credentials + // were issued for wins over the one the server reported + Credentials merged = provider->MergeOptionsWithCredentials(catalog_options_, token.token); + ASSERT_EQ("ak-1", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ("dlf-endpoint", merged.at(kOssEndpointOption)); } TEST_F(RestCredentialProviderTest, EmptyDlfOssEndpointIsNotApplied) { @@ -222,10 +228,10 @@ TEST_F(RestCredentialProviderTest, EmptyDlfOssEndpointIsNotApplied) { std::shared_ptr provider = CreateProvider(); ASSERT_NE(nullptr, provider); - // an unset dlf endpoint leaves the endpoint the server reported alone + // an unset dlf endpoint leaves the endpoint the credentials carry alone through the merge ASSERT_OK_AND_ASSIGN(RestToken token, provider->ValidToken()); - ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); - ASSERT_EQ(1u, token.token.size()); + Credentials merged = provider->MergeOptionsWithCredentials(catalog_options_, token.token); + ASSERT_EQ("server-endpoint", merged.at(kOssEndpointOption)); } TEST_F(RestCredentialProviderTest, ForbiddenIsReportedToTheCaller) { diff --git a/src/paimon/rest/rest_token_file_system.cpp b/src/paimon/rest/rest_token_file_system.cpp index 55535ec4..e5b673de 100644 --- a/src/paimon/rest/rest_token_file_system.cpp +++ b/src/paimon/rest/rest_token_file_system.cpp @@ -39,20 +39,12 @@ RestTokenFileSystem::RestTokenFileSystem(std::shared_ptr fs_cache_(std::move(fs_cache)), provider_(std::move(provider)) {} -std::map RestTokenFileSystem::MergeTokenOptions( - const std::map& catalog_options, const RestToken& token) { - std::map fs_options = catalog_options; - for (const auto& [key, value] : token.token) { - fs_options[key] = value; - } - return fs_options; -} - Result> RestTokenFileSystem::BuildFileSystem( const RestToken& token) const { - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(MergeTokenOptions(catalog_options_, token), - /*specified_file_system=*/nullptr)); + PAIMON_ASSIGN_OR_RAISE( + CoreOptions core_options, + CoreOptions::FromMap(provider_->MergeOptionsWithCredentials(catalog_options_, token.token), + /*specified_file_system=*/nullptr)); std::shared_ptr fs = core_options.GetFileSystem(); if (fs == nullptr) { return Status::Invalid("failed to build the file system from the data token credentials"); diff --git a/src/paimon/rest/rest_token_file_system.h b/src/paimon/rest/rest_token_file_system.h index d17af9fb..45f95341 100644 --- a/src/paimon/rest/rest_token_file_system.h +++ b/src/paimon/rest/rest_token_file_system.h @@ -89,14 +89,6 @@ class RestTokenFileSystem : public FileSystem { /// credentials are requested with. Result ValidToken() const; - /// Merges the issued credentials over the catalog options the delegate is built from. The - /// credentials are themselves file system options, so they are overlaid key by key and win - /// wherever they overlap, mirroring the Java client; how a concrete file system resolves the - /// catalog options the token does not carry is that file system's own concern, so this merge - /// stays scheme-agnostic. Exposed for tests. - static std::map MergeTokenOptions( - const std::map& catalog_options, const RestToken& token); - private: /// Returns the file system of the current credentials, reloading them when they /// expire in less than `RestApi::kTokenExpirationSafeTimeMillis`. diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index 13c3615f..925622be 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -37,6 +37,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/defs.h" +#include "paimon/fs/credential_provider.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" @@ -161,8 +162,8 @@ class RestTokenFileSystemTest : public ::testing::Test { if (fs_cache == nullptr) { fs_cache = RestTokenFileSystem::CreateFileSystemCache(); } - std::shared_ptr provider = std::make_shared( - shared_api, catalog_options_, Identifier("db1", "t1"), [this] { + std::shared_ptr provider = + std::make_shared(shared_api, Identifier("db1", "t1"), [this] { return std::chrono::system_clock::time_point( std::chrono::milliseconds(now_millis_.load())); }); @@ -245,8 +246,9 @@ TEST_F(RestTokenFileSystemTest, ValidTokenCarriesOnlyTheServerCredentials) { ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); - // the endpoint the credentials were issued for wins over the one the server reported - ASSERT_EQ("dlf-endpoint", token.token.at(kOssEndpointOption)); + // the token is the cache key and carries only the issued credentials, never the catalog + // secrets nor the DLF endpoint correction, which the provider applies only when merging + ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); // the catalog options are not part of the token, so its secrets stay private ASSERT_EQ(0u, token.token.count(CatalogOptions::TOKEN)); @@ -346,7 +348,7 @@ TEST_F(RestTokenFileSystemTest, DefaultClockIsTheSystemClock) { RestApi::Create(catalog_options_, "", /*config_required=*/false)); // the provider is built with its default clock, the system clock std::shared_ptr provider = std::make_shared( - std::shared_ptr(std::move(api)), catalog_options_, Identifier("db1", "t1")); + std::shared_ptr(std::move(api)), Identifier("db1", "t1")); RestTokenFileSystem fs(provider, catalog_options_, RestTokenFileSystem::CreateFileSystemCache()); ASSERT_OK(fs.ValidToken().status()); @@ -354,21 +356,6 @@ TEST_F(RestTokenFileSystemTest, DefaultClockIsTheSystemClock) { ASSERT_EQ(1, state_->request_count.load()); } -TEST_F(RestTokenFileSystemTest, EmptyDlfOssEndpointIsNotApplied) { - catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = ""; - { - std::lock_guard lock(state_->mutex); - state_->token = {{kOssEndpointOption, "server-endpoint"}}; - } - std::shared_ptr fs = CreateFileSystem(); - ASSERT_NE(nullptr, fs); - - // an unset dlf endpoint leaves the endpoint the server reported alone - ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); - ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); - ASSERT_EQ(1u, token.token.size()); -} - TEST_F(RestTokenFileSystemTest, TokenOverridesTheCatalogFileSystemOptions) { // the credentials must reach the options the delegate is built from: the catalog is // configured with the local file system, yet an option of the token replaces it @@ -386,37 +373,87 @@ TEST_F(RestTokenFileSystemTest, TokenOverridesTheCatalogFileSystemOptions) { ASSERT_EQ(1, state_->request_count.load()); } -TEST(RestTokenFileSystemMergeTokenOptions, IssuedCredentialsOverrideTheCatalogOptions) { - // The issued credentials are file system options merged over the catalog options: they win - // wherever the two overlap, and every catalog option the token does not carry is kept as-is, - // mirroring the Java client. How the file system resolves the kept options -- a per-bucket - // variant or an alias -- is its own concern, so the merge leaves them untouched. - std::map catalog_options = { +TEST(CredentialProviderMergeOptionsWithCredentials, DefaultOverlaysCredentialsOverTheBaseOptions) { + // The default merge treats the credentials as file system options overlaid over the base + // options: they win wherever the two overlap, and every base option the credentials do not + // carry is kept as-is, mirroring the Java client. How the file system resolves the kept + // options -- a per-bucket variant or an alias -- is its own concern, so the merge leaves them + // untouched. + class DefaultProvider : public CredentialProvider { + public: + Result> GetCredentials() const override { + return std::map{}; + } + }; + DefaultProvider provider; + std::map base_options = { {"fs.oss.accessKeyId", "catalog-ak"}, {"fs.oss.endpoint", "catalog-endpoint"}, {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, {"unrelated", "kept"}, }; - RestToken token; - token.token = {{"fs.oss.accessKeyId", "token-ak"}, - {"fs.oss.accessKeySecret", "token-sk"}, - {"fs.oss.securityToken", "token-sts"}}; + std::map credentials = {{"fs.oss.accessKeyId", "token-ak"}, + {"fs.oss.accessKeySecret", "token-sk"}, + {"fs.oss.securityToken", "token-sts"}}; std::map merged = - RestTokenFileSystem::MergeTokenOptions(catalog_options, token); + provider.MergeOptionsWithCredentials(base_options, credentials); - // The token value replaces the catalog's for the key they share, and the token-only keys are - // added. + // The credential value replaces the base one for the key they share, and the credential-only + // keys are added. ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); ASSERT_EQ("token-sk", merged.at("fs.oss.accessKeySecret")); ASSERT_EQ("token-sts", merged.at("fs.oss.securityToken")); - // Catalog options the token does not set are kept untouched, whatever their form. + // Base options the credentials do not set are kept untouched, whatever their form. ASSERT_EQ("catalog-endpoint", merged.at("fs.oss.endpoint")); ASSERT_EQ("catalog-bucket-ak", merged.at("fs.oss.bucket.b.accessKeyId")); ASSERT_EQ("kept", merged.at("unrelated")); } +TEST(CredentialProviderMergeOptionsWithCredentials, OverrideCanReshapeTheMergedOptions) { + // A provider that knows the file system its credentials are for overrides + // MergeOptionsWithCredentials to clear the bucket-scoped variants the fresh credentials + // replace, which the default overlay would keep. This is the extension point an OSS-aware + // plugin uses. + class BucketClearingProvider : public CredentialProvider { + public: + Result> GetCredentials() const override { + return std::map{}; + } + std::map MergeOptionsWithCredentials( + const std::map& base_options, + const std::map& credentials) const override { + std::map merged; + for (const auto& [key, value] : base_options) { + if (key.rfind("fs.oss.bucket.", 0) == 0) { + continue; + } + merged[key] = value; + } + for (const auto& [key, value] : credentials) { + merged[key] = value; + } + return merged; + } + }; + BucketClearingProvider provider; + std::map base_options = { + {"fs.oss.accessKeyId", "catalog-ak"}, + {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, + {"unrelated", "kept"}, + }; + std::map credentials = {{"fs.oss.accessKeyId", "token-ak"}}; + + std::map merged = + provider.MergeOptionsWithCredentials(base_options, credentials); + + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + // the bucket-scoped variant the credentials replace is cleared by the override + ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeyId")); + ASSERT_EQ("kept", merged.at("unrelated")); +} + TEST_F(RestTokenFileSystemTest, ConcurrentFirstAccessLoadsTheTokenOnce) { std::string path = WriteFile("data", "paimon"); std::shared_ptr fs = CreateFileSystem(); From c827e4ec6a9bbbfca2c8347050f6be2d1fbb6478 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 21 Sep 2026 16:40:58 +0800 Subject: [PATCH 13/16] fix --- src/paimon/rest/rest_credential_provider.cpp | 58 +++++++++++++++++++ .../rest/rest_credential_provider_test.cpp | 57 ++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/src/paimon/rest/rest_credential_provider.cpp b/src/paimon/rest/rest_credential_provider.cpp index a4538636..a5861508 100644 --- a/src/paimon/rest/rest_credential_provider.cpp +++ b/src/paimon/rest/rest_credential_provider.cpp @@ -18,12 +18,15 @@ #include "paimon/rest/rest_credential_provider.h" +#include #include #include #include +#include #include #include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" namespace paimon { @@ -31,6 +34,24 @@ namespace { /// Declared here rather than taken from the OSS file system, which is an optional build /// component this module must not depend on. constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; +constexpr const char kOssOptionPrefix[] = "fs.oss."; +constexpr const char kOssBucketPrefix[] = "fs.oss.bucket."; +// The two OSS names for the STS token: the backend reads securityToken first and only +// consults sessionToken when it is empty, so a token under either name has to clear the +// catalog value under both. +constexpr const char kOssSecurityTokenSuffix[] = "securityToken"; +constexpr const char kOssSessionTokenSuffix[] = "sessionToken"; + +/// The suffix of a flat OSS option, i.e. what follows "fs.oss." for a key like +/// "fs.oss.accessKeyId"; empty for a bucket-scoped ("fs.oss.bucket..") or a +/// non-OSS key. +std::string FlatOssOptionSuffix(const std::string& key) { + if (!StringUtils::StartsWith(key, kOssOptionPrefix) || + StringUtils::StartsWith(key, kOssBucketPrefix)) { + return ""; + } + return key.substr(std::string(kOssOptionPrefix).size()); +} } // namespace size_t RestToken::Hash::operator()(const RestToken& rest_token) const { @@ -65,6 +86,43 @@ std::map RestCredentialProvider::MergeOptionsWithCrede for (const auto& [key, value] : credentials) { merged[key] = value; } + // The OSS backend resolves a bucket-scoped option ("fs.oss.bucket..") ahead of the + // flat one and reads "fs.oss.securityToken" ahead of its "fs.oss.sessionToken" alias, so a + // stale catalog value can shadow a credential the token just refreshed. Drop the stale + // variants of every credential the token supplies -- including both security-token aliases + // when it supplies either -- so the issued credential is the one that wins. + std::set refreshed_suffixes; + bool refreshed_security_token = false; + for (const auto& [key, value] : credentials) { + std::string suffix = FlatOssOptionSuffix(key); + if (suffix.empty()) { + continue; + } + refreshed_suffixes.insert(suffix); + refreshed_security_token = refreshed_security_token || suffix == kOssSecurityTokenSuffix || + suffix == kOssSessionTokenSuffix; + } + if (refreshed_security_token) { + refreshed_suffixes.insert(kOssSecurityTokenSuffix); + refreshed_suffixes.insert(kOssSessionTokenSuffix); + } + for (auto it = merged.begin(); it != merged.end();) { + const std::string& key = it->first; + bool stale_bucket_scoped = StringUtils::StartsWith(key, kOssBucketPrefix) && + std::any_of(refreshed_suffixes.begin(), refreshed_suffixes.end(), + [&](const std::string& suffix) { + return StringUtils::EndsWith(key, "." + suffix); + }); + bool stale_security_token_alias = + refreshed_security_token && credentials.find(key) == credentials.end() && + (key == std::string(kOssOptionPrefix) + kOssSecurityTokenSuffix || + key == std::string(kOssOptionPrefix) + kOssSessionTokenSuffix); + if (stale_bucket_scoped || stale_security_token_alias) { + it = merged.erase(it); + } else { + ++it; + } + } // The DLF OSS endpoint overrides the standard one, since the credentials are issued // for the DLF endpoint rather than for the endpoint the catalog was configured with. auto dlf_oss_endpoint = base_options.find(CatalogOptions::DLF_OSS_ENDPOINT); diff --git a/src/paimon/rest/rest_credential_provider_test.cpp b/src/paimon/rest/rest_credential_provider_test.cpp index 2a2cabeb..ad753b56 100644 --- a/src/paimon/rest/rest_credential_provider_test.cpp +++ b/src/paimon/rest/rest_credential_provider_test.cpp @@ -234,6 +234,63 @@ TEST_F(RestCredentialProviderTest, EmptyDlfOssEndpointIsNotApplied) { ASSERT_EQ("server-endpoint", merged.at(kOssEndpointOption)); } +TEST_F(RestCredentialProviderTest, IssuedCredentialsClearStaleBucketScopedCatalogVariants) { + // The OSS backend resolves a bucket-scoped option ahead of the flat one, so a stale + // bucket-scoped catalog value would shadow the flat credential the token just refreshed. + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Credentials base = { + {"fs.oss.accessKeyId", "catalog-ak"}, + {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, + {"unrelated", "kept"}, + }; + Credentials credentials = {{"fs.oss.accessKeyId", "token-ak"}}; + + Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeyId")); + ASSERT_EQ("kept", merged.at("unrelated")); +} + +TEST_F(RestCredentialProviderTest, SessionTokenCredentialClearsStaleSecurityTokenAliases) { + // A token carrying a fresh session token must clear a stale catalog security token -- the + // OSS backend reads securityToken first, so leaving it in place would pair the refreshed + // key pair with the old STS token and fail authentication. Both the flat and the + // bucket-scoped stale securityToken have to go. + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Credentials base = { + {"fs.oss.securityToken", "stale-sts"}, + {"fs.oss.bucket.b.securityToken", "stale-bucket-sts"}, + }; + Credentials credentials = {{"fs.oss.accessKeyId", "token-ak"}, + {"fs.oss.accessKeySecret", "token-sk"}, + {"fs.oss.sessionToken", "fresh-sts"}}; + + Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ("token-sk", merged.at("fs.oss.accessKeySecret")); + ASSERT_EQ("fresh-sts", merged.at("fs.oss.sessionToken")); + ASSERT_EQ(0u, merged.count("fs.oss.securityToken")); + ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.securityToken")); +} + +TEST_F(RestCredentialProviderTest, SecurityTokenCredentialClearsStaleSessionTokenAlias) { + // The alias goes the other way too: a fresh securityToken clears a stale sessionToken so the + // backend cannot fall back to it. + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Credentials base = {{"fs.oss.sessionToken", "stale-sts"}}; + Credentials credentials = {{"fs.oss.securityToken", "fresh-sts"}}; + + Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); + ASSERT_EQ("fresh-sts", merged.at("fs.oss.securityToken")); + ASSERT_EQ(0u, merged.count("fs.oss.sessionToken")); +} + TEST_F(RestCredentialProviderTest, ForbiddenIsReportedToTheCaller) { { std::lock_guard lock(state_->mutex); From 3abb9a7cf2f96f612f1fe56a75ab8d85133c78db Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 21 Sep 2026 16:55:47 +0800 Subject: [PATCH 14/16] fix --- src/paimon/rest/rest_credential_provider.cpp | 8 ++++-- .../rest/rest_credential_provider_test.cpp | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/paimon/rest/rest_credential_provider.cpp b/src/paimon/rest/rest_credential_provider.cpp index a5861508..fda2de9c 100644 --- a/src/paimon/rest/rest_credential_provider.cpp +++ b/src/paimon/rest/rest_credential_provider.cpp @@ -108,13 +108,17 @@ std::map RestCredentialProvider::MergeOptionsWithCrede } for (auto it = merged.begin(); it != merged.end();) { const std::string& key = it->first; - bool stale_bucket_scoped = StringUtils::StartsWith(key, kOssBucketPrefix) && + // A credential the token itself supplies is authoritative and is never dropped, even when + // it is bucket-scoped or the alias of another key the token carries. + bool token_supplied = credentials.find(key) != credentials.end(); + bool stale_bucket_scoped = !token_supplied && + StringUtils::StartsWith(key, kOssBucketPrefix) && std::any_of(refreshed_suffixes.begin(), refreshed_suffixes.end(), [&](const std::string& suffix) { return StringUtils::EndsWith(key, "." + suffix); }); bool stale_security_token_alias = - refreshed_security_token && credentials.find(key) == credentials.end() && + !token_supplied && refreshed_security_token && (key == std::string(kOssOptionPrefix) + kOssSecurityTokenSuffix || key == std::string(kOssOptionPrefix) + kOssSessionTokenSuffix); if (stale_bucket_scoped || stale_security_token_alias) { diff --git a/src/paimon/rest/rest_credential_provider_test.cpp b/src/paimon/rest/rest_credential_provider_test.cpp index ad753b56..67c98fa1 100644 --- a/src/paimon/rest/rest_credential_provider_test.cpp +++ b/src/paimon/rest/rest_credential_provider_test.cpp @@ -291,6 +291,31 @@ TEST_F(RestCredentialProviderTest, SecurityTokenCredentialClearsStaleSessionToke ASSERT_EQ(0u, merged.count("fs.oss.sessionToken")); } +TEST_F(RestCredentialProviderTest, TokenSuppliedBucketScopedCredentialsArePreserved) { + // When the token itself carries a complete bucket-scoped credential set, those values are + // authoritative: the cleanup that a global credential of the same suffix would otherwise + // trigger must not erase the token's own bucket-scoped keys. + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Credentials base = {}; + Credentials credentials = { + {"fs.oss.accessKeyId", "token-ak"}, + {"fs.oss.accessKeySecret", "token-sk"}, + {"fs.oss.securityToken", "token-sts"}, + {"fs.oss.bucket.b.accessKeyId", "token-bucket-ak"}, + {"fs.oss.bucket.b.accessKeySecret", "token-bucket-sk"}, + {"fs.oss.bucket.b.securityToken", "token-bucket-sts"}, + }; + + Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ("token-sts", merged.at("fs.oss.securityToken")); + ASSERT_EQ("token-bucket-ak", merged.at("fs.oss.bucket.b.accessKeyId")); + ASSERT_EQ("token-bucket-sk", merged.at("fs.oss.bucket.b.accessKeySecret")); + ASSERT_EQ("token-bucket-sts", merged.at("fs.oss.bucket.b.securityToken")); +} + TEST_F(RestCredentialProviderTest, ForbiddenIsReportedToTheCaller) { { std::lock_guard lock(state_->mutex); From 6c161503af4f492792ad302d646276b747a51e4f Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 21 Sep 2026 17:32:48 +0800 Subject: [PATCH 15/16] fix --- docs/source/user_guide/catalog.rst | 8 -- include/paimon/catalog/catalog.h | 8 +- include/paimon/fs/credential_provider.h | 24 +--- src/paimon/core/catalog/catalog.cpp | 13 +- src/paimon/rest/rest_catalog.cpp | 22 +-- src/paimon/rest/rest_catalog.h | 15 ++- src/paimon/rest/rest_catalog_test.cpp | 23 ++++ src/paimon/rest/rest_credential_provider.cpp | 85 ++---------- src/paimon/rest/rest_credential_provider.h | 24 ++-- .../rest/rest_credential_provider_test.cpp | 104 ++------------ src/paimon/rest/rest_token_file_system.cpp | 22 ++- src/paimon/rest/rest_token_file_system.h | 18 ++- .../rest/rest_token_file_system_test.cpp | 127 ++++++++---------- 13 files changed, 187 insertions(+), 306 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index d881968c..7848d7f7 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -404,14 +404,6 @@ the factory is called with are those of the accesses to authenticate, which is where an implementation reads its own configuration, such as the address of the token service, from. -Merge the credentials into your file system's options with -``CredentialProvider::MergeOptionsWithCredentials`` rather than by hand, so they are -shaped the same way the built-in data token file system shapes them. The default -overlays the credentials key by key over the base options, mirroring the Java -client. A provider that knows the file system its credentials are for overrides -``MergeOptionsWithCredentials`` to normalize option aliases or clear the stale -bucket-scoped variants the fresh credentials replace. - All factories share one identifier space, so an identifier a file system factory already takes — ``oss``, ``s3``, ``local``, ``jindo`` — would replace it; name the provider after where its credentials come from instead. diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 18f25277..a94af62c 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -64,10 +64,16 @@ class PAIMON_EXPORT Catalog { /// @param file_system Specifies the file system for file operations. /// If not set, use default file system (configured in /// `Options::FILE_SYSTEM`) + /// @param fs_scheme_to_identifier_map Maps a URI scheme (like "oss") to the registered file + /// system identifier that serves it, so a catalog that resolves several + /// schemes, including the file systems it builds from per-table data + /// tokens, keeps routing each scheme to its backend. Ignored when + /// `file_system` is supplied, which serves every scheme itself. /// @return A result containing a unique pointer to a `Catalog` instance, or an error status. static Result> Create( const std::string& root_path, const std::map& options, - const std::shared_ptr& file_system = nullptr); + const std::shared_ptr& file_system = nullptr, + const std::map& fs_scheme_to_identifier_map = {}); virtual ~Catalog() = default; diff --git a/include/paimon/fs/credential_provider.h b/include/paimon/fs/credential_provider.h index 8e45cc14..27db2f81 100644 --- a/include/paimon/fs/credential_provider.h +++ b/include/paimon/fs/credential_provider.h @@ -46,29 +46,9 @@ class PAIMON_EXPORT CredentialProvider { virtual ~CredentialProvider() = default; /// Returns the credentials to sign an access with, reloading them when they are about - /// to expire. The keys are file system options, so a caller merges them over its own - /// file system options with `MergeOptionsWithCredentials`. + /// to expire. The keys are file system options, e.g. "fs.oss.accessKeyId" or + /// "fs.oss.securityToken", so a caller merges them over its own file system options. virtual Result> GetCredentials() const = 0; - - /// Merges the issued credentials into the file system options a delegate is built from. - /// This is the canonical way to shape credentials into the options of an access: a - /// caller that brings its own file system calls it so the credentials are applied the - /// same way the built-in data token file system applies them. - /// - /// The default overlays the credentials key by key over `base_options`, so they win - /// wherever they overlap, mirroring the Java client and staying scheme-agnostic. A - /// provider that knows the file system its credentials are for overrides this to - /// normalize option aliases or clear stale bucket-scoped variants the credentials - /// replace. - virtual std::map MergeOptionsWithCredentials( - const std::map& base_options, - const std::map& credentials) const { - std::map merged = base_options; - for (const auto& [key, value] : credentials) { - merged[key] = value; - } - return merged; - } }; } // namespace paimon diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 69813041..978c20e9 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -42,9 +42,10 @@ const char Catalog::SYSTEM_TABLE_SPLITTER[] = "$"; const char Catalog::DB_SUFFIX[] = ".db"; const char Catalog::DB_LOCATION_PROP[] = "location"; -Result> Catalog::Create(const std::string& root_path, - const std::map& options, - const std::shared_ptr& file_system) { +Result> Catalog::Create( + const std::string& root_path, const std::map& options, + const std::shared_ptr& file_system, + const std::map& fs_scheme_to_identifier_map) { std::string metastore = "filesystem"; auto metastore_iter = options.find(CatalogOptions::METASTORE); if (metastore_iter != options.end()) { @@ -54,7 +55,8 @@ Result> Catalog::Create(const std::string& root_path, } if (metastore == "rest") { #ifdef PAIMON_ENABLE_REST - return RestCatalog::Create(root_path, options, file_system); + return RestCatalog::Create(root_path, options, file_system, RestHttpClient::Config(), + fs_scheme_to_identifier_map); #else return Status::NotImplemented( "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); @@ -63,7 +65,8 @@ Result> Catalog::Create(const std::string& root_path, if (metastore != "filesystem") { return Status::Invalid("unsupported metastore: ", metastore); } - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, file_system, fs_scheme_to_identifier_map)); return std::make_unique(core_options.GetFileSystem(), root_path, options); } diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index da88cb24..d12a17ac 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -77,7 +77,8 @@ Result ToLoadIdentifier(const Identifier& identifier) { RestCatalog::RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, const std::string& warehouse, bool data_token_enabled, - bool fs_explicitly_supplied) + bool fs_explicitly_supplied, + const std::map& fs_scheme_to_identifier_map) : api_(std::move(api)), fs_(fs), warehouse_(warehouse), @@ -85,6 +86,7 @@ RestCatalog::RestCatalog(std::shared_ptr api, const std::shared_ptrGetMergedOptions(), CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX)), + fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), logger_(Logger::GetLogger("RestCatalog")) { if (data_token_enabled_) { token_fs_cache_ = RestTokenFileSystem::CreateFileSystemCache(); @@ -93,18 +95,20 @@ RestCatalog::RestCatalog(std::shared_ptr api, const std::shared_ptr> RestCatalog::Create( const std::string& warehouse, const std::map& options, - const std::shared_ptr& file_system, const RestHttpClient::Config& http_config) { + const std::shared_ptr& file_system, const RestHttpClient::Config& http_config, + const std::map& fs_scheme_to_identifier_map) { PAIMON_ASSIGN_OR_RAISE( std::unique_ptr api, RestApi::Create(options, warehouse, /*config_required=*/true, http_config)); - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(api->GetMergedOptions(), file_system)); + PAIMON_ASSIGN_OR_RAISE( + CoreOptions core_options, + CoreOptions::FromMap(api->GetMergedOptions(), file_system, fs_scheme_to_identifier_map)); PAIMON_ASSIGN_OR_RAISE(bool data_token_enabled, OptionsUtils::GetValueFromMap( api->GetMergedOptions(), CatalogOptions::DATA_TOKEN_ENABLED, false)); - return std::unique_ptr( - new RestCatalog(std::move(api), core_options.GetFileSystem(), warehouse, data_token_enabled, - /*fs_explicitly_supplied=*/file_system != nullptr)); + return std::unique_ptr(new RestCatalog( + std::move(api), core_options.GetFileSystem(), warehouse, data_token_enabled, + /*fs_explicitly_supplied=*/file_system != nullptr, fs_scheme_to_identifier_map)); } const std::map& RestCatalog::GetOptions() const { @@ -479,9 +483,9 @@ Result> RestCatalog::GetTableFileSystem( // from the catalog options, which every table agrees on, so a rotation rebuilds them. const std::map& catalog_options = api_->GetMergedOptions(); std::shared_ptr provider = - std::make_shared(api_, load_identifier); + std::make_shared(api_, catalog_options, load_identifier); return std::make_shared(std::move(provider), catalog_options, - token_fs_cache_); + token_fs_cache_, fs_scheme_to_identifier_map_); } Result> RestCatalog::ListSnapshots(const Identifier& identifier, diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h index ef3049a7..b967b995 100644 --- a/src/paimon/rest/rest_catalog.h +++ b/src/paimon/rest/rest_catalog.h @@ -47,10 +47,15 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// by `CatalogOptions::URI`, then builds the file system from the merged options. /// /// @param warehouse The warehouse identifier sent to the server; may be empty. + /// @param fs_scheme_to_identifier_map Maps a URI scheme to the registered file system + /// identifier that serves it, used both for the catalog-level file system and for + /// the file systems built from per-table data tokens. Ignored when `file_system` + /// is supplied. static Result> Create( const std::string& warehouse, const std::map& options, const std::shared_ptr& file_system, - const RestHttpClient::Config& http_config = RestHttpClient::Config()); + const RestHttpClient::Config& http_config = RestHttpClient::Config(), + const std::map& fs_scheme_to_identifier_map = {}); Status CreateDatabase(const std::string& name, const std::map& options, @@ -111,7 +116,8 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { private: RestCatalog(std::shared_ptr api, const std::shared_ptr& fs, - const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied); + const std::string& warehouse, bool data_token_enabled, bool fs_explicitly_supplied, + const std::map& fs_scheme_to_identifier_map); /// Loads the schema and catalog table ID from the same response. Result> LoadTableSchema(const Identifier& identifier, @@ -138,6 +144,11 @@ class RestCatalog : public Catalog, public VersionManagedCatalog { /// The "table-default." options of the merged config, applied to `CreateTable` /// options when absent. std::map table_default_options_; + /// Maps a URI scheme to the registered file system identifier that serves it, passed to + /// the file systems built from per-table data tokens so a data token access routes each + /// scheme the same way the catalog-level file system does. Unused when the caller + /// supplied a file system, which serves every scheme itself. + std::map fs_scheme_to_identifier_map_; /// The file systems of the data tokens, keyed by the credentials they were built from /// and shared by the data token file systems this catalog hands out. Only created when /// `data_token_enabled_` is set. diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp index e3abdc1a..6eafadb3 100644 --- a/src/paimon/rest/rest_catalog_test.cpp +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -608,6 +608,29 @@ TEST_F(RestCatalogTest, TableFileSystemWithDataToken) { TokenRequests()); } +TEST_F(RestCatalogTest, DataTokenFileSystemResolvesSchemesThroughTheIdentifierMap) { + options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; + // The merged options resolve the local scheme through "local", but a scheme-to-identifier + // map routes it to a file system that does not exist. The map must reach both the + // catalog-level file system and the delegate a data token builds. + ASSERT_OK_AND_ASSIGN( + std::unique_ptr catalog, + RestCatalog::Create(kWarehouse, options_, /*file_system=*/nullptr, RestHttpClient::Config(), + {{"file", "no-such-file-system"}})); + + // the catalog-level file system routes the local scheme to the missing backend + Status catalog_fs_status = catalog->GetFileSystem()->Exists("/no-such-file").status(); + ASSERT_NOK(catalog_fs_status); + ASSERT_NOK_WITH_MSG(catalog_fs_status, "no-such-file-system"); + + // the delegate the data token builds routes it the same way + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_fs, + catalog->GetTableFileSystem(Identifier("db1", "t1"))); + Status table_fs_status = table_fs->Exists("/no-such-file").status(); + ASSERT_NOK(table_fs_status); + ASSERT_NOK_WITH_MSG(table_fs_status, "no-such-file-system"); +} + TEST_F(RestCatalogTest, TableFileSystemIsBoundToTheTableItWasAskedFor) { options_[CatalogOptions::DATA_TOKEN_ENABLED] = "true"; ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); diff --git a/src/paimon/rest/rest_credential_provider.cpp b/src/paimon/rest/rest_credential_provider.cpp index fda2de9c..8a37c34a 100644 --- a/src/paimon/rest/rest_credential_provider.cpp +++ b/src/paimon/rest/rest_credential_provider.cpp @@ -18,15 +18,12 @@ #include "paimon/rest/rest_credential_provider.h" -#include #include #include #include -#include #include #include "paimon/catalog_options.h" -#include "paimon/common/utils/string_utils.h" namespace paimon { @@ -34,24 +31,6 @@ namespace { /// Declared here rather than taken from the OSS file system, which is an optional build /// component this module must not depend on. constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; -constexpr const char kOssOptionPrefix[] = "fs.oss."; -constexpr const char kOssBucketPrefix[] = "fs.oss.bucket."; -// The two OSS names for the STS token: the backend reads securityToken first and only -// consults sessionToken when it is empty, so a token under either name has to clear the -// catalog value under both. -constexpr const char kOssSecurityTokenSuffix[] = "securityToken"; -constexpr const char kOssSessionTokenSuffix[] = "sessionToken"; - -/// The suffix of a flat OSS option, i.e. what follows "fs.oss." for a key like -/// "fs.oss.accessKeyId"; empty for a bucket-scoped ("fs.oss.bucket..") or a -/// non-OSS key. -std::string FlatOssOptionSuffix(const std::string& key) { - if (!StringUtils::StartsWith(key, kOssOptionPrefix) || - StringUtils::StartsWith(key, kOssBucketPrefix)) { - return ""; - } - return key.substr(std::string(kOssOptionPrefix).size()); -} } // namespace size_t RestToken::Hash::operator()(const RestToken& rest_token) const { @@ -63,9 +42,11 @@ size_t RestToken::Hash::operator()(const RestToken& rest_token) const { return result; } -RestCredentialProvider::RestCredentialProvider(const std::shared_ptr& api, - const Identifier& identifier, Clock clock) +RestCredentialProvider::RestCredentialProvider( + const std::shared_ptr& api, const std::map& catalog_options, + const Identifier& identifier, Clock clock) : api_(api), + catalog_options_(catalog_options), identifier_(identifier), clock_(std::move(clock)), logger_(Logger::GetLogger("RestCredentialProvider")) {} @@ -79,58 +60,13 @@ bool RestCredentialProvider::ShouldRefresh() const { return token_->expires_at_millis - now_millis < RestApi::kTokenExpirationSafeTimeMillis; } -std::map RestCredentialProvider::MergeOptionsWithCredentials( - const std::map& base_options, - const std::map& credentials) const { - std::map merged = base_options; - for (const auto& [key, value] : credentials) { - merged[key] = value; - } - // The OSS backend resolves a bucket-scoped option ("fs.oss.bucket..") ahead of the - // flat one and reads "fs.oss.securityToken" ahead of its "fs.oss.sessionToken" alias, so a - // stale catalog value can shadow a credential the token just refreshed. Drop the stale - // variants of every credential the token supplies -- including both security-token aliases - // when it supplies either -- so the issued credential is the one that wins. - std::set refreshed_suffixes; - bool refreshed_security_token = false; - for (const auto& [key, value] : credentials) { - std::string suffix = FlatOssOptionSuffix(key); - if (suffix.empty()) { - continue; - } - refreshed_suffixes.insert(suffix); - refreshed_security_token = refreshed_security_token || suffix == kOssSecurityTokenSuffix || - suffix == kOssSessionTokenSuffix; - } - if (refreshed_security_token) { - refreshed_suffixes.insert(kOssSecurityTokenSuffix); - refreshed_suffixes.insert(kOssSessionTokenSuffix); - } - for (auto it = merged.begin(); it != merged.end();) { - const std::string& key = it->first; - // A credential the token itself supplies is authoritative and is never dropped, even when - // it is bucket-scoped or the alias of another key the token carries. - bool token_supplied = credentials.find(key) != credentials.end(); - bool stale_bucket_scoped = !token_supplied && - StringUtils::StartsWith(key, kOssBucketPrefix) && - std::any_of(refreshed_suffixes.begin(), refreshed_suffixes.end(), - [&](const std::string& suffix) { - return StringUtils::EndsWith(key, "." + suffix); - }); - bool stale_security_token_alias = - !token_supplied && refreshed_security_token && - (key == std::string(kOssOptionPrefix) + kOssSecurityTokenSuffix || - key == std::string(kOssOptionPrefix) + kOssSessionTokenSuffix); - if (stale_bucket_scoped || stale_security_token_alias) { - it = merged.erase(it); - } else { - ++it; - } - } +std::map RestCredentialProvider::ApplyDlfEndpointOverride( + const std::map& token) const { + std::map merged = token; // The DLF OSS endpoint overrides the standard one, since the credentials are issued // for the DLF endpoint rather than for the endpoint the catalog was configured with. - auto dlf_oss_endpoint = base_options.find(CatalogOptions::DLF_OSS_ENDPOINT); - if (dlf_oss_endpoint != base_options.end() && !dlf_oss_endpoint->second.empty()) { + auto dlf_oss_endpoint = catalog_options_.find(CatalogOptions::DLF_OSS_ENDPOINT); + if (dlf_oss_endpoint != catalog_options_.end() && !dlf_oss_endpoint->second.empty()) { merged[kOssEndpointOption] = dlf_oss_endpoint->second; } return merged; @@ -144,7 +80,8 @@ Status RestCredentialProvider::RefreshToken() const { identifier_.ToString().c_str(), static_cast(response.GetExpiresAtMillis())); - token_ = RestToken{response.GetToken(), response.GetExpiresAtMillis()}; + token_ = + RestToken{ApplyDlfEndpointOverride(response.GetToken()), response.GetExpiresAtMillis()}; return Status::OK(); } diff --git a/src/paimon/rest/rest_credential_provider.h b/src/paimon/rest/rest_credential_provider.h index 3860ee67..f9243fb9 100644 --- a/src/paimon/rest/rest_credential_provider.h +++ b/src/paimon/rest/rest_credential_provider.h @@ -68,9 +68,12 @@ class RestCredentialProvider : public CredentialProvider { /// @param api Client of the catalog that issues the credentials. Shared because a /// provider commonly outlives the catalog it was obtained from. + /// @param catalog_options Options the credentials are merged over. /// @param identifier The table the credentials are requested for. /// @param clock Source of the current time, overridable for tests. - RestCredentialProvider(const std::shared_ptr& api, const Identifier& identifier, + RestCredentialProvider(const std::shared_ptr& api, + const std::map& catalog_options, + const Identifier& identifier, Clock clock = std::chrono::system_clock::now); ~RestCredentialProvider() override = default; @@ -86,15 +89,6 @@ class RestCredentialProvider : public CredentialProvider { /// source. Result> GetCredentials() const override; - /// Merges the issued credentials over `base_options`, then corrects the OSS endpoint: the - /// credentials are issued for the catalog's DLF OSS endpoint, which overrides both the - /// endpoint the catalog was configured with and the one the server reported. The correction - /// lives here rather than in the token so the token stays a minimal cache key that carries - /// only the issued credentials. - std::map MergeOptionsWithCredentials( - const std::map& base_options, - const std::map& credentials) const override; - private: /// Reloads the credentials from the server. Called with the write lock of `mutex_` /// held. @@ -103,7 +97,17 @@ class RestCredentialProvider : public CredentialProvider { /// Whether `token_` is absent or expires within the safe time. bool ShouldRefresh() const; + /// The issued `token` with the catalog's DLF OSS endpoint, when set, overriding the endpoint + /// the server reported: the credentials are issued for the DLF endpoint, not the one the + /// catalog was configured with. The token is otherwise left exactly as issued -- it is a file + /// system cache key and what `ValidToken()` serves, so it carries only the credentials, never + /// the whole catalog options; merging those over the token to build a delegate is the file + /// system's `MergeTokenOptions`. + std::map ApplyDlfEndpointOverride( + const std::map& token) const; + std::shared_ptr api_; + std::map catalog_options_; Identifier identifier_; Clock clock_; std::shared_ptr logger_; diff --git a/src/paimon/rest/rest_credential_provider_test.cpp b/src/paimon/rest/rest_credential_provider_test.cpp index 67c98fa1..35548db3 100644 --- a/src/paimon/rest/rest_credential_provider_test.cpp +++ b/src/paimon/rest/rest_credential_provider_test.cpp @@ -117,7 +117,7 @@ class RestCredentialProviderTest : public ::testing::Test { } std::shared_ptr shared_api(std::move(api).value()); return std::make_shared( - shared_api, Identifier("db1", "t1"), [this] { + shared_api, catalog_options_, Identifier("db1", "t1"), [this] { return std::chrono::system_clock::time_point( std::chrono::milliseconds(now_millis_.load())); }); @@ -194,7 +194,7 @@ TEST_F(RestCredentialProviderTest, ExpiredTokenReloadsOnEveryCall) { ASSERT_EQ(2, state_->request_count.load()); } -TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpointOnMerge) { +TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpoint) { catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = "dlf-endpoint"; { std::lock_guard lock(state_->mutex); @@ -203,20 +203,14 @@ TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpointOnMerge) std::shared_ptr provider = CreateProvider(); ASSERT_NE(nullptr, provider); - // the token is the cache key and stays exactly as issued, carrying no catalog secrets and - // not the endpoint correction, which is a merge concern ASSERT_OK_AND_ASSIGN(RestToken token, provider->ValidToken()); ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); - ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); + // the endpoint the credentials were issued for wins over the one the server reported + ASSERT_EQ("dlf-endpoint", token.token.at(kOssEndpointOption)); ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); + // the catalog options are not part of the token, so its secrets stay private ASSERT_EQ(0u, token.token.count(CatalogOptions::TOKEN)); ASSERT_EQ(2u, token.token.size()); - - // merging shapes the credentials into the file system options: the endpoint the credentials - // were issued for wins over the one the server reported - Credentials merged = provider->MergeOptionsWithCredentials(catalog_options_, token.token); - ASSERT_EQ("ak-1", merged.at("fs.oss.accessKeyId")); - ASSERT_EQ("dlf-endpoint", merged.at(kOssEndpointOption)); } TEST_F(RestCredentialProviderTest, EmptyDlfOssEndpointIsNotApplied) { @@ -228,92 +222,10 @@ TEST_F(RestCredentialProviderTest, EmptyDlfOssEndpointIsNotApplied) { std::shared_ptr provider = CreateProvider(); ASSERT_NE(nullptr, provider); - // an unset dlf endpoint leaves the endpoint the credentials carry alone through the merge + // an unset dlf endpoint leaves the endpoint the server reported alone ASSERT_OK_AND_ASSIGN(RestToken token, provider->ValidToken()); - Credentials merged = provider->MergeOptionsWithCredentials(catalog_options_, token.token); - ASSERT_EQ("server-endpoint", merged.at(kOssEndpointOption)); -} - -TEST_F(RestCredentialProviderTest, IssuedCredentialsClearStaleBucketScopedCatalogVariants) { - // The OSS backend resolves a bucket-scoped option ahead of the flat one, so a stale - // bucket-scoped catalog value would shadow the flat credential the token just refreshed. - std::shared_ptr provider = CreateProvider(); - ASSERT_NE(nullptr, provider); - - Credentials base = { - {"fs.oss.accessKeyId", "catalog-ak"}, - {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, - {"unrelated", "kept"}, - }; - Credentials credentials = {{"fs.oss.accessKeyId", "token-ak"}}; - - Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); - ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); - ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeyId")); - ASSERT_EQ("kept", merged.at("unrelated")); -} - -TEST_F(RestCredentialProviderTest, SessionTokenCredentialClearsStaleSecurityTokenAliases) { - // A token carrying a fresh session token must clear a stale catalog security token -- the - // OSS backend reads securityToken first, so leaving it in place would pair the refreshed - // key pair with the old STS token and fail authentication. Both the flat and the - // bucket-scoped stale securityToken have to go. - std::shared_ptr provider = CreateProvider(); - ASSERT_NE(nullptr, provider); - - Credentials base = { - {"fs.oss.securityToken", "stale-sts"}, - {"fs.oss.bucket.b.securityToken", "stale-bucket-sts"}, - }; - Credentials credentials = {{"fs.oss.accessKeyId", "token-ak"}, - {"fs.oss.accessKeySecret", "token-sk"}, - {"fs.oss.sessionToken", "fresh-sts"}}; - - Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); - ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); - ASSERT_EQ("token-sk", merged.at("fs.oss.accessKeySecret")); - ASSERT_EQ("fresh-sts", merged.at("fs.oss.sessionToken")); - ASSERT_EQ(0u, merged.count("fs.oss.securityToken")); - ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.securityToken")); -} - -TEST_F(RestCredentialProviderTest, SecurityTokenCredentialClearsStaleSessionTokenAlias) { - // The alias goes the other way too: a fresh securityToken clears a stale sessionToken so the - // backend cannot fall back to it. - std::shared_ptr provider = CreateProvider(); - ASSERT_NE(nullptr, provider); - - Credentials base = {{"fs.oss.sessionToken", "stale-sts"}}; - Credentials credentials = {{"fs.oss.securityToken", "fresh-sts"}}; - - Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); - ASSERT_EQ("fresh-sts", merged.at("fs.oss.securityToken")); - ASSERT_EQ(0u, merged.count("fs.oss.sessionToken")); -} - -TEST_F(RestCredentialProviderTest, TokenSuppliedBucketScopedCredentialsArePreserved) { - // When the token itself carries a complete bucket-scoped credential set, those values are - // authoritative: the cleanup that a global credential of the same suffix would otherwise - // trigger must not erase the token's own bucket-scoped keys. - std::shared_ptr provider = CreateProvider(); - ASSERT_NE(nullptr, provider); - - Credentials base = {}; - Credentials credentials = { - {"fs.oss.accessKeyId", "token-ak"}, - {"fs.oss.accessKeySecret", "token-sk"}, - {"fs.oss.securityToken", "token-sts"}, - {"fs.oss.bucket.b.accessKeyId", "token-bucket-ak"}, - {"fs.oss.bucket.b.accessKeySecret", "token-bucket-sk"}, - {"fs.oss.bucket.b.securityToken", "token-bucket-sts"}, - }; - - Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); - ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); - ASSERT_EQ("token-sts", merged.at("fs.oss.securityToken")); - ASSERT_EQ("token-bucket-ak", merged.at("fs.oss.bucket.b.accessKeyId")); - ASSERT_EQ("token-bucket-sk", merged.at("fs.oss.bucket.b.accessKeySecret")); - ASSERT_EQ("token-bucket-sts", merged.at("fs.oss.bucket.b.securityToken")); + ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); + ASSERT_EQ(1u, token.token.size()); } TEST_F(RestCredentialProviderTest, ForbiddenIsReportedToTheCaller) { diff --git a/src/paimon/rest/rest_token_file_system.cpp b/src/paimon/rest/rest_token_file_system.cpp index e5b673de..a3402d00 100644 --- a/src/paimon/rest/rest_token_file_system.cpp +++ b/src/paimon/rest/rest_token_file_system.cpp @@ -32,19 +32,31 @@ std::shared_ptr RestTokenFileSystem::CreateFileSystemC return std::make_shared(std::move(cache_options)); } -RestTokenFileSystem::RestTokenFileSystem(std::shared_ptr provider, - const std::map& catalog_options, - std::shared_ptr fs_cache) +RestTokenFileSystem::RestTokenFileSystem( + std::shared_ptr provider, + const std::map& catalog_options, + std::shared_ptr fs_cache, + const std::map& fs_scheme_to_identifier_map) : catalog_options_(catalog_options), + fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), fs_cache_(std::move(fs_cache)), provider_(std::move(provider)) {} +std::map RestTokenFileSystem::MergeTokenOptions( + const std::map& catalog_options, const RestToken& token) { + std::map fs_options = catalog_options; + for (const auto& [key, value] : token.token) { + fs_options[key] = value; + } + return fs_options; +} + Result> RestTokenFileSystem::BuildFileSystem( const RestToken& token) const { PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap(provider_->MergeOptionsWithCredentials(catalog_options_, token.token), - /*specified_file_system=*/nullptr)); + CoreOptions::FromMap(MergeTokenOptions(catalog_options_, token), + /*specified_file_system=*/nullptr, fs_scheme_to_identifier_map_)); std::shared_ptr fs = core_options.GetFileSystem(); if (fs == nullptr) { return Status::Invalid("failed to build the file system from the data token credentials"); diff --git a/src/paimon/rest/rest_token_file_system.h b/src/paimon/rest/rest_token_file_system.h index 45f95341..81f0f7f6 100644 --- a/src/paimon/rest/rest_token_file_system.h +++ b/src/paimon/rest/rest_token_file_system.h @@ -62,9 +62,13 @@ class RestTokenFileSystem : public FileSystem { /// @param fs_cache Cache of the delegates, shared with the file systems of the other /// tables of the same catalog so that tables issued equal credentials /// reuse one delegate. The catalog owns it and hands it out. + /// @param fs_scheme_to_identifier_map Maps a URI scheme to the registered file system + /// identifier that serves it, so the delegate built from the credentials + /// routes each scheme the same way the catalog-level file system does. RestTokenFileSystem(std::shared_ptr provider, const std::map& catalog_options, - std::shared_ptr fs_cache); + std::shared_ptr fs_cache, + const std::map& fs_scheme_to_identifier_map = {}); ~RestTokenFileSystem() override = default; @@ -89,6 +93,14 @@ class RestTokenFileSystem : public FileSystem { /// credentials are requested with. Result ValidToken() const; + /// Merges the issued credentials over the catalog options the delegate is built from. The + /// credentials are themselves file system options, so they are overlaid key by key and win + /// wherever they overlap, mirroring the Java client; how a concrete file system resolves the + /// catalog options the token does not carry is that file system's own concern, so this merge + /// stays scheme-agnostic. Exposed for tests. + static std::map MergeTokenOptions( + const std::map& catalog_options, const RestToken& token); + private: /// Returns the file system of the current credentials, reloading them when they /// expire in less than `RestApi::kTokenExpirationSafeTimeMillis`. @@ -99,6 +111,10 @@ class RestTokenFileSystem : public FileSystem { Result> BuildFileSystem(const RestToken& token) const; std::map catalog_options_; + /// Maps a URI scheme to the registered file system identifier that serves it, applied when + /// the delegate is built from the merged options so a data token access routes each scheme + /// the same way the catalog-level file system does. + std::map fs_scheme_to_identifier_map_; std::shared_ptr fs_cache_; /// The credentials this file system delegates with, reloaded before they expire. diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index 925622be..a0ba5238 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -37,7 +37,6 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/defs.h" -#include "paimon/fs/credential_provider.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" @@ -152,7 +151,8 @@ class RestTokenFileSystemTest : public ::testing::Test { } std::shared_ptr CreateFileSystem( - std::shared_ptr fs_cache = nullptr) { + std::shared_ptr fs_cache = nullptr, + const std::map& fs_scheme_to_identifier_map = {}) { Result> api = RestApi::Create(catalog_options_, "", /*config_required=*/false); if (!api.ok()) { @@ -162,13 +162,14 @@ class RestTokenFileSystemTest : public ::testing::Test { if (fs_cache == nullptr) { fs_cache = RestTokenFileSystem::CreateFileSystemCache(); } - std::shared_ptr provider = - std::make_shared(shared_api, Identifier("db1", "t1"), [this] { + std::shared_ptr provider = std::make_shared( + shared_api, catalog_options_, Identifier("db1", "t1"), [this] { return std::chrono::system_clock::time_point( std::chrono::milliseconds(now_millis_.load())); }); return std::make_shared(std::move(provider), catalog_options_, - std::move(fs_cache)); + std::move(fs_cache), + fs_scheme_to_identifier_map); } // Writes `content` to a file of the temp directory with the local file system and @@ -246,9 +247,8 @@ TEST_F(RestTokenFileSystemTest, ValidTokenCarriesOnlyTheServerCredentials) { ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); - // the token is the cache key and carries only the issued credentials, never the catalog - // secrets nor the DLF endpoint correction, which the provider applies only when merging - ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); + // the endpoint the credentials were issued for wins over the one the server reported + ASSERT_EQ("dlf-endpoint", token.token.at(kOssEndpointOption)); ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); // the catalog options are not part of the token, so its secrets stay private ASSERT_EQ(0u, token.token.count(CatalogOptions::TOKEN)); @@ -348,7 +348,7 @@ TEST_F(RestTokenFileSystemTest, DefaultClockIsTheSystemClock) { RestApi::Create(catalog_options_, "", /*config_required=*/false)); // the provider is built with its default clock, the system clock std::shared_ptr provider = std::make_shared( - std::shared_ptr(std::move(api)), Identifier("db1", "t1")); + std::shared_ptr(std::move(api)), catalog_options_, Identifier("db1", "t1")); RestTokenFileSystem fs(provider, catalog_options_, RestTokenFileSystem::CreateFileSystemCache()); ASSERT_OK(fs.ValidToken().status()); @@ -356,6 +356,21 @@ TEST_F(RestTokenFileSystemTest, DefaultClockIsTheSystemClock) { ASSERT_EQ(1, state_->request_count.load()); } +TEST_F(RestTokenFileSystemTest, EmptyDlfOssEndpointIsNotApplied) { + catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = ""; + { + std::lock_guard lock(state_->mutex); + state_->token = {{kOssEndpointOption, "server-endpoint"}}; + } + std::shared_ptr fs = CreateFileSystem(); + ASSERT_NE(nullptr, fs); + + // an unset dlf endpoint leaves the endpoint the server reported alone + ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); + ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); + ASSERT_EQ(1u, token.token.size()); +} + TEST_F(RestTokenFileSystemTest, TokenOverridesTheCatalogFileSystemOptions) { // the credentials must reach the options the delegate is built from: the catalog is // configured with the local file system, yet an option of the token replaces it @@ -373,87 +388,53 @@ TEST_F(RestTokenFileSystemTest, TokenOverridesTheCatalogFileSystemOptions) { ASSERT_EQ(1, state_->request_count.load()); } -TEST(CredentialProviderMergeOptionsWithCredentials, DefaultOverlaysCredentialsOverTheBaseOptions) { - // The default merge treats the credentials as file system options overlaid over the base - // options: they win wherever the two overlap, and every base option the credentials do not - // carry is kept as-is, mirroring the Java client. How the file system resolves the kept - // options -- a per-bucket variant or an alias -- is its own concern, so the merge leaves them - // untouched. - class DefaultProvider : public CredentialProvider { - public: - Result> GetCredentials() const override { - return std::map{}; - } - }; - DefaultProvider provider; - std::map base_options = { +TEST_F(RestTokenFileSystemTest, DelegateResolvesSchemesThroughTheIdentifierMap) { + // The catalog resolves the local scheme through "local", but a scheme-to-identifier map + // routes it to a file system that does not exist. The delegate the token builds must honor + // the map, so a file operation fails with the routed identifier, proving the map reached + // CoreOptions::FromMap rather than being dropped. + std::string path = WriteFile("data", "paimon"); + std::shared_ptr fs = + CreateFileSystem(/*fs_cache=*/nullptr, {{"file", "no-such-file-system"}}); + ASSERT_NE(nullptr, fs); + + Status status = fs->Exists(path).status(); + ASSERT_NOK(status); + ASSERT_NOK_WITH_MSG(status, "no-such-file-system"); + ASSERT_EQ(1, state_->request_count.load()); +} + +TEST(RestTokenFileSystemMergeTokenOptions, IssuedCredentialsOverrideTheCatalogOptions) { + // The issued credentials are file system options merged over the catalog options: they win + // wherever the two overlap, and every catalog option the token does not carry is kept as-is, + // mirroring the Java client. How the file system resolves the kept options -- a per-bucket + // variant or an alias -- is its own concern, so the merge leaves them untouched. + std::map catalog_options = { {"fs.oss.accessKeyId", "catalog-ak"}, {"fs.oss.endpoint", "catalog-endpoint"}, {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, {"unrelated", "kept"}, }; - std::map credentials = {{"fs.oss.accessKeyId", "token-ak"}, - {"fs.oss.accessKeySecret", "token-sk"}, - {"fs.oss.securityToken", "token-sts"}}; + RestToken token; + token.token = {{"fs.oss.accessKeyId", "token-ak"}, + {"fs.oss.accessKeySecret", "token-sk"}, + {"fs.oss.securityToken", "token-sts"}}; std::map merged = - provider.MergeOptionsWithCredentials(base_options, credentials); + RestTokenFileSystem::MergeTokenOptions(catalog_options, token); - // The credential value replaces the base one for the key they share, and the credential-only - // keys are added. + // The token value replaces the catalog's for the key they share, and the token-only keys are + // added. ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); ASSERT_EQ("token-sk", merged.at("fs.oss.accessKeySecret")); ASSERT_EQ("token-sts", merged.at("fs.oss.securityToken")); - // Base options the credentials do not set are kept untouched, whatever their form. + // Catalog options the token does not set are kept untouched, whatever their form. ASSERT_EQ("catalog-endpoint", merged.at("fs.oss.endpoint")); ASSERT_EQ("catalog-bucket-ak", merged.at("fs.oss.bucket.b.accessKeyId")); ASSERT_EQ("kept", merged.at("unrelated")); } -TEST(CredentialProviderMergeOptionsWithCredentials, OverrideCanReshapeTheMergedOptions) { - // A provider that knows the file system its credentials are for overrides - // MergeOptionsWithCredentials to clear the bucket-scoped variants the fresh credentials - // replace, which the default overlay would keep. This is the extension point an OSS-aware - // plugin uses. - class BucketClearingProvider : public CredentialProvider { - public: - Result> GetCredentials() const override { - return std::map{}; - } - std::map MergeOptionsWithCredentials( - const std::map& base_options, - const std::map& credentials) const override { - std::map merged; - for (const auto& [key, value] : base_options) { - if (key.rfind("fs.oss.bucket.", 0) == 0) { - continue; - } - merged[key] = value; - } - for (const auto& [key, value] : credentials) { - merged[key] = value; - } - return merged; - } - }; - BucketClearingProvider provider; - std::map base_options = { - {"fs.oss.accessKeyId", "catalog-ak"}, - {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, - {"unrelated", "kept"}, - }; - std::map credentials = {{"fs.oss.accessKeyId", "token-ak"}}; - - std::map merged = - provider.MergeOptionsWithCredentials(base_options, credentials); - - ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); - // the bucket-scoped variant the credentials replace is cleared by the override - ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeyId")); - ASSERT_EQ("kept", merged.at("unrelated")); -} - TEST_F(RestTokenFileSystemTest, ConcurrentFirstAccessLoadsTheTokenOnce) { std::string path = WriteFile("data", "paimon"); std::shared_ptr fs = CreateFileSystem(); From f348bf3ff2fde0e4b0bec5f6bf3c9a36c19247b2 Mon Sep 17 00:00:00 2001 From: "yonghao.fyh" Date: Mon, 21 Sep 2026 18:20:33 +0800 Subject: [PATCH 16/16] fix --- docs/source/user_guide/catalog.rst | 8 ++ include/paimon/fs/credential_provider.h | 24 +++- src/paimon/rest/rest_catalog.cpp | 2 +- src/paimon/rest/rest_credential_provider.cpp | 85 ++++++++++++-- src/paimon/rest/rest_credential_provider.h | 24 ++-- .../rest/rest_credential_provider_test.cpp | 104 +++++++++++++++-- src/paimon/rest/rest_token_file_system.cpp | 11 +- src/paimon/rest/rest_token_file_system.h | 8 -- .../rest/rest_token_file_system_test.cpp | 105 ++++++++++++------ 9 files changed, 283 insertions(+), 88 deletions(-) diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 7848d7f7..d881968c 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -404,6 +404,14 @@ the factory is called with are those of the accesses to authenticate, which is where an implementation reads its own configuration, such as the address of the token service, from. +Merge the credentials into your file system's options with +``CredentialProvider::MergeOptionsWithCredentials`` rather than by hand, so they are +shaped the same way the built-in data token file system shapes them. The default +overlays the credentials key by key over the base options, mirroring the Java +client. A provider that knows the file system its credentials are for overrides +``MergeOptionsWithCredentials`` to normalize option aliases or clear the stale +bucket-scoped variants the fresh credentials replace. + All factories share one identifier space, so an identifier a file system factory already takes — ``oss``, ``s3``, ``local``, ``jindo`` — would replace it; name the provider after where its credentials come from instead. diff --git a/include/paimon/fs/credential_provider.h b/include/paimon/fs/credential_provider.h index 27db2f81..8e45cc14 100644 --- a/include/paimon/fs/credential_provider.h +++ b/include/paimon/fs/credential_provider.h @@ -46,9 +46,29 @@ class PAIMON_EXPORT CredentialProvider { virtual ~CredentialProvider() = default; /// Returns the credentials to sign an access with, reloading them when they are about - /// to expire. The keys are file system options, e.g. "fs.oss.accessKeyId" or - /// "fs.oss.securityToken", so a caller merges them over its own file system options. + /// to expire. The keys are file system options, so a caller merges them over its own + /// file system options with `MergeOptionsWithCredentials`. virtual Result> GetCredentials() const = 0; + + /// Merges the issued credentials into the file system options a delegate is built from. + /// This is the canonical way to shape credentials into the options of an access: a + /// caller that brings its own file system calls it so the credentials are applied the + /// same way the built-in data token file system applies them. + /// + /// The default overlays the credentials key by key over `base_options`, so they win + /// wherever they overlap, mirroring the Java client and staying scheme-agnostic. A + /// provider that knows the file system its credentials are for overrides this to + /// normalize option aliases or clear stale bucket-scoped variants the credentials + /// replace. + virtual std::map MergeOptionsWithCredentials( + const std::map& base_options, + const std::map& credentials) const { + std::map merged = base_options; + for (const auto& [key, value] : credentials) { + merged[key] = value; + } + return merged; + } }; } // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp index d12a17ac..468fbaca 100644 --- a/src/paimon/rest/rest_catalog.cpp +++ b/src/paimon/rest/rest_catalog.cpp @@ -483,7 +483,7 @@ Result> RestCatalog::GetTableFileSystem( // from the catalog options, which every table agrees on, so a rotation rebuilds them. const std::map& catalog_options = api_->GetMergedOptions(); std::shared_ptr provider = - std::make_shared(api_, catalog_options, load_identifier); + std::make_shared(api_, load_identifier); return std::make_shared(std::move(provider), catalog_options, token_fs_cache_, fs_scheme_to_identifier_map_); } diff --git a/src/paimon/rest/rest_credential_provider.cpp b/src/paimon/rest/rest_credential_provider.cpp index 8a37c34a..fda2de9c 100644 --- a/src/paimon/rest/rest_credential_provider.cpp +++ b/src/paimon/rest/rest_credential_provider.cpp @@ -18,12 +18,15 @@ #include "paimon/rest/rest_credential_provider.h" +#include #include #include #include +#include #include #include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" namespace paimon { @@ -31,6 +34,24 @@ namespace { /// Declared here rather than taken from the OSS file system, which is an optional build /// component this module must not depend on. constexpr const char kOssEndpointOption[] = "fs.oss.endpoint"; +constexpr const char kOssOptionPrefix[] = "fs.oss."; +constexpr const char kOssBucketPrefix[] = "fs.oss.bucket."; +// The two OSS names for the STS token: the backend reads securityToken first and only +// consults sessionToken when it is empty, so a token under either name has to clear the +// catalog value under both. +constexpr const char kOssSecurityTokenSuffix[] = "securityToken"; +constexpr const char kOssSessionTokenSuffix[] = "sessionToken"; + +/// The suffix of a flat OSS option, i.e. what follows "fs.oss." for a key like +/// "fs.oss.accessKeyId"; empty for a bucket-scoped ("fs.oss.bucket..") or a +/// non-OSS key. +std::string FlatOssOptionSuffix(const std::string& key) { + if (!StringUtils::StartsWith(key, kOssOptionPrefix) || + StringUtils::StartsWith(key, kOssBucketPrefix)) { + return ""; + } + return key.substr(std::string(kOssOptionPrefix).size()); +} } // namespace size_t RestToken::Hash::operator()(const RestToken& rest_token) const { @@ -42,11 +63,9 @@ size_t RestToken::Hash::operator()(const RestToken& rest_token) const { return result; } -RestCredentialProvider::RestCredentialProvider( - const std::shared_ptr& api, const std::map& catalog_options, - const Identifier& identifier, Clock clock) +RestCredentialProvider::RestCredentialProvider(const std::shared_ptr& api, + const Identifier& identifier, Clock clock) : api_(api), - catalog_options_(catalog_options), identifier_(identifier), clock_(std::move(clock)), logger_(Logger::GetLogger("RestCredentialProvider")) {} @@ -60,13 +79,58 @@ bool RestCredentialProvider::ShouldRefresh() const { return token_->expires_at_millis - now_millis < RestApi::kTokenExpirationSafeTimeMillis; } -std::map RestCredentialProvider::ApplyDlfEndpointOverride( - const std::map& token) const { - std::map merged = token; +std::map RestCredentialProvider::MergeOptionsWithCredentials( + const std::map& base_options, + const std::map& credentials) const { + std::map merged = base_options; + for (const auto& [key, value] : credentials) { + merged[key] = value; + } + // The OSS backend resolves a bucket-scoped option ("fs.oss.bucket..") ahead of the + // flat one and reads "fs.oss.securityToken" ahead of its "fs.oss.sessionToken" alias, so a + // stale catalog value can shadow a credential the token just refreshed. Drop the stale + // variants of every credential the token supplies -- including both security-token aliases + // when it supplies either -- so the issued credential is the one that wins. + std::set refreshed_suffixes; + bool refreshed_security_token = false; + for (const auto& [key, value] : credentials) { + std::string suffix = FlatOssOptionSuffix(key); + if (suffix.empty()) { + continue; + } + refreshed_suffixes.insert(suffix); + refreshed_security_token = refreshed_security_token || suffix == kOssSecurityTokenSuffix || + suffix == kOssSessionTokenSuffix; + } + if (refreshed_security_token) { + refreshed_suffixes.insert(kOssSecurityTokenSuffix); + refreshed_suffixes.insert(kOssSessionTokenSuffix); + } + for (auto it = merged.begin(); it != merged.end();) { + const std::string& key = it->first; + // A credential the token itself supplies is authoritative and is never dropped, even when + // it is bucket-scoped or the alias of another key the token carries. + bool token_supplied = credentials.find(key) != credentials.end(); + bool stale_bucket_scoped = !token_supplied && + StringUtils::StartsWith(key, kOssBucketPrefix) && + std::any_of(refreshed_suffixes.begin(), refreshed_suffixes.end(), + [&](const std::string& suffix) { + return StringUtils::EndsWith(key, "." + suffix); + }); + bool stale_security_token_alias = + !token_supplied && refreshed_security_token && + (key == std::string(kOssOptionPrefix) + kOssSecurityTokenSuffix || + key == std::string(kOssOptionPrefix) + kOssSessionTokenSuffix); + if (stale_bucket_scoped || stale_security_token_alias) { + it = merged.erase(it); + } else { + ++it; + } + } // The DLF OSS endpoint overrides the standard one, since the credentials are issued // for the DLF endpoint rather than for the endpoint the catalog was configured with. - auto dlf_oss_endpoint = catalog_options_.find(CatalogOptions::DLF_OSS_ENDPOINT); - if (dlf_oss_endpoint != catalog_options_.end() && !dlf_oss_endpoint->second.empty()) { + auto dlf_oss_endpoint = base_options.find(CatalogOptions::DLF_OSS_ENDPOINT); + if (dlf_oss_endpoint != base_options.end() && !dlf_oss_endpoint->second.empty()) { merged[kOssEndpointOption] = dlf_oss_endpoint->second; } return merged; @@ -80,8 +144,7 @@ Status RestCredentialProvider::RefreshToken() const { identifier_.ToString().c_str(), static_cast(response.GetExpiresAtMillis())); - token_ = - RestToken{ApplyDlfEndpointOverride(response.GetToken()), response.GetExpiresAtMillis()}; + token_ = RestToken{response.GetToken(), response.GetExpiresAtMillis()}; return Status::OK(); } diff --git a/src/paimon/rest/rest_credential_provider.h b/src/paimon/rest/rest_credential_provider.h index f9243fb9..3860ee67 100644 --- a/src/paimon/rest/rest_credential_provider.h +++ b/src/paimon/rest/rest_credential_provider.h @@ -68,12 +68,9 @@ class RestCredentialProvider : public CredentialProvider { /// @param api Client of the catalog that issues the credentials. Shared because a /// provider commonly outlives the catalog it was obtained from. - /// @param catalog_options Options the credentials are merged over. /// @param identifier The table the credentials are requested for. /// @param clock Source of the current time, overridable for tests. - RestCredentialProvider(const std::shared_ptr& api, - const std::map& catalog_options, - const Identifier& identifier, + RestCredentialProvider(const std::shared_ptr& api, const Identifier& identifier, Clock clock = std::chrono::system_clock::now); ~RestCredentialProvider() override = default; @@ -89,6 +86,15 @@ class RestCredentialProvider : public CredentialProvider { /// source. Result> GetCredentials() const override; + /// Merges the issued credentials over `base_options`, then corrects the OSS endpoint: the + /// credentials are issued for the catalog's DLF OSS endpoint, which overrides both the + /// endpoint the catalog was configured with and the one the server reported. The correction + /// lives here rather than in the token so the token stays a minimal cache key that carries + /// only the issued credentials. + std::map MergeOptionsWithCredentials( + const std::map& base_options, + const std::map& credentials) const override; + private: /// Reloads the credentials from the server. Called with the write lock of `mutex_` /// held. @@ -97,17 +103,7 @@ class RestCredentialProvider : public CredentialProvider { /// Whether `token_` is absent or expires within the safe time. bool ShouldRefresh() const; - /// The issued `token` with the catalog's DLF OSS endpoint, when set, overriding the endpoint - /// the server reported: the credentials are issued for the DLF endpoint, not the one the - /// catalog was configured with. The token is otherwise left exactly as issued -- it is a file - /// system cache key and what `ValidToken()` serves, so it carries only the credentials, never - /// the whole catalog options; merging those over the token to build a delegate is the file - /// system's `MergeTokenOptions`. - std::map ApplyDlfEndpointOverride( - const std::map& token) const; - std::shared_ptr api_; - std::map catalog_options_; Identifier identifier_; Clock clock_; std::shared_ptr logger_; diff --git a/src/paimon/rest/rest_credential_provider_test.cpp b/src/paimon/rest/rest_credential_provider_test.cpp index 35548db3..67c98fa1 100644 --- a/src/paimon/rest/rest_credential_provider_test.cpp +++ b/src/paimon/rest/rest_credential_provider_test.cpp @@ -117,7 +117,7 @@ class RestCredentialProviderTest : public ::testing::Test { } std::shared_ptr shared_api(std::move(api).value()); return std::make_shared( - shared_api, catalog_options_, Identifier("db1", "t1"), [this] { + shared_api, Identifier("db1", "t1"), [this] { return std::chrono::system_clock::time_point( std::chrono::milliseconds(now_millis_.load())); }); @@ -194,7 +194,7 @@ TEST_F(RestCredentialProviderTest, ExpiredTokenReloadsOnEveryCall) { ASSERT_EQ(2, state_->request_count.load()); } -TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpoint) { +TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpointOnMerge) { catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = "dlf-endpoint"; { std::lock_guard lock(state_->mutex); @@ -203,14 +203,20 @@ TEST_F(RestCredentialProviderTest, DlfEndpointOverridesTheServerEndpoint) { std::shared_ptr provider = CreateProvider(); ASSERT_NE(nullptr, provider); + // the token is the cache key and stays exactly as issued, carrying no catalog secrets and + // not the endpoint correction, which is a merge concern ASSERT_OK_AND_ASSIGN(RestToken token, provider->ValidToken()); ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); - // the endpoint the credentials were issued for wins over the one the server reported - ASSERT_EQ("dlf-endpoint", token.token.at(kOssEndpointOption)); + ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); - // the catalog options are not part of the token, so its secrets stay private ASSERT_EQ(0u, token.token.count(CatalogOptions::TOKEN)); ASSERT_EQ(2u, token.token.size()); + + // merging shapes the credentials into the file system options: the endpoint the credentials + // were issued for wins over the one the server reported + Credentials merged = provider->MergeOptionsWithCredentials(catalog_options_, token.token); + ASSERT_EQ("ak-1", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ("dlf-endpoint", merged.at(kOssEndpointOption)); } TEST_F(RestCredentialProviderTest, EmptyDlfOssEndpointIsNotApplied) { @@ -222,10 +228,92 @@ TEST_F(RestCredentialProviderTest, EmptyDlfOssEndpointIsNotApplied) { std::shared_ptr provider = CreateProvider(); ASSERT_NE(nullptr, provider); - // an unset dlf endpoint leaves the endpoint the server reported alone + // an unset dlf endpoint leaves the endpoint the credentials carry alone through the merge ASSERT_OK_AND_ASSIGN(RestToken token, provider->ValidToken()); - ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); - ASSERT_EQ(1u, token.token.size()); + Credentials merged = provider->MergeOptionsWithCredentials(catalog_options_, token.token); + ASSERT_EQ("server-endpoint", merged.at(kOssEndpointOption)); +} + +TEST_F(RestCredentialProviderTest, IssuedCredentialsClearStaleBucketScopedCatalogVariants) { + // The OSS backend resolves a bucket-scoped option ahead of the flat one, so a stale + // bucket-scoped catalog value would shadow the flat credential the token just refreshed. + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Credentials base = { + {"fs.oss.accessKeyId", "catalog-ak"}, + {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, + {"unrelated", "kept"}, + }; + Credentials credentials = {{"fs.oss.accessKeyId", "token-ak"}}; + + Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeyId")); + ASSERT_EQ("kept", merged.at("unrelated")); +} + +TEST_F(RestCredentialProviderTest, SessionTokenCredentialClearsStaleSecurityTokenAliases) { + // A token carrying a fresh session token must clear a stale catalog security token -- the + // OSS backend reads securityToken first, so leaving it in place would pair the refreshed + // key pair with the old STS token and fail authentication. Both the flat and the + // bucket-scoped stale securityToken have to go. + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Credentials base = { + {"fs.oss.securityToken", "stale-sts"}, + {"fs.oss.bucket.b.securityToken", "stale-bucket-sts"}, + }; + Credentials credentials = {{"fs.oss.accessKeyId", "token-ak"}, + {"fs.oss.accessKeySecret", "token-sk"}, + {"fs.oss.sessionToken", "fresh-sts"}}; + + Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ("token-sk", merged.at("fs.oss.accessKeySecret")); + ASSERT_EQ("fresh-sts", merged.at("fs.oss.sessionToken")); + ASSERT_EQ(0u, merged.count("fs.oss.securityToken")); + ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.securityToken")); +} + +TEST_F(RestCredentialProviderTest, SecurityTokenCredentialClearsStaleSessionTokenAlias) { + // The alias goes the other way too: a fresh securityToken clears a stale sessionToken so the + // backend cannot fall back to it. + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Credentials base = {{"fs.oss.sessionToken", "stale-sts"}}; + Credentials credentials = {{"fs.oss.securityToken", "fresh-sts"}}; + + Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); + ASSERT_EQ("fresh-sts", merged.at("fs.oss.securityToken")); + ASSERT_EQ(0u, merged.count("fs.oss.sessionToken")); +} + +TEST_F(RestCredentialProviderTest, TokenSuppliedBucketScopedCredentialsArePreserved) { + // When the token itself carries a complete bucket-scoped credential set, those values are + // authoritative: the cleanup that a global credential of the same suffix would otherwise + // trigger must not erase the token's own bucket-scoped keys. + std::shared_ptr provider = CreateProvider(); + ASSERT_NE(nullptr, provider); + + Credentials base = {}; + Credentials credentials = { + {"fs.oss.accessKeyId", "token-ak"}, + {"fs.oss.accessKeySecret", "token-sk"}, + {"fs.oss.securityToken", "token-sts"}, + {"fs.oss.bucket.b.accessKeyId", "token-bucket-ak"}, + {"fs.oss.bucket.b.accessKeySecret", "token-bucket-sk"}, + {"fs.oss.bucket.b.securityToken", "token-bucket-sts"}, + }; + + Credentials merged = provider->MergeOptionsWithCredentials(base, credentials); + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + ASSERT_EQ("token-sts", merged.at("fs.oss.securityToken")); + ASSERT_EQ("token-bucket-ak", merged.at("fs.oss.bucket.b.accessKeyId")); + ASSERT_EQ("token-bucket-sk", merged.at("fs.oss.bucket.b.accessKeySecret")); + ASSERT_EQ("token-bucket-sts", merged.at("fs.oss.bucket.b.securityToken")); } TEST_F(RestCredentialProviderTest, ForbiddenIsReportedToTheCaller) { diff --git a/src/paimon/rest/rest_token_file_system.cpp b/src/paimon/rest/rest_token_file_system.cpp index a3402d00..cf560c05 100644 --- a/src/paimon/rest/rest_token_file_system.cpp +++ b/src/paimon/rest/rest_token_file_system.cpp @@ -42,20 +42,11 @@ RestTokenFileSystem::RestTokenFileSystem( fs_cache_(std::move(fs_cache)), provider_(std::move(provider)) {} -std::map RestTokenFileSystem::MergeTokenOptions( - const std::map& catalog_options, const RestToken& token) { - std::map fs_options = catalog_options; - for (const auto& [key, value] : token.token) { - fs_options[key] = value; - } - return fs_options; -} - Result> RestTokenFileSystem::BuildFileSystem( const RestToken& token) const { PAIMON_ASSIGN_OR_RAISE( CoreOptions core_options, - CoreOptions::FromMap(MergeTokenOptions(catalog_options_, token), + CoreOptions::FromMap(provider_->MergeOptionsWithCredentials(catalog_options_, token.token), /*specified_file_system=*/nullptr, fs_scheme_to_identifier_map_)); std::shared_ptr fs = core_options.GetFileSystem(); if (fs == nullptr) { diff --git a/src/paimon/rest/rest_token_file_system.h b/src/paimon/rest/rest_token_file_system.h index 81f0f7f6..b1f6bc79 100644 --- a/src/paimon/rest/rest_token_file_system.h +++ b/src/paimon/rest/rest_token_file_system.h @@ -93,14 +93,6 @@ class RestTokenFileSystem : public FileSystem { /// credentials are requested with. Result ValidToken() const; - /// Merges the issued credentials over the catalog options the delegate is built from. The - /// credentials are themselves file system options, so they are overlaid key by key and win - /// wherever they overlap, mirroring the Java client; how a concrete file system resolves the - /// catalog options the token does not carry is that file system's own concern, so this merge - /// stays scheme-agnostic. Exposed for tests. - static std::map MergeTokenOptions( - const std::map& catalog_options, const RestToken& token); - private: /// Returns the file system of the current credentials, reloading them when they /// expire in less than `RestApi::kTokenExpirationSafeTimeMillis`. diff --git a/src/paimon/rest/rest_token_file_system_test.cpp b/src/paimon/rest/rest_token_file_system_test.cpp index a0ba5238..da749a5b 100644 --- a/src/paimon/rest/rest_token_file_system_test.cpp +++ b/src/paimon/rest/rest_token_file_system_test.cpp @@ -37,6 +37,7 @@ #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/defs.h" +#include "paimon/fs/credential_provider.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/rest/mock_rest_server.h" #include "paimon/rest/rest_api.h" @@ -162,8 +163,8 @@ class RestTokenFileSystemTest : public ::testing::Test { if (fs_cache == nullptr) { fs_cache = RestTokenFileSystem::CreateFileSystemCache(); } - std::shared_ptr provider = std::make_shared( - shared_api, catalog_options_, Identifier("db1", "t1"), [this] { + std::shared_ptr provider = + std::make_shared(shared_api, Identifier("db1", "t1"), [this] { return std::chrono::system_clock::time_point( std::chrono::milliseconds(now_millis_.load())); }); @@ -247,8 +248,9 @@ TEST_F(RestTokenFileSystemTest, ValidTokenCarriesOnlyTheServerCredentials) { ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); ASSERT_EQ("ak-1", token.token.at("fs.oss.accessKeyId")); - // the endpoint the credentials were issued for wins over the one the server reported - ASSERT_EQ("dlf-endpoint", token.token.at(kOssEndpointOption)); + // the token is the cache key and carries only the issued credentials, never the catalog + // secrets nor the DLF endpoint correction, which the provider applies only when merging + ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); ASSERT_EQ(kExpiresAtMillis, token.expires_at_millis); // the catalog options are not part of the token, so its secrets stay private ASSERT_EQ(0u, token.token.count(CatalogOptions::TOKEN)); @@ -348,7 +350,7 @@ TEST_F(RestTokenFileSystemTest, DefaultClockIsTheSystemClock) { RestApi::Create(catalog_options_, "", /*config_required=*/false)); // the provider is built with its default clock, the system clock std::shared_ptr provider = std::make_shared( - std::shared_ptr(std::move(api)), catalog_options_, Identifier("db1", "t1")); + std::shared_ptr(std::move(api)), Identifier("db1", "t1")); RestTokenFileSystem fs(provider, catalog_options_, RestTokenFileSystem::CreateFileSystemCache()); ASSERT_OK(fs.ValidToken().status()); @@ -356,21 +358,6 @@ TEST_F(RestTokenFileSystemTest, DefaultClockIsTheSystemClock) { ASSERT_EQ(1, state_->request_count.load()); } -TEST_F(RestTokenFileSystemTest, EmptyDlfOssEndpointIsNotApplied) { - catalog_options_[CatalogOptions::DLF_OSS_ENDPOINT] = ""; - { - std::lock_guard lock(state_->mutex); - state_->token = {{kOssEndpointOption, "server-endpoint"}}; - } - std::shared_ptr fs = CreateFileSystem(); - ASSERT_NE(nullptr, fs); - - // an unset dlf endpoint leaves the endpoint the server reported alone - ASSERT_OK_AND_ASSIGN(RestToken token, fs->ValidToken()); - ASSERT_EQ("server-endpoint", token.token.at(kOssEndpointOption)); - ASSERT_EQ(1u, token.token.size()); -} - TEST_F(RestTokenFileSystemTest, TokenOverridesTheCatalogFileSystemOptions) { // the credentials must reach the options the delegate is built from: the catalog is // configured with the local file system, yet an option of the token replaces it @@ -404,37 +391,87 @@ TEST_F(RestTokenFileSystemTest, DelegateResolvesSchemesThroughTheIdentifierMap) ASSERT_EQ(1, state_->request_count.load()); } -TEST(RestTokenFileSystemMergeTokenOptions, IssuedCredentialsOverrideTheCatalogOptions) { - // The issued credentials are file system options merged over the catalog options: they win - // wherever the two overlap, and every catalog option the token does not carry is kept as-is, - // mirroring the Java client. How the file system resolves the kept options -- a per-bucket - // variant or an alias -- is its own concern, so the merge leaves them untouched. - std::map catalog_options = { +TEST(CredentialProviderMergeOptionsWithCredentials, DefaultOverlaysCredentialsOverTheBaseOptions) { + // The default merge treats the credentials as file system options overlaid over the base + // options: they win wherever the two overlap, and every base option the credentials do not + // carry is kept as-is, mirroring the Java client. How the file system resolves the kept + // options -- a per-bucket variant or an alias -- is its own concern, so the merge leaves them + // untouched. + class DefaultProvider : public CredentialProvider { + public: + Result> GetCredentials() const override { + return std::map{}; + } + }; + DefaultProvider provider; + std::map base_options = { {"fs.oss.accessKeyId", "catalog-ak"}, {"fs.oss.endpoint", "catalog-endpoint"}, {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, {"unrelated", "kept"}, }; - RestToken token; - token.token = {{"fs.oss.accessKeyId", "token-ak"}, - {"fs.oss.accessKeySecret", "token-sk"}, - {"fs.oss.securityToken", "token-sts"}}; + std::map credentials = {{"fs.oss.accessKeyId", "token-ak"}, + {"fs.oss.accessKeySecret", "token-sk"}, + {"fs.oss.securityToken", "token-sts"}}; std::map merged = - RestTokenFileSystem::MergeTokenOptions(catalog_options, token); + provider.MergeOptionsWithCredentials(base_options, credentials); - // The token value replaces the catalog's for the key they share, and the token-only keys are - // added. + // The credential value replaces the base one for the key they share, and the credential-only + // keys are added. ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); ASSERT_EQ("token-sk", merged.at("fs.oss.accessKeySecret")); ASSERT_EQ("token-sts", merged.at("fs.oss.securityToken")); - // Catalog options the token does not set are kept untouched, whatever their form. + // Base options the credentials do not set are kept untouched, whatever their form. ASSERT_EQ("catalog-endpoint", merged.at("fs.oss.endpoint")); ASSERT_EQ("catalog-bucket-ak", merged.at("fs.oss.bucket.b.accessKeyId")); ASSERT_EQ("kept", merged.at("unrelated")); } +TEST(CredentialProviderMergeOptionsWithCredentials, OverrideCanReshapeTheMergedOptions) { + // A provider that knows the file system its credentials are for overrides + // MergeOptionsWithCredentials to clear the bucket-scoped variants the fresh credentials + // replace, which the default overlay would keep. This is the extension point an OSS-aware + // plugin uses. + class BucketClearingProvider : public CredentialProvider { + public: + Result> GetCredentials() const override { + return std::map{}; + } + std::map MergeOptionsWithCredentials( + const std::map& base_options, + const std::map& credentials) const override { + std::map merged; + for (const auto& [key, value] : base_options) { + if (key.rfind("fs.oss.bucket.", 0) == 0) { + continue; + } + merged[key] = value; + } + for (const auto& [key, value] : credentials) { + merged[key] = value; + } + return merged; + } + }; + BucketClearingProvider provider; + std::map base_options = { + {"fs.oss.accessKeyId", "catalog-ak"}, + {"fs.oss.bucket.b.accessKeyId", "catalog-bucket-ak"}, + {"unrelated", "kept"}, + }; + std::map credentials = {{"fs.oss.accessKeyId", "token-ak"}}; + + std::map merged = + provider.MergeOptionsWithCredentials(base_options, credentials); + + ASSERT_EQ("token-ak", merged.at("fs.oss.accessKeyId")); + // the bucket-scoped variant the credentials replace is cleared by the override + ASSERT_EQ(0u, merged.count("fs.oss.bucket.b.accessKeyId")); + ASSERT_EQ("kept", merged.at("unrelated")); +} + TEST_F(RestTokenFileSystemTest, ConcurrentFirstAccessLoadsTheTokenOnce) { std::string path = WriteFile("data", "paimon"); std::shared_ptr fs = CreateFileSystem();