diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index 2bf5d73cb..f5e562955 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -41,6 +41,27 @@ namespace paimon { +Result DataType::GetTimePrecision(const arrow::Field& field) { + return GetTimePrecision(field.type(), field.metadata()); +} + +Result DataType::GetTimePrecision( + const std::shared_ptr& type, + const std::shared_ptr& metadata) { + if (type->id() != arrow::Type::TIME32 || + checked_cast(*type).unit() != arrow::TimeUnit::MILLI) { + return Status::Invalid("Only millisecond TIME is supported: ", type->ToString()); + } + if (!metadata || !metadata->Contains(kTimePrecision)) { + return 0; + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::string precision, metadata->Get(kTimePrecision)); + if (precision.size() != 1 || precision[0] < '0' || precision[0] > '9') { + return Status::Invalid("Invalid TIME precision metadata: ", precision); + } + return precision[0] - '0'; +} + DataType::DataType(const std::shared_ptr& type, bool nullable, const std::shared_ptr& metadata) : type_(type), nullable_(nullable), metadata_(metadata) {} @@ -112,6 +133,13 @@ std::string DataType::DataTypeToString(const std::shared_ptr& t return "BYTES"; case arrow::Type::type::DATE32: return "DATE"; + case arrow::Type::type::TIME32: { + auto precision = GetTimePrecision(type, metadata_); + if (!precision.ok()) { + throw std::invalid_argument(precision.status().ToString()); + } + return fmt::format("TIME({})", precision.value()); + } case arrow::Type::type::DECIMAL128: { auto status = DecimalUtils::CheckDecimalType(*type); if (!status.ok()) { diff --git a/src/paimon/common/types/data_type.h b/src/paimon/common/types/data_type.h index 173960b4c..1e49ce7e1 100644 --- a/src/paimon/common/types/data_type.h +++ b/src/paimon/common/types/data_type.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include @@ -29,6 +30,7 @@ namespace arrow { class DataType; +class Field; class TimestampType; class KeyValueMetadata; } // namespace arrow @@ -37,6 +39,12 @@ namespace paimon { class DataType : public Jsonizable { public: + static constexpr char kTimePrecision[] = "paimon.time.precision"; + + // Arrow carries milliseconds, while metadata preserves the declared TIME precision. + // Arrow-only schemas use Paimon's default precision of zero. + static Result GetTimePrecision(const arrow::Field& field); + static std::unique_ptr Create( const std::shared_ptr& type, bool nullable, const std::shared_ptr& metadata); @@ -58,6 +66,10 @@ class DataType : public Jsonizable { std::shared_ptr metadata_; private: + static Result GetTimePrecision( + const std::shared_ptr& type, + const std::shared_ptr& metadata); + std::string TimestampToString(const std::shared_ptr& type) const; std::string DataTypeToString(const std::shared_ptr& type) const; }; diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index adcf5b73a..1416ae333 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -33,6 +33,7 @@ #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/types/vector_type.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/string_utils.h" @@ -84,11 +85,11 @@ struct Token { std::string value; }; -// Extension type attributes of a parsed atomic type. BLOB and VARIANT parse to plain arrow -// types (large_binary / struct) and need field-level metadata markers applied by the caller. +// Logical attributes not represented by the Arrow type are attached to the parsed field. struct AtomicTypeAttributes { bool is_blob = false; bool is_variant = false; + std::optional time_precision; }; // nullptr is returned in the case of parsing failed @@ -249,6 +250,7 @@ class TokenParser { Result> ParseStringType(); Result> ParseDecimalType(); Result> ParseDoubleType(); + Result> ParseTimeType(AtomicTypeAttributes* attributes); Result> ParseTimestampType(); Result> ParseTimestampLtzType(); Result> ParseVectorType(); @@ -522,6 +524,8 @@ Result> TokenParser::ParseTypeByKeyword( return ParseDoubleType(); case Keyword::DATE: return arrow::date32(); + case Keyword::TIME: + return ParseTimeType(attributes); case Keyword::TIMESTAMP: return ParseTimestampType(); case Keyword::TIMESTAMP_LTZ: @@ -581,6 +585,22 @@ Result> TokenParser::ParseDoubleType() { return arrow::float64(); } +Result> TokenParser::ParseTimeType( + AtomicTypeAttributes* attributes) { + PAIMON_ASSIGN_OR_RAISE(int32_t precision, ParseOptionalPrecision(/*default_precision=*/0)); + if (precision < 0 || precision > 9) { + return Status::Invalid("Time precision must be between 0 and 9 (both inclusive)"); + } + if (HasNextToken({Keyword::WITHOUT})) { + PAIMON_RETURN_NOT_OK(NextToken(Keyword::WITHOUT)); + PAIMON_RETURN_NOT_OK(NextToken(Keyword::TIME)); + PAIMON_RETURN_NOT_OK(NextToken(Keyword::ZONE)); + } + // Paimon stores TIME as milliseconds of the day, including PyPaimon's TIME(0). + attributes->time_precision = precision; + return arrow::time32(arrow::TimeUnit::MILLI); +} + Result> TokenParser::ParseTimestampType() { PAIMON_ASSIGN_OR_RAISE(int32_t precision, ParseOptionalPrecision(Timestamp::DEFAULT_PRECISION)); bool with_timezone = false; @@ -743,6 +763,11 @@ Result> DataTypeJsonParser::ParseAtomicTypeField( return BlobUtils::ToArrowField(name, nullable); } else if (attributes.is_variant) { return VariantTypeUtils::ToArrowField(name, nullable); + } else if (attributes.time_precision) { + return arrow::field( + name, type, nullable, + arrow::KeyValueMetadata::Make({DataType::kTimePrecision}, + {std::to_string(attributes.time_precision.value())})); } else { return arrow::field(name, type, nullable); } diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index 5049afc38..7c70289e0 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -22,9 +22,11 @@ #include #include +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/status.h" @@ -356,4 +358,35 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { } } +TEST(DataTypeJsonParserTest, ParseTimeType) { + std::vector types = {"TIME", "TIME WITHOUT TIME ZONE"}; + for (int32_t precision = 0; precision <= 9; ++precision) { + types.push_back(fmt::format("TIME({})", precision)); + types.push_back(fmt::format("TIME({}) WITHOUT TIME ZONE", precision)); + } + for (const auto& type : types) { + for (bool nullable : {true, false}) { + std::string type_str = nullable ? type : type + " NOT NULL"; + SCOPED_TRACE(type_str); + rapidjson::Document doc; + rapidjson::Value value(type_str.data(), doc.GetAllocator()); + ASSERT_OK_AND_ASSIGN(auto field, DataTypeJsonParser::ParseType("time", value)); + ASSERT_TRUE(field->type()->Equals(arrow::time32(arrow::TimeUnit::MILLI))); + ASSERT_EQ(field->nullable(), nullable); + auto logical_type = DataType::Create(field->type(), nullable, field->metadata()); + ASSERT_OK_AND_ASSIGN(auto serialized, logical_type->ToJsonString()); + int32_t precision = type.find('(') == std::string::npos ? 0 : type[5] - '0'; + ASSERT_EQ(serialized, + fmt::format("\"TIME({}){}\"", precision, nullable ? "" : " NOT NULL")); + } + } + for (const char* type : {"TIME(-1)", "TIME(10)", "TIME(2147483648)", "TIME()", "TIME(3, 0)", + "TIME WITH TIME ZONE", "TIME(3) WITHOUT TIME"}) { + SCOPED_TRACE(type); + rapidjson::Document doc; + rapidjson::Value value(type, doc.GetAllocator()); + ASSERT_NOK(DataTypeJsonParser::ParseType("time", value)); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index c008d222d..daf364c66 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -28,6 +28,7 @@ #include "paimon/common/data/variant/variant_access_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/types/vector_type.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/decimal_utils.h" @@ -123,6 +124,7 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId( case arrow::Type::type::DATE32: case arrow::Type::type::DECIMAL128: case arrow::Type::type::TIMESTAMP: + case arrow::Type::type::TIME32: return Status::OK(); case arrow::Type::type::LIST: { const auto& value_field = checked_cast(type.get())->value_field(); @@ -208,6 +210,9 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& case arrow::Type::type::DATE32: case arrow::Type::type::TIMESTAMP: break; + case arrow::Type::type::TIME32: + PAIMON_RETURN_NOT_OK(DataType::GetTimePrecision(*field)); + break; case arrow::Type::type::DECIMAL128: PAIMON_RETURN_NOT_OK(DecimalUtils::CheckDecimalType(*field->type())); break; diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index 283b61ec2..6a9a55f38 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -29,6 +29,7 @@ #include "paimon/common/data/variant/variant_defs.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/data_type.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/testing/utils/testharness.h" @@ -76,6 +77,39 @@ TEST(ArrowSchemaValidatorTest, TestVectorElementType) { } } +TEST(ArrowSchemaValidatorTest, TestTimeType) { + for (const auto& type : + {arrow::time32(arrow::TimeUnit::MILLI), arrow::time32(arrow::TimeUnit::SECOND), + arrow::time64(arrow::TimeUnit::MICRO), arrow::time64(arrow::TimeUnit::NANO)}) { + SCOPED_TRACE(type->ToString()); + for (const auto& field_type : {type, arrow::list(type)}) { + auto schema = DataField::ConvertDataFieldsToArrowSchema( + {DataField(0, arrow::field("time", field_type))}); + if (type->Equals(arrow::time32(arrow::TimeUnit::MILLI))) { + ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*schema)); + ASSERT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*schema)); + } else { + ASSERT_NOK(ArrowSchemaValidator::ValidateSchema(*schema)); + ASSERT_NOK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*schema)); + } + } + } +} + +TEST(ArrowSchemaValidatorTest, TestInvalidTimePrecision) { + for (const char* precision : {"", "-1", "10", "3x", "1.5", "2147483648"}) { + SCOPED_TRACE(precision); + auto field = + arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI), true, + arrow::KeyValueMetadata::Make({DataType::kTimePrecision}, {precision})); + ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow::schema({field})), + "Invalid TIME precision metadata"); + ASSERT_NOK_WITH_MSG(DataField(0, field).ToJsonString(), "Invalid TIME precision metadata"); + } + auto seconds = DataType::Create(arrow::time32(arrow::TimeUnit::SECOND), true, nullptr); + ASSERT_NOK_WITH_MSG(seconds->ToJsonString(), "Only millisecond TIME is supported"); +} + TEST(ArrowSchemaValidatorTest, TestValidateNoRedundantFields) { auto col1_field = arrow::field("col1", arrow::int64()); auto col2_field = arrow::field("col2", arrow::int32()); diff --git a/src/paimon/core/schema/schema_manager_test.cpp b/src/paimon/core/schema/schema_manager_test.cpp index 0b078462b..cd4823891 100644 --- a/src/paimon/core/schema/schema_manager_test.cpp +++ b/src/paimon/core/schema/schema_manager_test.cpp @@ -25,12 +25,76 @@ #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/common/types/data_type.h" +#include "paimon/common/types/data_type_json_parser.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +TEST(SchemaManagerTest, TimePrecisionRoundTrip) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + SchemaManager manager(fs, dir->Str()); + arrow::FieldVector fields; + for (int32_t precision = 0; precision <= 9; ++precision) { + for (bool nullable : {true, false}) { + std::string name = "t" + std::to_string(fields.size()); + std::string type = + "TIME(" + std::to_string(precision) + ")" + (nullable ? "" : " NOT NULL"); + rapidjson::Document doc; + rapidjson::Value value(type.c_str(), doc.GetAllocator()); + ASSERT_OK_AND_ASSIGN(auto field, DataTypeJsonParser::ParseType(name, value)); + fields.push_back(field); + } + } + fields.push_back(arrow::field("default_time", arrow::time32(arrow::TimeUnit::MILLI))); + fields.push_back(arrow::field("times", arrow::list(fields[6]->WithName("item")))); + fields.push_back( + arrow::field("mapping", std::make_shared(fields[13]->WithName("key"), + fields[18]->WithName("value")))); + fields.push_back(arrow::field("nested", arrow::struct_({fields[6], fields[13], fields[18]}))); + ASSERT_OK_AND_ASSIGN(auto created, + manager.CreateTable(arrow::schema(fields), {}, {}, + {{"file.format", "parquet"}, {"bucket", "-1"}})); + ASSERT_OK_AND_ASSIGN(auto serialized, created->ToJsonString()); + SchemaManager reloaded_manager(fs, dir->Str()); + ASSERT_OK_AND_ASSIGN(auto reloaded, reloaded_manager.ReadSchema(0)); + ASSERT_OK_AND_ASSIGN(auto reserialized, reloaded->ToJsonString()); + ASSERT_EQ(serialized, reserialized); + const auto& restored_fields = reloaded->Fields(); + for (int32_t i = 0; i < 20; ++i) { + ASSERT_OK_AND_ASSIGN(auto precision, + DataType::GetTimePrecision(*restored_fields[i].ArrowField())); + ASSERT_EQ(precision, i / 2); + ASSERT_EQ(restored_fields[i].ArrowField()->nullable(), i % 2 == 0); + } + ASSERT_OK_AND_ASSIGN(auto default_precision, + DataType::GetTimePrecision(*restored_fields[20].ArrowField())); + ASSERT_EQ(default_precision, 0); + for (int32_t i = 21; i < 24; ++i) { + SCOPED_TRACE(i); + ASSERT_TRUE(DataField::ConvertDataFieldToArrowField(created->Fields()[i]) + ->Equals(DataField::ConvertDataFieldToArrowField(restored_fields[i]), + /*check_metadata=*/true)); + } +} + +TEST(SchemaManagerTest, RejectTimePartitionKey) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + SchemaManager manager(std::make_shared(), dir->Str()); + auto schema = arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI))}); + ASSERT_NOK_WITH_MSG( + manager.CreateTable(schema, {"time"}, {}, {{"file.format", "parquet"}, {"bucket", "-1"}}), + "partition field time cannot be TIME"); + ASSERT_OK_AND_ASSIGN(auto latest, manager.Latest()); + ASSERT_FALSE(latest.has_value()); +} + TEST(SchemaManagerTest, ConcurrentHistoricalSchemaReads) { SchemaManager manager( std::make_shared(), diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index d97d73a86..2a9c33222 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -397,6 +397,10 @@ Status SchemaValidation::ValidateNotContainSpecificType( auto it = fields_map.find(field_name); if (it != fields_map.end()) { auto field = it->second; + if (field->type()->id() == arrow::Type::TIME32) { + return Status::Invalid( + fmt::format("partition field {} cannot be TIME", field_name)); + } if (IsComplexType(field)) { return Status::Invalid( fmt::format("partition field {} cannot be TIMESTAMP/DECIMAL/BLOB", field_name)); diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 547289877..f6c08b868 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -289,9 +289,13 @@ TEST(SchemaValidationTest, TestLanceDataTypes) { arrow::field("map", arrow::map(arrow::int32(), arrow::utf8())), arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")), VariantTypeUtils::ToArrowField("variant"), + arrow::field("time_millis", arrow::time32(arrow::TimeUnit::MILLI)), + arrow::field("nested_time", + arrow::struct_({arrow::field( + "values", arrow::list(arrow::time32(arrow::TimeUnit::MILLI)))})), }; - std::vector expected_errors = {"type MAP", "LOCAL_ZONED_TIMESTAMP", - "type VARIANT"}; + std::vector expected_errors = {"type MAP", "LOCAL_ZONED_TIMESTAMP", "type VARIANT", + "type time32", "type time32"}; for (size_t i = 0; i < unsupported_fields.size(); ++i) { ASSERT_OK_AND_ASSIGN( table_schema, @@ -303,14 +307,13 @@ TEST(SchemaValidationTest, TestLanceDataTypes) { for (const auto& field : arrow::FieldVector{ arrow::field("time_seconds", arrow::time32(arrow::TimeUnit::SECOND)), - arrow::field("time_millis", arrow::time32(arrow::TimeUnit::MILLI)), arrow::field("nested_time", arrow::struct_({arrow::field( - "values", arrow::list(arrow::time32(arrow::TimeUnit::MILLI)))}))}) { + "values", arrow::list(arrow::time32(arrow::TimeUnit::SECOND)))}))}) { ASSERT_NOK_WITH_MSG( TableSchema::Create(/*schema_id=*/0, arrow::schema({field}), /*partition_keys=*/{}, /*primary_keys=*/{}, options), - "Unknown or unsupported arrow type: time32"); + "Only millisecond TIME is supported"); } for (const auto& [option_key, option_value] : std::vector>{ @@ -691,6 +694,23 @@ TEST(SchemaValidationTest, TestSpecificPartitionKey) { } } +TEST(SchemaValidationTest, TestTimePartitionKey) { + auto schema = arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI))}); + for (const std::vector& partition_keys : + {std::vector{}, std::vector{"time"}}) { + ASSERT_OK_AND_ASSIGN(auto table_schema, + TableSchema::Create(0, schema, partition_keys, {}, + {{"file.format", "parquet"}, {"bucket", "-1"}})); + if (partition_keys.empty()) { + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); + } else { + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "partition field time cannot be TIME"); + } + } +} + TEST(SchemaValidationTest, TestComplexPartitionKeyWithBlob) { auto f0 = arrow::field("f0", arrow::utf8()); auto f1 = BlobUtils::ToArrowField("f1"); diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index be7287dd6..61a40365b 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -177,11 +177,6 @@ Result>> FieldMappingBuilder::CreateDa std::vector> cast_executors; cast_executors.reserve(read_fields.size()); for (size_t i = 0; i < read_fields.size(); i++) { - PAIMON_ASSIGN_OR_RAISE(FieldType read_type, - FieldTypeUtils::ConvertToFieldType(read_fields[i].Type()->id())); - PAIMON_ASSIGN_OR_RAISE(FieldType data_type, - FieldTypeUtils::ConvertToFieldType(data_fields[i].Type()->id())); - if (!read_fields[i].Type()->Equals(data_fields[i].Type())) { auto read_type_id = read_fields[i].Type()->id(); if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST || @@ -191,6 +186,10 @@ Result>> FieldMappingBuilder::CreateDa cast_executors.push_back(nullptr); continue; } + PAIMON_ASSIGN_OR_RAISE(FieldType read_type, + FieldTypeUtils::ConvertToFieldType(read_fields[i].Type()->id())); + PAIMON_ASSIGN_OR_RAISE(FieldType data_type, + FieldTypeUtils::ConvertToFieldType(data_fields[i].Type()->id())); auto executor_factory = CastExecutorFactory::GetCastExecutorFactory(); auto cast_executor = executor_factory->GetCastExecutor(/*src=*/data_type, /*target=*/read_type); diff --git a/src/paimon/core/utils/field_mapping_test.cpp b/src/paimon/core/utils/field_mapping_test.cpp index 1ba674cd7..a970a31ea 100644 --- a/src/paimon/core/utils/field_mapping_test.cpp +++ b/src/paimon/core/utils/field_mapping_test.cpp @@ -677,4 +677,22 @@ TEST_F(FieldMappingTest, TestMapSelectedKeysMetadataPropagatedToDataSchema) { ASSERT_FALSE(custom_metadata_result.ok()); } +TEST_F(FieldMappingTest, TestTimeWithoutCast) { + std::vector fields = { + DataField(0, arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI)))}; + auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK_AND_ASSIGN(auto builder, FieldMappingBuilder::Create(schema, /*partition_keys=*/{}, + /*predicate=*/nullptr)); + ASSERT_OK_AND_ASSIGN(auto mapping, builder->CreateFieldMapping(fields)); + const auto& info = mapping->non_partition_info; + ASSERT_EQ(info.non_partition_read_schema, fields); + ASSERT_EQ(info.non_partition_data_schema, fields); + ASSERT_EQ(info.cast_executors.size(), 1); + ASSERT_EQ(info.cast_executors[0], nullptr); + + std::vector int_fields = {DataField(0, arrow::field("time", arrow::int32()))}; + ASSERT_NOK(FieldMappingBuilder::CreateDataCastExecutors(fields, int_fields)); + ASSERT_NOK(FieldMappingBuilder::CreateDataCastExecutors(int_fields, fields)); +} + } // namespace paimon::test diff --git a/test/inte/paimon_read_compat_inte_test.cpp b/test/inte/paimon_read_compat_inte_test.cpp index 0af1dd1fe..f6468b6f8 100644 --- a/test/inte/paimon_read_compat_inte_test.cpp +++ b/test/inte/paimon_read_compat_inte_test.cpp @@ -406,6 +406,23 @@ TEST_P(PaimonReadCompatInteTest, ReadsCompatibleTypeValues) { } } +TEST_P(PaimonReadCompatInteTest, ReadsTimeValues) { + const CompatibilityParam& param = GetParam(); + if (param.file_format != "parquet") { + GTEST_SKIP() << "TIME reading is only supported for Parquet"; + } + for (int32_t precision : {0, 3, 6, 9}) { + std::string field_name = "f_time_" + std::to_string(precision); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(param, param.writer_prefix + "_time_types", {field_name})); + auto rows = GetOnlyStructChunk(result); + ASSERT_TRUE(rows); + ASSERT_EQ(rows->length(), 2); + AssertFieldEqualsJson(rows, field_name, arrow::time32(arrow::TimeUnit::MILLI), + precision == 0 ? "[45296000, null]" : "[45296123, null]"); + } +} + TEST_P(PaimonReadCompatInteTest, ReadsBlobValues) { const CompatibilityParam& param = GetParam(); @@ -566,20 +583,30 @@ std::vector UnsupportedReadParams() { const std::vector read_cases = { {"ArrayBlob", "array_blob_types", "f_array_blob", "BLOB field must be a top-level field or the direct value of a top-level MAP field"}, - {"TimePrecision0", "time_types", "f_time_0", "Unsupported type: TIME"}, - {"TimePrecision3", "time_types", "f_time_3", "Unsupported type: TIME"}, - {"TimePrecision6", "time_types", "f_time_6", "Unsupported type: TIME"}, - {"TimePrecision9", "time_types", "f_time_9", "Unsupported type: TIME"}, + {"TimePrecision0", "time_types", "f_time_0", ""}, + {"TimePrecision3", "time_types", "f_time_3", ""}, + {"TimePrecision6", "time_types", "f_time_6", ""}, + {"TimePrecision9", "time_types", "f_time_9", ""}, }; std::vector result; for (const CompatibilityParam& param : CompatibilityParams()) { for (const UnsupportedReadCase& read_case : read_cases) { + bool is_time = read_case.table_suffix == "time_types"; + if (is_time && param.file_format == "parquet") { + continue; + } // Java Avro cannot create TIME(6/9), so its negative table contains TIME(0/3) only. if (param.file_format == "avro" && (read_case.name == "TimePrecision6" || read_case.name == "TimePrecision9")) { continue; } - result.push_back({param.file_format, param.writer_prefix, read_case}); + auto expected_case = read_case; + if (is_time) { + expected_case.expected_error = param.file_format == "orc" + ? "Unknown or unsupported Arrow type: time32[ms]" + : "invalid avro logical type"; + } + result.push_back({param.file_format, param.writer_prefix, expected_case}); } } return result;