diff --git a/documentation/components/extensions/arrow-ext.md b/documentation/components/extensions/arrow-ext.md index 28f69549bb..73bedf0c0f 100644 --- a/documentation/components/extensions/arrow-ext.md +++ b/documentation/components/extensions/arrow-ext.md @@ -35,7 +35,7 @@ The Arrow Rust crates offer additional I/O capabilities that are candidates for ## Features - Read and write Apache Parquet files through PHP streaming interfaces -- Flat types: INT32, INT64, FLOAT, DOUBLE, BOOLEAN, STRING, BINARY, DATE32, TIMESTAMP +- Flat types: INT32, INT64, FLOAT, DOUBLE, BOOLEAN, STRING, BINARY, DATE, TIMESTAMP, TIME, DECIMAL - Nested types: LIST, STRUCT, MAP (arbitrarily nested) - Compression codecs: UNCOMPRESSED, SNAPPY, GZIP, ZSTD, LZ4_RAW, BROTLI - Column projection for selective reads @@ -179,20 +179,23 @@ $writer->close(); The schema is an array of column definitions. Each column has a `name`, `type`, and optional `optional` flag. -| Type | PHP Read Value | Notes | -|------|---------------|-------| -| `BOOLEAN` | `bool` | | -| `INT32` | `int` | | -| `INT64` | `int` | | -| `FLOAT` | `float` | | -| `DOUBLE` | `float` | | -| `STRING` | `string` | | -| `BINARY` | `string` (raw bytes) | | -| `DATE32` | `string` (YYYY-MM-DD) | | -| `TIMESTAMP` | `string` (ISO 8601) | | -| `LIST` | `array` | Requires `children` key with 1 element | -| `STRUCT` | `array` (associative) | Requires `children` key with N elements | -| `MAP` | `array` (associative) | Requires `children` key with 2 elements (key + value) | +| Type | PHP Read Value | Notes | +|-------------|-----------------------|-------------------------------------------------------------------------------------------------------------------| +| `BOOLEAN` | `bool` | | +| `INT32` | `int` | | +| `INT64` | `int` | | +| `FLOAT` | `float` | | +| `DOUBLE` | `float` | | +| `STRING` | `string` | | +| `BINARY` | `string` (raw bytes) | | +| `DATE` | `DateTimeImmutable` | `int` lane: days since epoch | +| `TIMESTAMP` | `DateTimeImmutable` | Keys `unit` (`MILLIS\|MICROS\|NANOS`, default `MICROS`) and `utc` (default `true`); `int` lane in the column unit | +| `TIME` | `DateInterval` | Key `unit` (`MILLIS\|MICROS\|NANOS`, default `MICROS`) | +| `DECIMAL` | `float` | Keys `precision`, `scale` | +| `UUID` | `string` | | +| `LIST` | `array` | Requires `children` key with 1 element | +| `STRUCT` | `array` (associative) | Requires `children` key with N elements | +| `MAP` | `array` (associative) | Requires `children` key with 2 elements (key + value) | **Nested schema example:** diff --git a/documentation/components/libs/parquet.md b/documentation/components/libs/parquet.md index c09ee2103c..cd7fed2030 100644 --- a/documentation/components/libs/parquet.md +++ b/documentation/components/libs/parquet.md @@ -256,6 +256,8 @@ $schema = Schema::with( ); ``` +`FlatColumn::dateTime()` is written as `TIMESTAMP(isAdjustedToUTC=true, MICROS)`; files with `isAdjustedToUTC=false` read as UTC wall clock. + Once we have a schema, we can create a writer. ```php @@ -333,8 +335,6 @@ $writer->close(); - `GZIP_COMPRESSION_LEVEL` - default: `9` - compression level for GZIP compression (applied only when GZIP compression is enabled). - `PAGE_SIZE_BYTES` - default: `8Kb` - maximum size of data page. -- `ROUND_NANOSECONDS` - default: `false` - Since PHP does not support nanoseconds precision for DateTime objects, when - this options is set to true, reader will round nanoseconds to microseconds. - `ROW_GROUP_SIZE_BYTES` - default: `8Mb` - maximum size of row group. - `ROW_GROUP_SIZE_CHECK_INTERVAL` default: `1000` - number of rows to write before checking if row group size limit is reached. diff --git a/documentation/upgrading.md b/documentation/upgrading.md index d6fea871cc..ba32b9a938 100644 --- a/documentation/upgrading.md +++ b/documentation/upgrading.md @@ -419,6 +419,54 @@ The same applies to map values and structure elements. | `new DateTimeDefinition('at', true)` | `new DateTimeDefinition('at', type_datetime(), true)` or `datetime_schema('at', nullable: true, zone: 'Europe/Warsaw')` | | a `flow_php` extension older than this release | ignored - the PHP engine runs; a current extension with an older `flow-php/etl` keeps that library's behaviour | +### 41) `flow-php/parquet` - `Converter::isFor()` replaced by `static Converter::forColumn()`, `Int32DateTimeConverter` removed + +| Before | After | +|--------------------------------------------------------------------|------------------------------------------------------------------------------------| +| `Converter::isFor(FlatColumn, Options): bool` on a shared instance | `static Converter::forColumn(FlatColumn, Options): ?self` - a converter per column | +| `new DataConverter([new TimeConverter(), ...], $options)` | `new DataConverter([TimeConverter::class, ...], $options)` | +| `Int32DateTimeConverter` | removed - INT32 TIMESTAMP is not a legal Parquet carrier | + +### 42) `flow-php/parquet` - `LogicalType\Timestamp` / `Time` take a `TimeUnit` + +| Before | After | +|-------------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| `new Timestamp($isAdjustedToUTC, $millis, $micros, $nanos)` | `new Timestamp($isAdjustedToUTC, TimeUnit::MICROSECONDS)`, same for `Time` | +| `TimeUnit` - pure enum with `MICROSECONDS` only | string-backed enum `MILLISECONDS = 'MILLIS'`, `MICROSECONDS = 'MICROS'`, `NANOSECONDS = 'NANOS'` | + +### 43) `flow-php/parquet` - `Option::ROUND_NANOSECONDS` removed + +| Before | After | +|-----------------------------|---------------------------------------------------------------------------------------| +| `Option::ROUND_NANOSECONDS` | removed - NANOS timestamps always read as `DateTimeImmutable` floored to microseconds | + +### 44) `flow-php/parquet` - `encode_decimal()` / `decode_decimal()` drop `ByteOrder` and the read-side precision check + +| Before | After | +|---------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------| +| `encode_decimal(ByteOrder, float, int $byteLength, int $precision, int $scale)` | `encode_decimal(float, int $precision, int $scale, ?int $byteLength)` - `null` = minimal length (BYTE_ARRAY) | +| `decode_decimal(ByteOrder, string, int $precision, int $scale)` | `decode_decimal(string, int $scale)` - no precision check | + +### 45) `flow-php/parquet`, `flow-php/arrow-ext` - TIMESTAMP columns are written `isAdjustedToUTC=true` + +| Before | After | +|--------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------| +| `FlatColumn::dateTime()` / arrow `TIMESTAMP` written `isAdjustedToUTC=false` (DuckDB/Spark read `TIMESTAMP`) | written `isAdjustedToUTC=true` (DuckDB/Spark read `TIMESTAMPTZ`); reading `false` files is unchanged | + +### 46) `flow-php/parquet`, `flow-php/etl-adapter-parquet` - ConvertedType-only TIMESTAMP/TIME/DECIMAL columns are typed + +| Before | After | +|-------------------------------------------------------------------------------|------------| +| `TIMESTAMP_MILLIS` / `TIMESTAMP_MICROS` column without a logical type → `int` | `datetime` | +| `TIME_MILLIS` / `TIME_MICROS` column without a logical type → `int` | `time` | +| `DECIMAL` converted type on INT32/INT64 → unscaled `int` | `float` | + +### 47) `flow-php/arrow-ext` - `Writer` DATE `int` lane is days since epoch + +| Before | After | +|---------------------------------------------|-------------------------------------| +| `'d' => [1641600000]` - seconds since epoch | `'d' => [19000]` - days since epoch | + --- ## Upgrading from 0.43.x to 0.44.x diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ConvertedTypeOnlyColumnsTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ConvertedTypeOnlyColumnsTest.php new file mode 100644 index 0000000000..1482b1da37 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Integration/ConvertedTypeOnlyColumnsTest.php @@ -0,0 +1,95 @@ +openForStream( + $memory->writeTo($path), + Schema::with( + new FlatColumn( + 'ts_ms', + PhysicalType::INT64, + ConvertedType::TIMESTAMP_MILLIS, + null, + Repetition::OPTIONAL, + ), + new FlatColumn( + 'ts_us', + PhysicalType::INT64, + ConvertedType::TIMESTAMP_MICROS, + null, + Repetition::OPTIONAL, + ), + new FlatColumn('t_ms', PhysicalType::INT32, ConvertedType::TIME_MILLIS, null, Repetition::OPTIONAL), + new FlatColumn('t_us', PhysicalType::INT64, ConvertedType::TIME_MICROS, null, Repetition::OPTIONAL), + new FlatColumn('dec9', PhysicalType::INT32, ConvertedType::DECIMAL, null, Repetition::OPTIONAL, 9, 2), + ), + ); + $writer->writeBatch([[ + 'ts_ms' => 1577934245678, + 'ts_us' => 1577934245678901, + 't_ms' => 11045678, + 't_us' => 11045678901, + 'dec9' => 1234567, + ]]); + $writer->close(); + + $rows = data_frame()->read(from_parquet($path, filesystem: $memory))->fetch(); + + static::assertEquals( + schema( + datetime_schema('ts_ms', true), + datetime_schema('ts_us', true), + time_schema('t_ms', true), + time_schema('t_us', true), + float_schema('dec9', true), + ), + $rows->schema(), + ); + + $row = $rows[0]; + $tsMs = $row->get('ts_ms'); + $tsUs = $row->get('ts_us'); + $tMs = $row->get('t_ms'); + $tUs = $row->get('t_us'); + + static::assertInstanceOf(DateTimeInterface::class, $tsMs); + static::assertInstanceOf(DateTimeInterface::class, $tsUs); + static::assertInstanceOf(DateInterval::class, $tMs); + static::assertInstanceOf(DateInterval::class, $tUs); + static::assertSame('2020-01-02 03:04:05.678000 +00:00', $tsMs->format('Y-m-d H:i:s.u P')); + static::assertSame('2020-01-02 03:04:05.678901 +00:00', $tsUs->format('Y-m-d H:i:s.u P')); + static::assertSame('03:04:05.678000', $tMs->format('%H:%I:%S.%F')); + static::assertSame('03:04:05.678901', $tUs->format('%H:%I:%S.%F')); + static::assertSame(12345.67, $row->get('dec9')); + } +} diff --git a/src/extension/arrow-ext/src/parquet/reader.rs b/src/extension/arrow-ext/src/parquet/reader.rs index cc5ca44e93..f948fd4143 100644 --- a/src/extension/arrow-ext/src/parquet/reader.rs +++ b/src/extension/arrow-ext/src/parquet/reader.rs @@ -1,4 +1,4 @@ -use arrow_schema::DataType; +use arrow_schema::{DataType, TimeUnit}; use ext_php_rs::boxed::ZBox; use ext_php_rs::prelude::*; use ext_php_rs::types::{ZendHashTable, ZendObject}; @@ -108,6 +108,15 @@ impl Reader { } } +fn time_unit_name(unit: &TimeUnit) -> &'static str { + match unit { + // Parquet has no second unit; arrow-rs stores seconds as MILLIS. + TimeUnit::Second | TimeUnit::Millisecond => "MILLIS", + TimeUnit::Microsecond => "MICROS", + TimeUnit::Nanosecond => "NANOS", + } +} + fn field_to_php_schema(field: &arrow_schema::Field) -> PhpResult> { let mut entry = ZendHashTable::new(); entry @@ -131,7 +140,7 @@ fn field_to_php_schema(field: &arrow_schema::Field) -> PhpResult "DATE", DataType::Timestamp(_, _) => "TIMESTAMP", DataType::Time32(_) | DataType::Time64(_) => "TIME", - DataType::Decimal128(_, _) => "DECIMAL", + DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => "DECIMAL", DataType::FixedSizeBinary(_) => "FIXED_SIZE_BINARY", DataType::List(_) | DataType::LargeList(_) => "LIST", DataType::Struct(_) => "STRUCT", @@ -152,7 +161,7 @@ fn field_to_php_schema(field: &arrow_schema::Field) -> PhpResult { + DataType::Decimal128(precision, scale) | DataType::Decimal256(precision, scale) => { entry .insert("precision", *precision as i64) .map_err(|_| parquet_exception("Failed to build schema entry"))?; @@ -160,6 +169,19 @@ fn field_to_php_schema(field: &arrow_schema::Field) -> PhpResult { + entry + .insert("unit", time_unit_name(unit)) + .map_err(|_| parquet_exception("Failed to build schema entry"))?; + entry + .insert("utc", tz.is_some()) + .map_err(|_| parquet_exception("Failed to build schema entry"))?; + } + DataType::Time32(unit) | DataType::Time64(unit) => { + entry + .insert("unit", time_unit_name(unit)) + .map_err(|_| parquet_exception("Failed to build schema entry"))?; + } DataType::FixedSizeBinary(n) => { entry .insert("length", *n as i64) diff --git a/src/extension/arrow-ext/src/parquet/type_converter.rs b/src/extension/arrow-ext/src/parquet/type_converter.rs index 5bb7681a99..323e7c9e71 100644 --- a/src/extension/arrow-ext/src/parquet/type_converter.rs +++ b/src/extension/arrow-ext/src/parquet/type_converter.rs @@ -1,9 +1,9 @@ use arrow_array::builder::{ BinaryBuilder, BooleanBuilder, Date32Builder, Decimal128Builder, FixedSizeBinaryBuilder, Float32Builder, Float64Builder, Int16Builder, Int32Builder, Int64Builder, Int8Builder, - StringBuilder, Time64MicrosecondBuilder, TimestampMicrosecondBuilder, UInt16Builder, - UInt32Builder, UInt64Builder, UInt8Builder, + StringBuilder, UInt16Builder, UInt32Builder, UInt64Builder, UInt8Builder, }; +use arrow_array::types::{Decimal128Type, Decimal256Type, DecimalType}; use arrow_array::*; use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{ @@ -18,6 +18,13 @@ use std::sync::Arc; use crate::parquet::exception::parquet_exception; +fn split_epoch(value: i64, units_per_second: i64) -> (i64, i64) { + ( + value.div_euclid(units_per_second), + value.rem_euclid(units_per_second) * (1_000_000 / units_per_second), + ) +} + fn create_datetime_immutable(seconds: i64, microseconds: i64) -> PhpResult { let mut callable_ht = ZendHashTable::with_capacity(2); let mut class_zv = Zval::new(); @@ -39,7 +46,7 @@ fn create_datetime_immutable(seconds: i64, microseconds: i64) -> PhpResult callable.set_hashtable(callable_ht); let format = "U.u".to_string(); - let value = format!("{}.{:06}", seconds, microseconds.unsigned_abs()); + let value = format!("{}.{:06}", seconds, microseconds); callable .try_call(vec![ @@ -121,6 +128,17 @@ fn extract_datetime_timestamp(zv: &Zval, column_name: &str) -> Result<(i64, i64) Ok((seconds, microseconds)) } +fn extract_datetime_offset(zv: &Zval, column_name: &str) -> Result { + let obj = zv + .object() + .ok_or_else(|| format!("Column '{}': expected object", column_name))?; + + obj.try_call_method("getOffset", vec![]) + .map_err(|e| format!("Column '{}': getOffset() failed: {:?}", column_name, e))? + .long() + .ok_or_else(|| format!("Column '{}': getOffset() did not return int", column_name)) +} + fn extract_date_interval_microseconds(zv: &Zval, column_name: &str) -> Result { let obj = zv .object() @@ -144,6 +162,19 @@ fn extract_date_interval_microseconds(zv: &Zval, column_name: &str) -> Result(value: T::Native, precision: u8, scale: i8) -> PhpResult { + let str_val = T::format_decimal(value, precision, scale); + let float_val: f64 = str_val.parse().map_err(|e| { + parquet_exception(format!( + "Failed to parse decimal value '{}': {}", + str_val, e + )) + })?; + let mut zv = Zval::new(); + zv.set_double(float_val); + Ok(zv) +} + fn downcast_array<'a, T: 'static>(array: &'a dyn Array, type_name: &str) -> PhpResult<&'a T> { array .as_any() @@ -335,8 +366,8 @@ fn convert_single_arrow_value(array: &dyn Array, index: usize, is_uuid: bool) -> } DataType::Date64 => { let arr = downcast_array::(array, "Date64Array")?; - let ms = arr.value(index); - create_datetime_immutable(ms / 1000, (ms % 1000) * 1000) + let (s, us) = split_epoch(arr.value(index), 1_000); + create_datetime_immutable(s, us) } DataType::Timestamp(TimeUnit::Second, _) => { let arr = downcast_array::(array, "TimestampSecondArray")?; @@ -345,20 +376,20 @@ fn convert_single_arrow_value(array: &dyn Array, index: usize, is_uuid: bool) -> DataType::Timestamp(TimeUnit::Millisecond, _) => { let arr = downcast_array::(array, "TimestampMillisecondArray")?; - let v = arr.value(index); - create_datetime_immutable(v / 1000, (v % 1000) * 1000) + let (s, us) = split_epoch(arr.value(index), 1_000); + create_datetime_immutable(s, us) } DataType::Timestamp(TimeUnit::Microsecond, _) => { let arr = downcast_array::(array, "TimestampMicrosecondArray")?; - let v = arr.value(index); - create_datetime_immutable(v / 1_000_000, v % 1_000_000) + let (s, us) = split_epoch(arr.value(index), 1_000_000); + create_datetime_immutable(s, us) } DataType::Timestamp(TimeUnit::Nanosecond, _) => { let arr = downcast_array::(array, "TimestampNanosecondArray")?; - let v = arr.value(index); - create_datetime_immutable(v / 1_000_000_000, (v / 1000) % 1_000_000) + let (s, us) = split_epoch(arr.value(index).div_euclid(1_000), 1_000_000); + create_datetime_immutable(s, us) } DataType::Time32(TimeUnit::Second) => { let arr = downcast_array::(array, "Time32SecondArray")?; @@ -374,20 +405,15 @@ fn convert_single_arrow_value(array: &dyn Array, index: usize, is_uuid: bool) -> } DataType::Time64(TimeUnit::Nanosecond) => { let arr = downcast_array::(array, "Time64NanosecondArray")?; - create_date_interval(arr.value(index) / 1000) + create_date_interval(arr.value(index).div_euclid(1_000)) } - DataType::Decimal128(_, _) => { + DataType::Decimal128(precision, scale) => { let arr = downcast_array::(array, "Decimal128Array")?; - let str_val = arr.value_as_string(index); - let float_val: f64 = str_val.parse().map_err(|e| { - parquet_exception(format!( - "Failed to parse decimal value '{}': {}", - str_val, e - )) - })?; - let mut zv = Zval::new(); - zv.set_double(float_val); - Ok(zv) + decimal_to_zval::(arr.value(index), *precision, *scale) + } + DataType::Decimal256(precision, scale) => { + let arr = downcast_array::(array, "Decimal256Array")?; + decimal_to_zval::(arr.value(index), *precision, *scale) } DataType::List(element_field) => { let list_array = downcast_array::(array, "ListArray")?; @@ -622,7 +648,8 @@ pub fn arrow_array_to_php_values(array: &dyn Array, field: Option<&Field>) -> Ph DataType::Date64 => { let arr = downcast_array::(array, "Date64Array")?; convert_primitive_values(arr, |v| { - create_datetime_immutable(v / 1000, (v % 1000) * 1000) + let (s, us) = split_epoch(v, 1_000); + create_datetime_immutable(s, us) }) } @@ -634,21 +661,24 @@ pub fn arrow_array_to_php_values(array: &dyn Array, field: Option<&Field>) -> Ph let arr = downcast_array::(array, "TimestampMillisecondArray")?; convert_primitive_values(arr, |v| { - create_datetime_immutable(v / 1000, (v % 1000) * 1000) + let (s, us) = split_epoch(v, 1_000); + create_datetime_immutable(s, us) }) } DataType::Timestamp(TimeUnit::Microsecond, _) => { let arr = downcast_array::(array, "TimestampMicrosecondArray")?; convert_primitive_values(arr, |v| { - create_datetime_immutable(v / 1_000_000, v % 1_000_000) + let (s, us) = split_epoch(v, 1_000_000); + create_datetime_immutable(s, us) }) } DataType::Timestamp(TimeUnit::Nanosecond, _) => { let arr = downcast_array::(array, "TimestampNanosecondArray")?; convert_primitive_values(arr, |v| { - create_datetime_immutable(v / 1_000_000_000, (v / 1000) % 1_000_000) + let (s, us) = split_epoch(v.div_euclid(1_000), 1_000_000); + create_datetime_immutable(s, us) }) } @@ -666,30 +696,21 @@ pub fn arrow_array_to_php_values(array: &dyn Array, field: Option<&Field>) -> Ph } DataType::Time64(TimeUnit::Nanosecond) => { let arr = downcast_array::(array, "Time64NanosecondArray")?; - convert_primitive_values(arr, |v| create_date_interval(v / 1000)) + convert_primitive_values(arr, |v| create_date_interval(v.div_euclid(1_000))) } - DataType::Decimal128(_, _) => { + DataType::Decimal128(precision, scale) => { let arr = downcast_array::(array, "Decimal128Array")?; - let len = arr.len(); - let mut result = Vec::with_capacity(len); - for i in 0..len { - let mut zv = Zval::new(); - if arr.is_null(i) { - zv.set_null(); - } else { - let str_val = arr.value_as_string(i); - let float_val: f64 = str_val.parse().map_err(|e| { - parquet_exception(format!( - "Failed to parse decimal value '{}': {}", - str_val, e - )) - })?; - zv.set_double(float_val); - } - result.push(zv); - } - Ok(result) + convert_primitive_values(arr, |v| { + decimal_to_zval::(v, *precision, *scale) + }) + } + + DataType::Decimal256(precision, scale) => { + let arr = downcast_array::(array, "Decimal256Array")?; + convert_primitive_values(arr, |v| { + decimal_to_zval::(v, *precision, *scale) + }) } DataType::List(_) | DataType::LargeList(_) | DataType::Struct(_) | DataType::Map(_, _) => { @@ -729,6 +750,52 @@ fn zval_type_name(zv: &Zval) -> &'static str { } } +fn round_decimal_half_away(repr: &str, scale: i8) -> String { + let (sign, digits) = match repr.strip_prefix('-') { + Some(stripped) => ("-", stripped), + None => ("", repr), + }; + let (integer_part, fractional_part) = match digits.find('.') { + Some(dot_pos) => (&digits[..dot_pos], &digits[dot_pos + 1..]), + None => (digits, ""), + }; + let scale = scale.max(0) as usize; + + if fractional_part.len() <= scale { + return format!( + "{}{}.{:0 = format!("{}{}", integer_part, &fractional_part[..scale]).into_bytes(); + + if fractional_part.as_bytes()[scale] >= b'5' { + let mut i = kept.len(); + loop { + if i == 0 { + kept.insert(0, b'1'); + break; + } + i -= 1; + if kept[i] == b'9' { + kept[i] = b'0'; + } else { + kept[i] += 1; + break; + } + } + } + + let kept = String::from_utf8(kept).unwrap_or_default(); + let split = kept.len() - scale; + + format!("{}{}.{}", sign, &kept[..split], &kept[split..]) +} + fn parse_decimal_string(s: &str, scale: i8) -> Result { let trimmed = s.trim(); if trimmed.is_empty() { @@ -835,6 +902,18 @@ macro_rules! build_float_array { }}; } +fn parse_time_unit(entry_ht: &ZendHashTable, name: &str) -> Result { + match entry_ht.get("unit").and_then(|z| z.str()) { + None | Some("MICROS") => Ok(TimeUnit::Microsecond), + Some("MILLIS") => Ok(TimeUnit::Millisecond), + Some("NANOS") => Ok(TimeUnit::Nanosecond), + Some(s) => Err(format!( + "Column '{}': unsupported unit '{}', expected MILLIS, MICROS or NANOS", + name, s + )), + } +} + fn php_schema_entry_to_field(entry_ht: &ZendHashTable) -> Result { let name = entry_ht .get("name") @@ -868,8 +947,15 @@ fn php_schema_entry_to_field(entry_ht: &ZendHashTable) -> Result "STRING" => DataType::Utf8, "BINARY" => DataType::Binary, "DATE" => DataType::Date32, - "TIMESTAMP" => DataType::Timestamp(TimeUnit::Microsecond, None), - "TIME" => DataType::Time64(TimeUnit::Microsecond), + "TIMESTAMP" => { + let unit = parse_time_unit(entry_ht, &name)?; + let utc = entry_ht.get("utc").and_then(|z| z.bool()).unwrap_or(true); + DataType::Timestamp(unit, if utc { Some("UTC".into()) } else { None }) + } + "TIME" => match parse_time_unit(entry_ht, &name)? { + TimeUnit::Millisecond => DataType::Time32(TimeUnit::Millisecond), + unit => DataType::Time64(unit), + }, "DECIMAL" => { let precision = entry_ht .get("precision") @@ -1166,10 +1252,16 @@ pub fn php_array_to_arrow( if zv.is_null() { builder.append_null(); } else if let Some(v) = zv.long() { - builder.append_value((v / 86400) as i32); + builder.append_value(i32::try_from(v).map_err(|_| { + format!( + "Column '{}': value {} out of range for DATE days", + column_name, v + ) + })?); } else if zv.is_object() { let (seconds, _) = extract_datetime_timestamp(zv, column_name)?; - builder.append_value((seconds / 86400) as i32); + let offset = extract_datetime_offset(zv, column_name)?; + builder.append_value((seconds + offset).div_euclid(86_400) as i32); } else { return Err(format!( "Column '{}': expected DateTimeInterface or int, got {}", @@ -1180,16 +1272,27 @@ pub fn php_array_to_arrow( } Ok(Arc::new(builder.finish()) as ArrayRef) } - DataType::Timestamp(TimeUnit::Microsecond, _) => { - let mut builder = TimestampMicrosecondBuilder::with_capacity(values.len()); + DataType::Timestamp(unit, tz) => { + let mut result: Vec> = Vec::with_capacity(values.len()); for zv in values { if zv.is_null() { - builder.append_null(); + result.push(None); } else if let Some(v) = zv.long() { - builder.append_value(v); + result.push(Some(v)); } else if zv.is_object() { let (seconds, microseconds) = extract_datetime_timestamp(zv, column_name)?; - builder.append_value(seconds * 1_000_000 + microseconds); + let total = seconds * 1_000_000 + microseconds; + result.push(Some(match unit { + TimeUnit::Second => total.div_euclid(1_000_000), + TimeUnit::Millisecond => total.div_euclid(1_000), + TimeUnit::Microsecond => total, + TimeUnit::Nanosecond => total.checked_mul(1_000).ok_or_else(|| { + format!( + "Column '{}': DateTime outside the TIMESTAMP(NANOS) range", + column_name + ) + })?, + })); } else { return Err(format!( "Column '{}': expected DateTimeInterface or int, got {}", @@ -1198,17 +1301,42 @@ pub fn php_array_to_arrow( )); } } - Ok(Arc::new(builder.finish()) as ArrayRef) + Ok(match unit { + TimeUnit::Second => { + Arc::new(TimestampSecondArray::from(result).with_timezone_opt(tz.clone())) + as ArrayRef + } + TimeUnit::Millisecond => { + Arc::new(TimestampMillisecondArray::from(result).with_timezone_opt(tz.clone())) + as ArrayRef + } + TimeUnit::Microsecond => { + Arc::new(TimestampMicrosecondArray::from(result).with_timezone_opt(tz.clone())) + as ArrayRef + } + TimeUnit::Nanosecond => { + Arc::new(TimestampNanosecondArray::from(result).with_timezone_opt(tz.clone())) + as ArrayRef + } + }) } - DataType::Time64(TimeUnit::Microsecond) => { - let mut builder = Time64MicrosecondBuilder::with_capacity(values.len()); + DataType::Time32(TimeUnit::Millisecond) => { + let mut result: Vec> = Vec::with_capacity(values.len()); for zv in values { if zv.is_null() { - builder.append_null(); + result.push(None); } else if let Some(v) = zv.long() { - builder.append_value(v); + result.push(Some(i32::try_from(v).map_err(|_| { + format!( + "Column '{}': value {} out of range for TIME(MILLIS)", + column_name, v + ) + })?)); } else if zv.is_object() { - builder.append_value(extract_date_interval_microseconds(zv, column_name)?); + result.push(Some( + extract_date_interval_microseconds(zv, column_name)?.div_euclid(1_000) + as i32, + )); } else { return Err(format!( "Column '{}': expected DateInterval or int, got {}", @@ -1217,7 +1345,35 @@ pub fn php_array_to_arrow( )); } } - Ok(Arc::new(builder.finish()) as ArrayRef) + Ok(Arc::new(Time32MillisecondArray::from(result)) as ArrayRef) + } + DataType::Time64(unit @ (TimeUnit::Microsecond | TimeUnit::Nanosecond)) => { + let mut result: Vec> = Vec::with_capacity(values.len()); + for zv in values { + if zv.is_null() { + result.push(None); + } else if let Some(v) = zv.long() { + result.push(Some(v)); + } else if zv.is_object() { + let micros = extract_date_interval_microseconds(zv, column_name)?; + result.push(Some(if *unit == TimeUnit::Nanosecond { + micros * 1_000 + } else { + micros + })); + } else { + return Err(format!( + "Column '{}': expected DateInterval or int, got {}", + column_name, + zval_type_name(zv) + )); + } + } + Ok(if *unit == TimeUnit::Nanosecond { + Arc::new(Time64NanosecondArray::from(result)) as ArrayRef + } else { + Arc::new(Time64MicrosecondArray::from(result)) as ArrayRef + }) } DataType::Decimal128(p, s) => { let mut builder = Decimal128Builder::with_capacity(values.len()); @@ -1228,7 +1384,7 @@ pub fn php_array_to_arrow( let unscaled = parse_decimal_string(str_val, *s)?; builder.append_value(unscaled); } else if let Some(v) = zv.double() { - let str_val = format!("{:.prec$}", v, prec = *s as usize); + let str_val = round_decimal_half_away(&format!("{}", v), *s); let unscaled = parse_decimal_string(&str_val, *s)?; builder.append_value(unscaled); } else if let Some(v) = zv.long() { diff --git a/src/extension/arrow-ext/tests/fixtures/decimals_fixed_len.parquet b/src/extension/arrow-ext/tests/fixtures/decimals_fixed_len.parquet new file mode 100644 index 0000000000..33508eb43c Binary files /dev/null and b/src/extension/arrow-ext/tests/fixtures/decimals_fixed_len.parquet differ diff --git a/src/extension/arrow-ext/tests/phpt/017_writer_roundtrip.phpt b/src/extension/arrow-ext/tests/phpt/017_writer_roundtrip.phpt index b47cc66171..84b221e8eb 100644 --- a/src/extension/arrow-ext/tests/phpt/017_writer_roundtrip.phpt +++ b/src/extension/arrow-ext/tests/phpt/017_writer_roundtrip.phpt @@ -65,7 +65,7 @@ $writer->writeBatch([ 'col_double' => [3.14159, -2.71828, null], 'col_string' => ['hello', 'world', null], 'col_binary' => ["\x00\x01\x02", "\xff\xfe", null], - 'col_date' => [1641600000, 0, null], + 'col_date' => [19000, 0, null], 'col_timestamp' => [1700000000000000, 0, null], 'col_decimal' => ['123.45', '-999.99', null], ]); diff --git a/src/extension/arrow-ext/tests/phpt/039_reader_pre_epoch_timestamps.phpt b/src/extension/arrow-ext/tests/phpt/039_reader_pre_epoch_timestamps.phpt new file mode 100644 index 0000000000..511e71049c --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/039_reader_pre_epoch_timestamps.phpt @@ -0,0 +1,64 @@ +--TEST-- +Reader floors pre-epoch timestamps with a sub-second fraction +--SKIPIF-- + +--FILE-- +data .= $data; + return $this; + } +} + +class TestSourceStream implements Flow\Arrow\RandomAccessFile { + private string $data; + public function __construct(string $data) { + $this->data = $data; + } + public function read(int $length, int $offset): string { + return substr($this->data, $offset, $length); + } + public function size(): ?int { + return strlen($this->data); + } +} + +$dest = new TestDestinationStream(); +$writer = new Flow\Arrow\Parquet\Writer($dest, [ + ['name' => 'ts', 'type' => 'TIMESTAMP', 'optional' => true], + ['name' => 'ts_list', 'type' => 'LIST', 'optional' => true, 'children' => [ + ['name' => 'item', 'type' => 'TIMESTAMP', 'optional' => true], + ]], +], 'UNCOMPRESSED'); +$writer->writeBatch([ + 'ts' => [-1500000, -1, 1577934245678901], + 'ts_list' => [[-1500000, -1, 1577934245678901], null, null], +]); +$writer->close(); + +$source = new TestSourceStream($dest->data); +$reader = new Flow\Arrow\Parquet\Reader($source); +$data = $reader->readRowGroup(); + +foreach ($data['ts'] as $value) { + echo $value->format('Y-m-d H:i:s.u'), "\n"; +} + +foreach ($data['ts_list'][0] as $value) { + echo $value->format('Y-m-d H:i:s.u'), "\n"; +} +?> +--EXPECT-- +1969-12-31 23:59:58.500000 +1969-12-31 23:59:59.999999 +2020-01-02 03:04:05.678901 +1969-12-31 23:59:58.500000 +1969-12-31 23:59:59.999999 +2020-01-02 03:04:05.678901 diff --git a/src/extension/arrow-ext/tests/phpt/040_reader_decimal256.phpt b/src/extension/arrow-ext/tests/phpt/040_reader_decimal256.phpt new file mode 100644 index 0000000000..183d118b01 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/040_reader_decimal256.phpt @@ -0,0 +1,51 @@ +--TEST-- +Reader converts FIXED_LEN_BYTE_ARRAY decimals of every width, Decimal256 included +--SKIPIF-- + +--FILE-- +data .= $data; + return $this; + } +} + +class TestSourceStream implements Flow\Arrow\RandomAccessFile { + private string $data; + public function __construct(string $data) { + $this->data = $data; + } + public function read(int $length, int $offset): string { + return substr($this->data, $offset, $length); + } + public function size(): ?int { + return strlen($this->data); + } +} + +$source = new TestSourceStream(file_get_contents(__DIR__ . '/../fixtures/decimals_fixed_len.parquet')); +$reader = new Flow\Arrow\Parquet\Reader($source); +$data = $reader->readRowGroup(); + +foreach ([0, 1] as $row) { + foreach (['dec9', 'dec18', 'dec38', 'dec50'] as $column) { + var_dump($data[$column][$row]); + } +} +?> +--EXPECT-- +float(12345.67) +float(1234567890123456.8) +float(1.2345678901234568E+18) +float(1.2345678901234568E+39) +float(-12345.67) +float(-0.01) +float(-12345.67) +float(-12345.67) diff --git a/src/extension/arrow-ext/tests/phpt/041_writer_date_lanes.phpt b/src/extension/arrow-ext/tests/phpt/041_writer_date_lanes.phpt new file mode 100644 index 0000000000..fd437a5352 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/041_writer_date_lanes.phpt @@ -0,0 +1,61 @@ +--TEST-- +Writer DATE int lane is days since epoch, objects write their wall-clock day +--SKIPIF-- + +--FILE-- +data .= $data; + return $this; + } +} + +class TestSourceStream implements Flow\Arrow\RandomAccessFile { + private string $data; + public function __construct(string $data) { + $this->data = $data; + } + public function read(int $length, int $offset): string { + return substr($this->data, $offset, $length); + } + public function size(): ?int { + return strlen($this->data); + } +} + +$dest = new TestDestinationStream(); +$writer = new Flow\Arrow\Parquet\Writer($dest, [ + ['name' => 'd_int', 'type' => 'DATE', 'optional' => true], + ['name' => 'd_object', 'type' => 'DATE', 'optional' => true], +], 'UNCOMPRESSED'); +$writer->writeBatch([ + 'd_int' => [18263, -1], + 'd_object' => [ + new DateTimeImmutable('2024-01-01 00:00:00', new DateTimeZone('Europe/Warsaw')), + new DateTimeImmutable('1969-12-31 12:00:00', new DateTimeZone('UTC')), + ], +]); +$writer->close(); + +$source = new TestSourceStream($dest->data); +$reader = new Flow\Arrow\Parquet\Reader($source); +$data = $reader->readRowGroup(); + +foreach (['d_int', 'd_object'] as $column) { + foreach ($data[$column] as $value) { + echo $value->format('Y-m-d'), "\n"; + } +} +?> +--EXPECT-- +2020-01-02 +1969-12-31 +2024-01-01 +1969-12-31 diff --git a/src/extension/arrow-ext/tests/phpt/042_writer_temporal_units.phpt b/src/extension/arrow-ext/tests/phpt/042_writer_temporal_units.phpt new file mode 100644 index 0000000000..863631e446 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/042_writer_temporal_units.phpt @@ -0,0 +1,74 @@ +--TEST-- +Writer honours the declared TIMESTAMP and TIME units +--SKIPIF-- + +--FILE-- +data .= $data; + return $this; + } +} + +class TestSourceStream implements Flow\Arrow\RandomAccessFile { + private string $data; + public function __construct(string $data) { + $this->data = $data; + } + public function read(int $length, int $offset): string { + return substr($this->data, $offset, $length); + } + public function size(): ?int { + return strlen($this->data); + } +} + +$time = new DateInterval('PT3H4M5S'); +$time->f = 0.678901; + +$dest = new TestDestinationStream(); +$writer = new Flow\Arrow\Parquet\Writer($dest, [ + ['name' => 'ts_ms', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'MILLIS', 'utc' => false], + ['name' => 'ts_ns', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'NANOS'], + ['name' => 't_ms', 'type' => 'TIME', 'optional' => true, 'unit' => 'MILLIS'], + ['name' => 't_ns', 'type' => 'TIME', 'optional' => true, 'unit' => 'NANOS'], +], 'UNCOMPRESSED'); +$writer->writeBatch([ + 'ts_ms' => [new DateTimeImmutable('2020-01-02 03:04:05.678901 UTC')], + 'ts_ns' => [new DateTimeImmutable('2020-01-02 03:04:05.678901 UTC')], + 't_ms' => [$time], + 't_ns' => [$time], +]); +$writer->close(); + +$source = new TestSourceStream($dest->data); +$reader = new Flow\Arrow\Parquet\Reader($source); +$data = $reader->readRowGroup(); + +echo $data['ts_ms'][0]->format('Y-m-d H:i:s.u'), "\n"; +echo $data['ts_ns'][0]->format('Y-m-d H:i:s.u'), "\n"; +echo $data['t_ms'][0]->format('%H:%I:%S.%F'), "\n"; +echo $data['t_ns'][0]->format('%H:%I:%S.%F'), "\n"; + +try { + $invalid = new TestDestinationStream(); + new Flow\Arrow\Parquet\Writer($invalid, [ + ['name' => 'ts', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'SECONDS'], + ], 'UNCOMPRESSED'); +} catch (Flow\Arrow\Parquet\Exception $e) { + echo $e->getMessage(), "\n"; +} +?> +--EXPECT-- +2020-01-02 03:04:05.678000 +2020-01-02 03:04:05.678901 +03:04:05.678000 +03:04:05.678901 +Column 'ts': unsupported unit 'SECONDS', expected MILLIS, MICROS or NANOS diff --git a/src/extension/arrow-ext/tests/phpt/043_writer_decimal_rounding.phpt b/src/extension/arrow-ext/tests/phpt/043_writer_decimal_rounding.phpt new file mode 100644 index 0000000000..af4032e9f2 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/043_writer_decimal_rounding.phpt @@ -0,0 +1,53 @@ +--TEST-- +Writer rounds float DECIMAL values half away from zero +--SKIPIF-- + +--FILE-- +data .= $data; + return $this; + } +} + +class TestSourceStream implements Flow\Arrow\RandomAccessFile { + private string $data; + public function __construct(string $data) { + $this->data = $data; + } + public function read(int $length, int $offset): string { + return substr($this->data, $offset, $length); + } + public function size(): ?int { + return strlen($this->data); + } +} + +$dest = new TestDestinationStream(); +$writer = new Flow\Arrow\Parquet\Writer($dest, [ + ['name' => 'd', 'type' => 'DECIMAL', 'optional' => true, 'precision' => 9, 'scale' => 2], +], 'UNCOMPRESSED'); +$writer->writeBatch(['d' => [0.125, 2.675, -0.125, 1.005, 9.995, -99.995]]); +$writer->close(); + +$source = new TestSourceStream($dest->data); +$reader = new Flow\Arrow\Parquet\Reader($source); + +foreach ($reader->readRowGroup()['d'] as $value) { + var_dump($value); +} +?> +--EXPECT-- +float(0.13) +float(2.68) +float(-0.13) +float(1.01) +float(10) +float(-100) diff --git a/src/extension/arrow-ext/tests/phpt/044_reader_schema_units_and_decimal256.phpt b/src/extension/arrow-ext/tests/phpt/044_reader_schema_units_and_decimal256.phpt new file mode 100644 index 0000000000..2a8e110a96 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/044_reader_schema_units_and_decimal256.phpt @@ -0,0 +1,67 @@ +--TEST-- +Reader schema describes temporal units, the UTC flag and Decimal256 columns +--SKIPIF-- + +--FILE-- +data .= $data; + return $this; + } +} + +class TestSourceStream implements Flow\Arrow\RandomAccessFile { + private string $data; + public function __construct(string $data) { + $this->data = $data; + } + public function read(int $length, int $offset): string { + return substr($this->data, $offset, $length); + } + public function size(): ?int { + return strlen($this->data); + } +} + +$dest = new TestDestinationStream(); +$writer = new Flow\Arrow\Parquet\Writer($dest, [ + ['name' => 'ts', 'type' => 'TIMESTAMP', 'optional' => true], + ['name' => 'ts_ms', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'MILLIS', 'utc' => false], + ['name' => 'ts_ns', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'NANOS'], + ['name' => 't_ms', 'type' => 'TIME', 'optional' => true, 'unit' => 'MILLIS'], + ['name' => 't_ns', 'type' => 'TIME', 'optional' => true, 'unit' => 'NANOS'], +], 'UNCOMPRESSED'); +$writer->writeBatch(['ts' => [0], 'ts_ms' => [0], 'ts_ns' => [0], 't_ms' => [0], 't_ns' => [0]]); +$writer->close(); + +$source = new TestSourceStream($dest->data); +$reader = new Flow\Arrow\Parquet\Reader($source); + +foreach ($reader->schema() as $column) { + echo $column['name'], ' ', $column['type'], ' ', $column['unit'], ' ', var_export($column['utc'] ?? null, true), "\n"; +} + +$fixture = new TestSourceStream(file_get_contents(__DIR__ . '/../fixtures/decimals_fixed_len.parquet')); +$decimals = new Flow\Arrow\Parquet\Reader($fixture); + +foreach ($decimals->schema() as $column) { + echo $column['name'], ' ', $column['type'], ' ', $column['precision'], ' ', $column['scale'], "\n"; +} +?> +--EXPECT-- +ts TIMESTAMP MICROS true +ts_ms TIMESTAMP MILLIS false +ts_ns TIMESTAMP NANOS true +t_ms TIME MILLIS NULL +t_ns TIME NANOS NULL +dec9 DECIMAL 9 2 +dec18 DECIMAL 18 2 +dec38 DECIMAL 38 10 +dec50 DECIMAL 50 10 diff --git a/src/lib/parquet/resources/python/README.md b/src/lib/parquet/resources/python/README.md index 29b14fce1a..0a7519138b 100644 --- a/src/lib/parquet/resources/python/README.md +++ b/src/lib/parquet/resources/python/README.md @@ -20,6 +20,7 @@ Once all dependencies are installed, you can run the following command to genera ```shell python generators/lists.py +python generators/logical_types.py python generators/maps.py python generators/orders.py python generators/primitives.py diff --git a/src/lib/parquet/resources/python/generators/logical_types.py b/src/lib/parquet/resources/python/generators/logical_types.py new file mode 100644 index 0000000000..bcf11ad54e --- /dev/null +++ b/src/lib/parquet/resources/python/generators/logical_types.py @@ -0,0 +1,35 @@ +import datetime as dt +from decimal import Decimal as D + +import pyarrow as pa +import pyarrow.parquet as pq + +UTC = dt.timezone.utc + + +def ts(*parts, **kwargs): + return dt.datetime(*parts, **kwargs) + + +int_backed = pa.table({ + 'ts_ms': pa.array([ts(2020, 1, 2, 3, 4, 5, 678000), ts(1969, 12, 31, 23, 59, 58, 500000), None], type=pa.timestamp('ms')), + 'ts_us': pa.array([ts(2020, 1, 2, 3, 4, 5, 678901), ts(1900, 1, 1, 0, 0, 0, 1), None], type=pa.timestamp('us')), + 'ts_ns': pa.array([1577934245678901234, -1500, None], type=pa.timestamp('ns')), + 'ts_us_utc': pa.array([ts(2020, 1, 2, 3, 4, 5, 678901, tzinfo=UTC), ts(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), None], type=pa.timestamp('us', tz='UTC')), + 'ts_us_far': pa.array([ts(2262, 4, 11, 23, 47, 16, 854775), ts(1677, 9, 21, 0, 12, 43, 145225), None], type=pa.timestamp('us')), + 'd': pa.array([dt.date(2020, 1, 2), dt.date(1969, 12, 31), None], type=pa.date32()), + 't_ms': pa.array([dt.time(3, 4, 5, 678000), dt.time(0, 0, 0), None], type=pa.time32('ms')), + 't_us': pa.array([dt.time(3, 4, 5, 678901), dt.time(23, 59, 59, 999999), None], type=pa.time64('us')), + 't_ns': pa.array([11045678901234, 999, None], type=pa.time64('ns')), + 'dec9': pa.array([D('12345.67'), D('-12345.67'), None], type=pa.decimal128(9, 2)), + 'dec18': pa.array([D('1234567890123456.78'), D('-0.01'), None], type=pa.decimal128(18, 2)), +}) +pq.write_table(int_backed, 'output/logical_types.parquet', store_decimal_as_integer=True) + +fixed_len = pa.table({ + 'dec9': pa.array([D('12345.67'), D('-12345.67'), None], type=pa.decimal128(9, 2)), + 'dec18': pa.array([D('1234567890123456.78'), D('-0.01'), None], type=pa.decimal128(18, 2)), + 'dec38': pa.array([D('1234567890123456789.0123456789'), D('-12345.6700000000'), None], type=pa.decimal128(38, 10)), + 'dec50': pa.array([D('1234567890123456789012345678901234567890.0123456789'), D('-12345.6700000000'), None], type=pa.decimal256(50, 10)), +}) +pq.write_table(fixed_len, 'output/decimals_fixed_len.parquet') diff --git a/src/lib/parquet/src/Flow/Parquet/Binary/functions.php b/src/lib/parquet/src/Flow/Parquet/Binary/functions.php index cfe83ff05a..1b38a2d69c 100644 --- a/src/lib/parquet/src/Flow/Parquet/Binary/functions.php +++ b/src/lib/parquet/src/Flow/Parquet/Binary/functions.php @@ -4,22 +4,36 @@ namespace Flow\Parquet\Binary; +use Flow\Parquet\Exception\InvalidArgumentException; use OverflowException; use function abs; -use function array_reverse; use function array_values; +use function bcadd; use function bccomp; use function bcdiv; +use function bcmod; +use function bcmul; use function bcpow; +use function bcsub; +use function chr; use function count; +use function explode; +use function is_numeric; +use function ltrim; use function max; -use function number_format; +use function ord; use function pack; use function sprintf; +use function str_pad; use function str_repeat; +use function str_replace; +use function str_starts_with; use function strlen; +use function strpos; +use function substr; use function unpack; +use function var_export; /** * @param array $values @@ -287,71 +301,131 @@ function decode_f64(ByteOrder $order, string $bytes): array return array_values($values); } -function encode_decimal(ByteOrder $order, float $value, int $byteLength, int $precision, int $scale): string +/** + * Rounds the shortest round-trip representation of the float half away from zero - the same digits arrow-ext writes. + * + * @return numeric-string + */ +function decimal_unscaled(float $value, int $precision, int $scale): string { - $decimalInt = (int) number_format($value, $scale, '', ''); + $repr = var_export($value, true); + $parts = explode('E', $repr); + $mantissa = ltrim($parts[0], '-'); + $dot = strpos($mantissa, '.'); + $digits = str_replace('.', '', $mantissa); + $point = ($dot === false ? strlen($mantissa) : $dot) + (int) ($parts[1] ?? 0); + $scale = max(0, $scale); + + $plain = (str_starts_with($repr, '-') ? '-' : '') . match (true) { + $point <= 0 => '0.' . str_repeat('0', abs($point)) . $digits, + $point >= strlen($digits) => str_pad($digits, $point, '0'), + default => substr($digits, 0, $point) . '.' . substr($digits, $point), + }; + + if (!is_numeric($plain)) { + throw new InvalidArgumentException(sprintf('Decimal value %s is not a finite number', $repr)); + } - $maxUnscaled = bcpow('10', (string) $precision); + $half = bcdiv('5', bcpow('10', (string) ($scale + 1)), $scale + 1); + $unscaled = bcmul( + bcadd($plain, str_starts_with($plain, '-') ? bcsub('0', $half, $scale + 1) : $half, $scale), + bcpow('10', (string) $scale), + 0, + ); - if (bccomp((string) abs($decimalInt), $maxUnscaled) >= 0) { + if (strlen(ltrim($unscaled, '-')) > $precision) { throw new OverflowException(sprintf( 'Decimal value %s exceeds maximum precision of %d digits', - $value, + $unscaled, $precision, )); } - $bytes = []; + return $unscaled; +} + +/** + * @param numeric-string $unscaled + */ +function decimal_from_unscaled(string $unscaled, int $scale): float +{ + $scale = max(0, $scale); + + return (float) bcdiv($unscaled, bcpow('10', (string) $scale), $scale); +} + +/** + * Decimals are big-endian two's complement. A null byte length encodes the minimal number of bytes (BYTE_ARRAY). + */ +function encode_decimal(float $value, int $precision, int $scale, ?int $byteLength): string +{ + $unscaled = decimal_unscaled($value, $precision, $scale); + $negative = str_starts_with($unscaled, '-'); + + if (strlen(ltrim($unscaled, '-')) <= 18) { + $bytes = pack('J', (int) $unscaled); - for ($i = $byteLength - 1; $i >= 0; $i--) { - $shift = $i * 8; - $bytes[] = ($decimalInt >> $shift) & 0xFF; + if ($byteLength === null) { + $signByte = $negative ? "\xFF" : "\x00"; + + while (strlen($bytes) > 1 && $bytes[0] === $signByte && ord($bytes[1]) >= 0x80 === $negative) { + $bytes = substr($bytes, 1); + } + + return $bytes; + } + + return $byteLength <= 8 + ? substr($bytes, -$byteLength) + : str_pad($bytes, $byteLength, $negative ? "\xFF" : "\x00", STR_PAD_LEFT); } - if ($order === ByteOrder::BIG_ENDIAN) { - $bytes = array_reverse($bytes); + if ($byteLength === null) { + $byteLength = 1; + + while ( + bccomp($unscaled, bcpow('2', (string) ((8 * $byteLength) - 1))) >= 0 + || bccomp($unscaled, bcsub('0', bcpow('2', (string) ((8 * $byteLength) - 1)))) < 0 + ) { + $byteLength++; + } } - $packedBytes = ''; + $unsigned = $negative ? bcadd(bcpow('2', (string) (8 * $byteLength)), $unscaled) : $unscaled; + $bytes = ''; - foreach ($bytes as $byte) { - $packedBytes .= pack('C', $byte); + for ($i = 0; $i < $byteLength; $i++) { + $bytes = chr((int) bcmod($unsigned, '256')) . $bytes; + $unsigned = bcdiv($unsigned, '256', 0); } - return $packedBytes; + return $bytes; } -function decode_decimal(ByteOrder $order, string $bytes, int $precision, int $scale): float +function decode_decimal(string $bytes, int $scale): float { - $byteLength = strlen($bytes); - $intValue = 0; + if ($bytes === '') { + return 0.0; + } - /** @var array $byteArray */ - $byteArray = unpack('C*', $bytes); + $byteLength = strlen($bytes); + $negative = ord($bytes[0]) >= 0x80; - if ($order === ByteOrder::BIG_ENDIAN) { - $byteArray = array_values(array_reverse($byteArray)); + if ($byteLength <= 8) { + /** @var array{1: int} $unpacked */ + $unpacked = unpack('J', str_pad($bytes, 8, $negative ? "\xFF" : "\x00", STR_PAD_LEFT)); - foreach ($byteArray as $i => $byte) { - $shift = ($byteLength - $i - 1) * 8; - $intValue |= $byte << $shift; - } - } else { - foreach ($byteArray as $i => $byte) { - $shift = ($byteLength - $i) * 8; - $intValue |= $byte << $shift; - } + return decimal_from_unscaled((string) $unpacked[1], $scale); } - $maxUnscaled = bcpow('10', (string) $precision); + $unscaled = '0'; - if (bccomp((string) abs($intValue), $maxUnscaled) >= 0) { - throw new OverflowException(sprintf( - 'Decoded decimal value %d exceeds maximum precision of %d digits', - $intValue, - $precision, - )); + for ($i = 0; $i < $byteLength; $i++) { + $unscaled = bcadd(bcmul($unscaled, '256'), (string) ord($bytes[$i])); } - return (float) bcdiv((string) $intValue, bcpow('10', (string) $scale), max(0, $scale)); + return decimal_from_unscaled( + $negative ? bcsub($unscaled, bcpow('2', (string) (8 * $byteLength))) : $unscaled, + $scale, + ); } diff --git a/src/lib/parquet/src/Flow/Parquet/Data/PlainValueUnpacker.php b/src/lib/parquet/src/Flow/Parquet/Data/PlainValueUnpacker.php index a245324fdb..abf0bef1a5 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/PlainValueUnpacker.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/PlainValueUnpacker.php @@ -77,17 +77,16 @@ private function unpackBooleans(int $total): Generator } /** - * @return \Generator + * @return \Generator */ private function unpackByteArray(FlatColumn $column, int $total): Generator { + $decimal = $column->logicalType()?->decimalData(); + for ($i = 0; $i < $total; $i++) { - $length = decode_u32($this->byteOrder, $this->reader->readBytes(4))[0]; + $bytes = $this->reader->readBytes(decode_u32($this->byteOrder, $this->reader->readBytes(4))[0]); - yield match ($column->logicalType()?->name()) { - LogicalType::STRING, LogicalType::JSON, LogicalType::UUID => $this->reader->readBytes($length), - default => $this->reader->readBytes($length), - }; + yield $decimal === null ? $bytes : decode_decimal($bytes, $decimal->scale()); } } @@ -117,12 +116,7 @@ private function unpackFixedLenByteArray(FlatColumn $column, int $total): Genera $raw = $this->reader->readBytes($typeLength); yield match ($logicalType?->name()) { - LogicalType::DECIMAL => decode_decimal( - $this->byteOrder, - $raw, - $decimalData?->precision() ?? 10, - $decimalData?->scale() ?? 0, - ), + LogicalType::DECIMAL => decode_decimal($raw, $decimalData?->scale() ?? 0), LogicalType::UUID => $this->rawBytesToUuidString($raw), default => $raw, }; diff --git a/src/lib/parquet/src/Flow/Parquet/Data/PlainValuesPacker.php b/src/lib/parquet/src/Flow/Parquet/Data/PlainValuesPacker.php index 873e153ff2..f16832915e 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/PlainValuesPacker.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/PlainValuesPacker.php @@ -12,6 +12,7 @@ use RuntimeException; use function array_filter; +use function array_map; use function Flow\Parquet\Binary\encode_decimal; use function Flow\Parquet\Binary\encode_f32; use function Flow\Parquet\Binary\encode_f64; @@ -43,26 +44,13 @@ public function packValues(FlatColumn $column, array $values): void break; case PhysicalType::INT32: - switch ($column->logicalType()?->name()) { - case LogicalType::DATE: - case null: - /** @var array $values */ - $this->packInt32s($values); - - break; - } + /** @var array $values */ + $this->packInt32s($values); break; case PhysicalType::INT64: - switch ($column->logicalType()?->name()) { - case LogicalType::TIME: - case LogicalType::TIMESTAMP: - case null: - /** @var array $values */ - $this->packInt64s($values); - - break; - } + /** @var array $values */ + $this->packInt64s($values); break; case PhysicalType::FLOAT: @@ -87,8 +75,8 @@ public function packValues(FlatColumn $column, array $values): void $this->packDecimals( $values, (int) $column->typeLength(), - (int) $column->precision(), - (int) $column->scale(), + (int) $column->logicalType()?->decimalData()?->precision(), + (int) $column->logicalType()?->decimalData()?->scale(), ); break; @@ -107,6 +95,19 @@ public function packValues(FlatColumn $column, array $values): void /** @var array $values */ $this->packStrings($values); + break; + case LogicalType::DECIMAL: + $precision = (int) $column->logicalType()?->decimalData()?->precision(); + $scale = (int) $column->logicalType()?->decimalData()?->scale(); + + /** @var array $values */ + $this->packByteArrays(array_map(static fn(float|int $value): string => encode_decimal( + (float) $value, + $precision, + $scale, + null, + ), $values)); + break; default: /** @var array $values */ @@ -154,7 +155,7 @@ private function packByteArrays(array $values): void private function packDecimals(array $decimals, int $byteLength, int $precision, int $scale): void { foreach ($decimals as $decimal) { - $this->writer->append(encode_decimal($this->byteOrder, $decimal, $byteLength, $precision, $scale)); + $this->writer->append(encode_decimal($decimal, $precision, $scale, $byteLength)); } } diff --git a/src/lib/parquet/src/Flow/Parquet/Dremel/Statistics/Comparator.php b/src/lib/parquet/src/Flow/Parquet/Dremel/Statistics/Comparator.php index ac82b346fe..478aa06e14 100644 --- a/src/lib/parquet/src/Flow/Parquet/Dremel/Statistics/Comparator.php +++ b/src/lib/parquet/src/Flow/Parquet/Dremel/Statistics/Comparator.php @@ -6,6 +6,7 @@ use DateInterval; use Flow\Parquet\ParquetFile\Data\Converter\TimeConverter; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; use RuntimeException; use function gettype; @@ -29,8 +30,8 @@ public function isGreaterThan(mixed $value, mixed $nextValue): bool } if ($value instanceof DateInterval) { - $value = (new TimeConverter())->toParquetType($value); - $nextValue = (new TimeConverter())->toParquetType($nextValue); + $value = (new TimeConverter(TimeUnit::MICROSECONDS))->toParquetType($value); + $nextValue = (new TimeConverter(TimeUnit::MICROSECONDS))->toParquetType($nextValue); } if (!is_scalar($value) || !is_scalar($nextValue)) { @@ -59,8 +60,8 @@ public function isLessThan(mixed $value, mixed $nextValue): bool } if ($value instanceof DateInterval) { - $value = (new TimeConverter())->toParquetType($value); - $nextValue = (new TimeConverter())->toParquetType($nextValue); + $value = (new TimeConverter(TimeUnit::MICROSECONDS))->toParquetType($value); + $nextValue = (new TimeConverter(TimeUnit::MICROSECONDS))->toParquetType($nextValue); } if (!is_scalar($value) || !is_scalar($nextValue)) { diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SchemaConverter.php b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SchemaConverter.php index 7789e202b5..f68b013b6a 100644 --- a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SchemaConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SchemaConverter.php @@ -65,6 +65,19 @@ private static function flatColumnToExtension(FlatColumn $column): array $entry['scale'] = $column->scale(); } + $timestamp = $column->logicalType()?->timestampData(); + + if ($timestamp !== null) { + $entry['unit'] = $timestamp->unit()->value; + $entry['utc'] = $timestamp->isAdjustedToUTC(); + } + + $time = $column->logicalType()?->timeData(); + + if ($time !== null) { + $entry['unit'] = $time->unit()->value; + } + if ( $column->type() === PhysicalType::FIXED_LEN_BYTE_ARRAY && $column->logicalType()?->name() !== LogicalType::DECIMAL diff --git a/src/lib/parquet/src/Flow/Parquet/Option.php b/src/lib/parquet/src/Flow/Parquet/Option.php index ed14f64eed..5f39b0c88e 100644 --- a/src/lib/parquet/src/Flow/Parquet/Option.php +++ b/src/lib/parquet/src/Flow/Parquet/Option.php @@ -150,14 +150,6 @@ enum Option */ case PAGE_SIZE_CHECK_INTERVAL; - /** - * Since PHP does not support nanoseconds precision for DateTime objects, when this options is set to true, - * reader will round nanoseconds to microseconds. - * - * Default value is false - */ - case ROUND_NANOSECONDS; - /** * RowGroupBuilder is going to use this value to determine for how long it should keep adding rows to the buffer * before flushing it on disk. diff --git a/src/lib/parquet/src/Flow/Parquet/Options.php b/src/lib/parquet/src/Flow/Parquet/Options.php index 06441636c5..b16d2c4494 100644 --- a/src/lib/parquet/src/Flow/Parquet/Options.php +++ b/src/lib/parquet/src/Flow/Parquet/Options.php @@ -26,7 +26,6 @@ public function __construct() Option::ARROW_BATCH_SIZE->name => 1024, Option::ARROW_WRITE_BATCH_SIZE->name => 1000, Option::BYTE_ARRAY_TO_STRING->name => true, - Option::ROUND_NANOSECONDS->name => false, Option::INT_96_AS_DATETIME->name => true, Option::PAGE_MAXIMUM_ROWS_COUNT->name => 1000, Option::PAGE_SIZE_BYTES->name => SizeUnits::KiB_SIZE * 128, diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter.php index 8bf8d04fcc..63b21a050b 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter.php @@ -9,9 +9,9 @@ interface Converter { - public function fromParquetType(mixed $data): mixed; + public static function forColumn(FlatColumn $column, Options $options): ?self; - public function isFor(FlatColumn $column, Options $options): bool; + public function fromParquetType(mixed $data): mixed; public function toParquetType(mixed $data): mixed; } diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/DecimalConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/DecimalConverter.php new file mode 100644 index 0000000000..dbeff12bc4 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/DecimalConverter.php @@ -0,0 +1,42 @@ +logicalType()?->decimalData(); + + return ($column->type() === PhysicalType::INT32 || $column->type() === PhysicalType::INT64) && $decimal !== null + ? new self($decimal->precision(), $decimal->scale()) + : null; + } + + public function fromParquetType(mixed $data): float + { + /** @var int $data */ + return decimal_from_unscaled((string) $data, $this->scale); + } + + public function toParquetType(mixed $data): int + { + /** @var float|int $data */ + return (int) decimal_unscaled((float) $data, $this->precision, $this->scale); + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int32DateConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int32DateConverter.php index 4b83cda5cf..2b6a7c522d 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int32DateConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int32DateConverter.php @@ -16,32 +16,33 @@ use InvalidArgumentException; use function abs; +use function Flow\Parquet\floor_div; use function get_debug_type; use function is_int; use function sprintf; -final class Int32DateConverter implements Converter +final readonly class Int32DateConverter implements Converter { - public function fromParquetType(mixed $data): DateTimeImmutable + public static function forColumn(FlatColumn $column, Options $options): ?self { - if (!is_int($data)) { - throw new InvalidArgumentException(sprintf('Expected int, got %s', get_debug_type($data))); + if ($column->type() === PhysicalType::INT32 && $column->logicalType()?->name() === LogicalType::DATE) { + return new self(); } - return $this->numberOfDaysToDateTime($data); + if ($column->type() === PhysicalType::INT32 && $column->convertedType() === ConvertedType::DATE) { + return new self(); + } + + return null; } - public function isFor(FlatColumn $column, Options $options): bool + public function fromParquetType(mixed $data): DateTimeImmutable { - if ($column->type() === PhysicalType::INT32 && $column->logicalType()?->name() === LogicalType::DATE) { - return true; - } - - if ($column->type() === PhysicalType::INT32 && $column->convertedType() === ConvertedType::DATE) { - return true; + if (!is_int($data)) { + throw new InvalidArgumentException(sprintf('Expected int, got %s', get_debug_type($data))); } - return false; + return $this->numberOfDaysToDateTime($data); } public function toParquetType(mixed $data): int @@ -53,15 +54,7 @@ public function toParquetType(mixed $data): int )); } - return $this->dateTimeToNumberOfDays($data); - } - - private function dateTimeToNumberOfDays(DateTime|DateTimeImmutable $date): int - { - $epoch = new DateTimeImmutable('1970-01-01 00:00:00 UTC'); - $interval = $epoch->diff($date->setTime(0, 0, 0, 0)); - - return $interval->invert ? -(int) $interval->format('%a') : (int) $interval->format('%a'); + return floor_div($data->getTimestamp() + $data->getOffset(), 86_400); } private function numberOfDaysToDateTime(int $data): DateTimeImmutable diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int32DateTimeConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int32DateTimeConverter.php deleted file mode 100644 index c6a57b9745..0000000000 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int32DateTimeConverter.php +++ /dev/null @@ -1,79 +0,0 @@ -millisecondsToDateTimeImmutable($data); - } - - public function isFor(FlatColumn $column, Options $options): bool - { - if ($column->type() === PhysicalType::INT32 && $column->logicalType()?->name() === LogicalType::TIMESTAMP) { - return true; - } - - return false; - } - - public function toParquetType(mixed $data): int - { - if (!$data instanceof DateTimeInterface) { - throw new InvalidArgumentException(sprintf('Expected DateTimeInterface, got %s', get_debug_type($data))); - } - - return $this->dateTimeToMicroseconds($data); - } - - private function dateTimeToMicroseconds(DateTimeInterface $dateTime): int - { - $unixSeconds = (string) (int) $dateTime->format('U'); - $microseconds = (string) (int) (number_format((int) $dateTime->format('u') / 1000, 0, '', '') . '000'); - - return (int) bcadd(bcmul($unixSeconds, '1000000'), $microseconds); - } - - private function millisecondsToDateTimeImmutable(int $microseconds): DateTimeImmutable - { - $seconds = $microseconds / 1000000; - $fraction = str_pad((string) ($microseconds % 1000000), 6, '0', STR_PAD_LEFT); - - $dateTime = DateTimeImmutable::createFromFormat('U.u', sprintf('%d.%s', $seconds, $fraction)); - - if ($dateTime === false) { - throw new RuntimeException( - 'Failed to convert INT32 to DateTime, given microseconds: ' - . json_encode(['microseconds' => $microseconds, 'fraction' => $fraction], JSON_THROW_ON_ERROR), - ); - } - - return $dateTime; - } -} diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int64DateTimeConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int64DateTimeConverter.php index a654069e3b..54743ba622 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int64DateTimeConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int64DateTimeConverter.php @@ -5,34 +5,74 @@ namespace Flow\Parquet\ParquetFile\Data\Converter; use DateTimeImmutable; +use DateTimeInterface; +use Flow\Parquet\Exception\InvalidArgumentException; +use Flow\Parquet\Exception\RuntimeException; use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Data\Converter; use Flow\Parquet\ParquetFile\Schema\FlatColumn; -use Flow\Parquet\ParquetFile\Schema\LogicalType; use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; -use function number_format; +use function Flow\Parquet\floor_div; +use function intdiv; +use function sprintf; -final class Int64DateTimeConverter implements Converter +final readonly class Int64DateTimeConverter implements Converter { - public function fromParquetType(mixed $data): DateTimeImmutable + public function __construct( + private TimeUnit $unit, + ) {} + + public static function forColumn(FlatColumn $column, Options $options): ?self { - /** @var int $data */ - return new DateTimeImmutable('@' . number_format($data / 1_000_000, 6, '.', '')); + $timestamp = $column->logicalType()?->timestampData(); + + return $column->type() === PhysicalType::INT64 && $timestamp !== null ? new self($timestamp->unit()) : null; } - public function isFor(FlatColumn $column, Options $options): bool + public function fromParquetType(mixed $data): DateTimeImmutable { - if ($column->type() === PhysicalType::INT64 && $column->logicalType()?->name() === LogicalType::TIMESTAMP) { - return true; + /** @var int $data */ + $totalMicros = match ($this->unit) { + TimeUnit::MILLISECONDS => $data * 1_000, + TimeUnit::MICROSECONDS => $data, + TimeUnit::NANOSECONDS => floor_div($data, 1_000), + }; + $seconds = floor_div($totalMicros, 1_000_000); + + $dateTime = DateTimeImmutable::createFromFormat('U.u', sprintf( + '%d.%06d', + $seconds, + $totalMicros - ($seconds * 1_000_000), + )); + + if ($dateTime === false) { + throw new RuntimeException('Failed to convert INT64 to DateTime, given: ' . $data); } - return false; + return $dateTime; } public function toParquetType(mixed $data): int { - /** @var \DateTimeInterface $data */ - return ($data->getTimestamp() * 1_000_000) + (int) $data->format('u'); + /** @var DateTimeInterface $data */ + $totalMicros = ($data->getTimestamp() * 1_000_000) + (int) $data->format('u'); + + if ( + $this->unit === TimeUnit::NANOSECONDS + && ($totalMicros > intdiv(PHP_INT_MAX, 1_000) || $totalMicros < intdiv(PHP_INT_MIN, 1_000)) + ) { + throw new InvalidArgumentException(sprintf( + 'DateTime %s is outside the TIMESTAMP(NANOS) range 1677-09-21 – 2262-04-11', + $data->format(DATE_ATOM), + )); + } + + return match ($this->unit) { + TimeUnit::MILLISECONDS => floor_div($totalMicros, 1_000), + TimeUnit::MICROSECONDS => $totalMicros, + TimeUnit::NANOSECONDS => $totalMicros * 1_000, + }; } } diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int96DateTimeConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int96DateTimeConverter.php index bfd29e7829..2ecd509d5e 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int96DateTimeConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int96DateTimeConverter.php @@ -19,21 +19,17 @@ use function sprintf; use function unpack; -final class Int96DateTimeConverter implements Converter +final readonly class Int96DateTimeConverter implements Converter { - public function fromParquetType(mixed $data): DateTimeImmutable + public static function forColumn(FlatColumn $column, Options $options): ?self { - /** @var string $data */ - return $this->convertRawBytesToDateTime($data); + return $column->type() === PhysicalType::INT96 && $options->get(Option::INT_96_AS_DATETIME) ? new self() : null; } - public function isFor(FlatColumn $column, Options $options): bool + public function fromParquetType(mixed $data): DateTimeImmutable { - if ($column->type() === PhysicalType::INT96 && $options->get(Option::INT_96_AS_DATETIME)) { - return true; - } - - return false; + /** @var string $data */ + return $this->convertRawBytesToDateTime($data); } /** diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/JsonConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/JsonConverter.php index 1ef33c6b90..0a8f856ddc 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/JsonConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/JsonConverter.php @@ -12,8 +12,13 @@ use function is_string; -final class JsonConverter implements Converter +final readonly class JsonConverter implements Converter { + public static function forColumn(FlatColumn $column, Options $options): ?self + { + return $column->logicalType()?->name() === LogicalType::JSON ? new self() : null; + } + public function fromParquetType(mixed $data): string { if (!is_string($data)) { @@ -23,15 +28,6 @@ public function fromParquetType(mixed $data): string return $data; } - public function isFor(FlatColumn $column, Options $options): bool - { - if ($column->logicalType()?->name() === LogicalType::JSON) { - return true; - } - - return false; - } - public function toParquetType(mixed $data): string { if (!is_string($data)) { diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/TimeConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/TimeConverter.php index 1ba7f8449d..daa3f47995 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/TimeConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/TimeConverter.php @@ -11,31 +11,40 @@ use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Data\Converter; use Flow\Parquet\ParquetFile\Schema\FlatColumn; -use Flow\Parquet\ParquetFile\Schema\LogicalType; use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; +use function Flow\Parquet\floor_div; use function get_debug_type; use function is_int; use function sprintf; -final class TimeConverter implements Converter +final readonly class TimeConverter implements Converter { - public function fromParquetType(mixed $data): DateInterval + public function __construct( + private TimeUnit $unit, + ) {} + + public static function forColumn(FlatColumn $column, Options $options): ?self { - if (!is_int($data)) { - throw new InvalidArgumentException(sprintf('Expected int, got %s', get_debug_type($data))); - } + $time = $column->logicalType()?->timeData(); - return $this->toDateInterval($data); + return ($column->type() === PhysicalType::INT32 || $column->type() === PhysicalType::INT64) && $time !== null + ? new self($time->unit()) + : null; } - public function isFor(FlatColumn $column, Options $options): bool + public function fromParquetType(mixed $data): DateInterval { - if ($column->type() === PhysicalType::INT64 && $column->logicalType()?->name() === LogicalType::TIME) { - return true; + if (!is_int($data)) { + throw new InvalidArgumentException(sprintf('Expected int, got %s', get_debug_type($data))); } - return false; + return $this->toDateInterval(match ($this->unit) { + TimeUnit::MILLISECONDS => $data * 1_000, + TimeUnit::MICROSECONDS => $data, + TimeUnit::NANOSECONDS => floor_div($data, 1_000), + }); } public function toParquetType(mixed $data): int @@ -44,7 +53,13 @@ public function toParquetType(mixed $data): int throw new InvalidArgumentException(sprintf('Expected DateInterval, got %s', get_debug_type($data))); } - return $this->toInt($data); + $micros = $this->toInt($data); + + return match ($this->unit) { + TimeUnit::MILLISECONDS => floor_div($micros, 1_000), + TimeUnit::MICROSECONDS => $micros, + TimeUnit::NANOSECONDS => $micros * 1_000, + }; } private function toDateInterval(int $microseconds): DateInterval diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/UuidConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/UuidConverter.php index 6e97fe531e..2397bce772 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/UuidConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/UuidConverter.php @@ -17,8 +17,13 @@ use function method_exists; use function sprintf; -final class UuidConverter implements Converter +final readonly class UuidConverter implements Converter { + public static function forColumn(FlatColumn $column, Options $options): ?self + { + return $column->logicalType()?->name() === LogicalType::UUID ? new self() : null; + } + public function fromParquetType(mixed $data): string { if (!is_string($data)) { @@ -28,15 +33,6 @@ public function fromParquetType(mixed $data): string return $data; } - public function isFor(FlatColumn $column, Options $options): bool - { - if ($column->logicalType()?->name() === LogicalType::UUID) { - return true; - } - - return false; - } - public function toParquetType(mixed $data): string { if (is_string($data)) { diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/DataConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/DataConverter.php index 70ed4da867..0351230a1f 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/DataConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/DataConverter.php @@ -6,8 +6,8 @@ use Flow\Parquet\Exception\DataConversionException; use Flow\Parquet\Options; +use Flow\Parquet\ParquetFile\Data\Converter\DecimalConverter; use Flow\Parquet\ParquetFile\Data\Converter\Int32DateConverter; -use Flow\Parquet\ParquetFile\Data\Converter\Int32DateTimeConverter; use Flow\Parquet\ParquetFile\Data\Converter\Int64DateTimeConverter; use Flow\Parquet\ParquetFile\Data\Converter\Int96DateTimeConverter; use Flow\Parquet\ParquetFile\Data\Converter\JsonConverter; @@ -26,7 +26,7 @@ final class DataConverter private array $cache; /** - * @param array $converters + * @param list> $converters */ public function __construct( private readonly array $converters, @@ -38,13 +38,13 @@ public function __construct( public static function initialize(Options $options): self { return new self([ - new TimeConverter(), - new Int32DateConverter(), - new Int32DateTimeConverter(), - new Int64DateTimeConverter(), - new Int96DateTimeConverter(), - new UuidConverter(), - new JsonConverter(), + TimeConverter::class, + Int32DateConverter::class, + Int64DateTimeConverter::class, + Int96DateTimeConverter::class, + DecimalConverter::class, + UuidConverter::class, + JsonConverter::class, ], $options); } @@ -54,35 +54,21 @@ public function fromParquetType(FlatColumn $column, mixed $data): mixed return null; } - $flatPath = $column->flatPath(); + $converter = $this->resolveConverter($column); - if (array_key_exists($flatPath, $this->cache)) { - if ($this->cache[$flatPath] === null) { - return $data; - } - - return $this->cache[$flatPath]->fromParquetType($data); + if ($converter === null) { + return $data; } - foreach ($this->converters as $converter) { - if ($converter->isFor($column, $this->options)) { - $this->cache[$flatPath] = $converter; - - try { - return $converter->fromParquetType($data); - } catch (Throwable $e) { - throw new DataConversionException( - "Failed to convert data from parquet type for column '{$flatPath}'. {$e->getMessage()}", - 0, - $e, - ); - } - } + try { + return $converter->fromParquetType($data); + } catch (Throwable $e) { + throw new DataConversionException( + "Failed to convert data from parquet type for column '{$column->flatPath()}'. {$e->getMessage()}", + 0, + $e, + ); } - - $this->cache[$flatPath] = null; - - return $data; } public function resolveConverter(FlatColumn $column): ?Converter @@ -93,17 +79,16 @@ public function resolveConverter(FlatColumn $column): ?Converter return $this->cache[$flatPath]; } - foreach ($this->converters as $converter) { - if ($converter->isFor($column, $this->options)) { - $this->cache[$flatPath] = $converter; + foreach ($this->converters as $class) { + // @mago-ignore analysis:possibly-static-access-on-interface + $converter = $class::forColumn($column, $this->options); - return $converter; + if ($converter !== null) { + return $this->cache[$flatPath] = $converter; } } - $this->cache[$flatPath] = null; - - return null; + return $this->cache[$flatPath] = null; } public function toParquetType(FlatColumn $column, mixed $data): mixed @@ -112,26 +97,8 @@ public function toParquetType(FlatColumn $column, mixed $data): mixed return null; } - $flatPath = $column->flatPath(); - - if (array_key_exists($flatPath, $this->cache)) { - if ($this->cache[$flatPath] === null) { - return $data; - } - - return $this->cache[$flatPath]->toParquetType($data); - } - - foreach ($this->converters as $converter) { - if ($converter->isFor($column, $this->options)) { - $this->cache[$flatPath] = $converter; - - return $converter->toParquetType($data); - } - } - - $this->cache[$flatPath] = null; + $converter = $this->resolveConverter($column); - return $data; + return $converter === null ? $data : $converter->toParquetType($data); } } diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php index 8c9272afa8..be2660f665 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php @@ -132,24 +132,32 @@ public static function float(string $name, Repetition $repetition = Repetition:: public static function fromThrift(SchemaElement $thrift): self { + // @mago-ignore analysis:redundant-comparison + // @mago-ignore analysis:impossible-condition + $convertedType = $thrift->converted_type === null ? null : ConvertedType::from($thrift->converted_type); + // @mago-ignore analysis:redundant-comparison + // @mago-ignore analysis:redundant-condition + $precision = $thrift->precision !== null ? (int) $thrift->precision : null; + // @mago-ignore analysis:redundant-comparison + // @mago-ignore analysis:redundant-condition + $scale = $thrift->scale !== null ? (int) $thrift->scale : null; + // @mago-ignore analysis:redundant-comparison + // @mago-ignore analysis:impossible-condition + $logicalType = $thrift->logicalType === null ? null : LogicalType::fromThrift($thrift->logicalType); + return new self( $thrift->name, PhysicalType::from($thrift->type), + $convertedType, + $logicalType // @mago-ignore analysis:redundant-comparison // @mago-ignore analysis:impossible-condition - $thrift->converted_type === null ? null : ConvertedType::from($thrift->converted_type), - // @mago-ignore analysis:redundant-comparison - // @mago-ignore analysis:impossible-condition - $thrift->logicalType === null ? null : LogicalType::fromThrift($thrift->logicalType), + ?? ($convertedType === null ? null : LogicalType::fromConvertedType($convertedType, $scale, $precision)), // @mago-ignore analysis:redundant-comparison // @mago-ignore analysis:impossible-condition $thrift->repetition_type === null ? null : Repetition::from($thrift->repetition_type), - // @mago-ignore analysis:redundant-comparison - // @mago-ignore analysis:redundant-condition - $thrift->precision !== null ? (int) $thrift->precision : null, - // @mago-ignore analysis:redundant-comparison - // @mago-ignore analysis:redundant-condition - $thrift->scale !== null ? (int) $thrift->scale : null, + $precision, + $scale, // @mago-ignore analysis:redundant-comparison // @mago-ignore analysis:redundant-condition $thrift->type_length !== null ? (int) $thrift->type_length : null, diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php index 67da4ef942..61dcef2228 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php @@ -24,7 +24,7 @@ use Flow\Parquet\ThriftModel\StringType; use Flow\Parquet\ThriftModel\TimestampType; use Flow\Parquet\ThriftModel\TimeType; -use Flow\Parquet\ThriftModel\TimeUnit; +use Flow\Parquet\ThriftModel\TimeUnit as ThriftTimeUnit; use Flow\Parquet\ThriftModel\UUIDType; final readonly class LogicalType @@ -84,6 +84,25 @@ public static function enum(): self return new self(self::ENUM); } + public static function fromConvertedType(ConvertedType $convertedType, ?int $scale, ?int $precision): ?self + { + return match ($convertedType) { + ConvertedType::DATE => self::date(), + ConvertedType::DECIMAL => self::decimal($scale ?? 0, $precision ?? 0), + ConvertedType::TIME_MILLIS => new self(self::TIME, time: new Time(true, TimeUnit::MILLISECONDS)), + ConvertedType::TIME_MICROS => new self(self::TIME, time: new Time(true, TimeUnit::MICROSECONDS)), + ConvertedType::TIMESTAMP_MILLIS => new self( + self::TIMESTAMP, + timestamp: new Timestamp(true, TimeUnit::MILLISECONDS), + ), + ConvertedType::TIMESTAMP_MICROS => new self( + self::TIMESTAMP, + timestamp: new Timestamp(true, TimeUnit::MICROSECONDS), + ), + default => null, + }; + } + public static function fromThrift(ThriftLogicalType $logicalType): ?self { $name = null; @@ -219,12 +238,12 @@ public static function string(): self public static function time(): self { - return new self(self::TIME, time: new Time(false, false, true, false)); + return new self(self::TIME, time: new Time(false, TimeUnit::MICROSECONDS)); } public static function timestamp(): self { - return new self(self::TIMESTAMP, timestamp: new Timestamp(false, false, true, false)); + return new self(self::TIMESTAMP, timestamp: new Timestamp(true, TimeUnit::MICROSECONDS)); } public static function unknown(): self @@ -281,7 +300,7 @@ public function toThrift(): ThriftLogicalType self::TIME => $this->is(self::TIME) ? new TimeType([ 'isAdjustedToUTC' => $this->timeData()?->isAdjustedToUTC(), - 'unit' => new TimeUnit([ + 'unit' => new ThriftTimeUnit([ 'MILLIS' => $this->timeData()?->millis() ? new MilliSeconds() : null, 'MICROS' => $this->timeData()?->micros() ? new MicroSeconds() : null, 'NANOS' => $this->timeData()?->nanos() ? new NanoSeconds() : null, @@ -290,7 +309,7 @@ public function toThrift(): ThriftLogicalType self::TIMESTAMP => $this->is(self::TIMESTAMP) ? new TimestampType([ 'isAdjustedToUTC' => $this->timestampData()?->isAdjustedToUTC(), - 'unit' => new TimeUnit([ + 'unit' => new ThriftTimeUnit([ 'MILLIS' => $this->timestampData()?->millis() ? new MilliSeconds() : null, 'MICROS' => $this->timestampData()?->micros() ? new MicroSeconds() : null, 'NANOS' => $this->timestampData()?->nanos() ? new NanoSeconds() : null, diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Time.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Time.php index 2ea61f73bc..2178802cb9 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Time.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Time.php @@ -4,28 +4,19 @@ namespace Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; use Flow\Parquet\ThriftModel\TimeType; final readonly class Time { public function __construct( private bool $isAdjustedToUTC, - private bool $millis, - private bool $micros, - private bool $nanos, + private TimeUnit $unit, ) {} - public static function fromThrift(TimeType $timestamp): self + public static function fromThrift(TimeType $time): self { - return new self( - $timestamp->isAdjustedToUTC, - // @mago-ignore analysis:redundant-comparison - $timestamp->unit->MILLIS !== null, - // @mago-ignore analysis:redundant-comparison - $timestamp->unit->MICROS !== null, - // @mago-ignore analysis:redundant-comparison - $timestamp->unit->NANOS !== null, - ); + return new self($time->isAdjustedToUTC, TimeUnit::fromThrift($time->unit)); } public function isAdjustedToUTC(): bool @@ -35,16 +26,21 @@ public function isAdjustedToUTC(): bool public function micros(): bool { - return $this->micros; + return $this->unit === TimeUnit::MICROSECONDS; } public function millis(): bool { - return $this->millis; + return $this->unit === TimeUnit::MILLISECONDS; } public function nanos(): bool { - return $this->nanos; + return $this->unit === TimeUnit::NANOSECONDS; + } + + public function unit(): TimeUnit + { + return $this->unit; } } diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Timestamp.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Timestamp.php index 9864c6b461..20c7fb23d7 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Timestamp.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Timestamp.php @@ -4,28 +4,19 @@ namespace Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; use Flow\Parquet\ThriftModel\TimestampType; final readonly class Timestamp { public function __construct( private bool $isAdjustedToUTC, - private bool $millis, - private bool $micros, - private bool $nanos, + private TimeUnit $unit, ) {} public static function fromThrift(TimestampType $timestamp): self { - return new self( - $timestamp->isAdjustedToUTC, - // @mago-ignore analysis:redundant-comparison - $timestamp->unit->MILLIS !== null, - // @mago-ignore analysis:redundant-comparison - $timestamp->unit->MICROS !== null, - // @mago-ignore analysis:redundant-comparison - $timestamp->unit->NANOS !== null, - ); + return new self($timestamp->isAdjustedToUTC, TimeUnit::fromThrift($timestamp->unit)); } public function isAdjustedToUTC(): bool @@ -35,16 +26,21 @@ public function isAdjustedToUTC(): bool public function micros(): bool { - return $this->micros; + return $this->unit === TimeUnit::MICROSECONDS; } public function millis(): bool { - return $this->millis; + return $this->unit === TimeUnit::MILLISECONDS; } public function nanos(): bool { - return $this->nanos; + return $this->unit === TimeUnit::NANOSECONDS; + } + + public function unit(): TimeUnit + { + return $this->unit; } } diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/TimeUnit.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/TimeUnit.php index e787c44ffe..fd3632d120 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/TimeUnit.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/TimeUnit.php @@ -4,10 +4,35 @@ namespace Flow\Parquet\ParquetFile\Schema; -enum TimeUnit +use Flow\Parquet\Exception\InvalidArgumentException; +use Flow\Parquet\ThriftModel\TimeUnit as ThriftTimeUnit; + +enum TimeUnit: string { - // case MILLISECONDS; Not Implemented yet - case MICROSECONDS; + case MILLISECONDS = 'MILLIS'; + case MICROSECONDS = 'MICROS'; + case NANOSECONDS = 'NANOS'; + + public static function fromThrift(ThriftTimeUnit $unit): self + { + // @mago-ignore analysis:redundant-condition + // @mago-ignore analysis:redundant-comparison + if ($unit->MILLIS !== null) { + return self::MILLISECONDS; + } + + // @mago-ignore analysis:redundant-condition + // @mago-ignore analysis:redundant-comparison + if ($unit->MICROS !== null) { + return self::MICROSECONDS; + } + + // @mago-ignore analysis:redundant-condition + // @mago-ignore analysis:redundant-comparison + if ($unit->NANOS !== null) { + return self::NANOSECONDS; + } - // case NANOSECONDS; PHP Does not support nanoseconds + throw new InvalidArgumentException('Unsupported time unit, expected one of MILLIS, MICROS, NANOS'); + } } diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/StatisticsCounter.php b/src/lib/parquet/src/Flow/Parquet/Writer/StatisticsCounter.php index 2e11e4b5b4..c4c25d19ae 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/StatisticsCounter.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/StatisticsCounter.php @@ -16,7 +16,7 @@ use function count; use function Flow\Parquet\array_flatten; use function is_array; -use function is_string; +use function substr; final class StatisticsCounter { @@ -180,25 +180,23 @@ public function toStatistics(): Statistics $max = $this->max(); if ($min !== null) { - if ($this->column->type() === PhysicalType::BYTE_ARRAY && is_string($min)) { - (new BinaryBufferWriter($minBuffer))->append($min); - } else { - (new PlainValuesPacker(new BinaryBufferWriter($minBuffer), $this->byteOrder))->packValues( - $this->column, - [$min], - ); - } + (new PlainValuesPacker(new BinaryBufferWriter($minBuffer), $this->byteOrder))->packValues( + $this->column, + [$min], + ); } if ($max !== null) { - if ($this->column->type() === PhysicalType::BYTE_ARRAY && is_string($max)) { - (new BinaryBufferWriter($maxBuffer))->append($max); - } else { - (new PlainValuesPacker(new BinaryBufferWriter($maxBuffer), $this->byteOrder))->packValues( - $this->column, - [$max], - ); - } + (new PlainValuesPacker(new BinaryBufferWriter($maxBuffer), $this->byteOrder))->packValues( + $this->column, + [$max], + ); + } + + if ($this->column->type() === PhysicalType::BYTE_ARRAY) { + // Statistics hold the PLAIN encoding of the value without the BYTE_ARRAY length prefix. + $minBuffer = substr($minBuffer, 4); + $maxBuffer = substr($maxBuffer, 4); } return new Statistics( diff --git a/src/lib/parquet/src/Flow/Parquet/functions.php b/src/lib/parquet/src/Flow/Parquet/functions.php index 6d2a72a689..fa8f48136a 100644 --- a/src/lib/parquet/src/Flow/Parquet/functions.php +++ b/src/lib/parquet/src/Flow/Parquet/functions.php @@ -9,6 +9,7 @@ use Generator; use function get_debug_type; +use function intdiv; use function is_array; use function is_int; use function is_string; @@ -134,3 +135,10 @@ function empty_generator(): Generator { yield from []; } + +function floor_div(int $dividend, int $divisor): int +{ + $quotient = intdiv($dividend, $divisor); + + return ($dividend % $divisor) !== 0 && $dividend < 0 !== $divisor < 0 ? $quotient - 1 : $quotient; +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/TemporalValues.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/TemporalValues.php new file mode 100644 index 0000000000..09d353cab5 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/TemporalValues.php @@ -0,0 +1,33 @@ +> $rows + * + * @return list> + */ + public static function format(iterable $rows): array + { + $formatted = []; + + foreach ($rows as $row) { + $formatted[] = array_map(static fn(mixed $value): mixed => match (true) { + $value instanceof DateTimeInterface => $value->format('Y-m-d H:i:s.u P'), + $value instanceof DateInterval => $value->format('%H:%I:%S.%F'), + default => $value, + }, $row); + } + + return $formatted; + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Binary/BinaryReaderWriterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Binary/BinaryReaderWriterTest.php index 5163584be4..772963c1bb 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Binary/BinaryReaderWriterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Binary/BinaryReaderWriterTest.php @@ -7,11 +7,11 @@ use Flow\Parquet\Binary\ByteOrder; use Flow\Parquet\BinaryReader\BinaryBufferReader; use Flow\Parquet\BinaryWriter\BinaryBufferWriter; +use Flow\Parquet\ParquetFile\Schema\FlatColumn; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use function array_values; -use function ceil; use function count; use function Flow\Parquet\Binary\decode_decimal; use function Flow\Parquet\Binary\decode_f32; @@ -25,7 +25,6 @@ use function Flow\Parquet\Binary\encode_i32; use function Flow\Parquet\Binary\encode_i64; use function Flow\Parquet\Binary\encode_u32; -use function log; use function pack; use function strlen; use function unpack; @@ -107,9 +106,7 @@ public function test_writing_and_reading_big_integers_with_functions(): void #[DataProvider('decimalProvider')] public function test_writing_and_reading_decimals_with_functions(array $decimals, int $precision, int $scale): void { - $byteOrder = ByteOrder::LITTLE_ENDIAN; - $bitsNeeded = ceil(log(10 ** $precision, 2)); - $byteLength = (int) ceil($bitsNeeded / 8); + $byteLength = (int) FlatColumn::decimal('decimal', $precision, $scale)->typeLength(); $buffer = ''; $writer = new BinaryBufferWriter($buffer); @@ -117,14 +114,14 @@ public function test_writing_and_reading_decimals_with_functions(array $decimals // @mago-ignore analysis:mixed-assignment foreach ($decimals as $decimal) { static::assertIsFloat($decimal); - $writer->append(encode_decimal($byteOrder, $decimal, $byteLength, $precision, $scale)); + $writer->append(encode_decimal($decimal, $precision, $scale, $byteLength)); } $reader = new BinaryBufferReader($buffer); $decoded = []; foreach ($decimals as $_ignored) { - $decoded[] = decode_decimal($byteOrder, $reader->readBytes($byteLength), $precision, $scale); + $decoded[] = decode_decimal($reader->readBytes($byteLength), $scale); } static::assertSame($decimals, $decoded); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/decimals_fixed_len.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/decimals_fixed_len.parquet new file mode 100644 index 0000000000..33508eb43c Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/decimals_fixed_len.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/logical_types.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/logical_types.parquet new file mode 100644 index 0000000000..49c1be10c1 Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/logical_types.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesReadingTest.php new file mode 100644 index 0000000000..06bf2f44f3 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesReadingTest.php @@ -0,0 +1,173 @@ +write( + $path, + Schema::with( + new FlatColumn( + 'ts_ms', + PhysicalType::INT64, + ConvertedType::TIMESTAMP_MILLIS, + null, + Repetition::OPTIONAL, + ), + new FlatColumn( + 'ts_us', + PhysicalType::INT64, + ConvertedType::TIMESTAMP_MICROS, + null, + Repetition::OPTIONAL, + ), + new FlatColumn('t_ms', PhysicalType::INT32, ConvertedType::TIME_MILLIS, null, Repetition::OPTIONAL), + new FlatColumn('t_us', PhysicalType::INT64, ConvertedType::TIME_MICROS, null, Repetition::OPTIONAL), + new FlatColumn('dec9', PhysicalType::INT32, ConvertedType::DECIMAL, null, Repetition::OPTIONAL, 9, 2), + ), + [[ + 'ts_ms' => 1577934245678, + 'ts_us' => 1577934245678901, + 't_ms' => 11045678, + 't_us' => 11045678901, + 'dec9' => 1234567, + ]], + ); + + $file = (new Reader(engine: $engine))->read($path); + + static::assertSame( + [[ + 'ts_ms' => '2020-01-02 03:04:05.678000 +00:00', + 'ts_us' => '2020-01-02 03:04:05.678901 +00:00', + 't_ms' => '03:04:05.678000', + 't_us' => '03:04:05.678901', + 'dec9' => 12345.67, + ]], + TemporalValues::format($file->values()), + ); + static::assertTrue($file->schema()->get('ts_ms')->logicalType()?->timestampData()?->isAdjustedToUTC()); + } + + #[DataProvider('engine_provider')] + public function test_reading_dates(ParquetEngine $engine): void + { + static::assertSame( + [ + ['d' => '2020-01-02 00:00:00.000000 +00:00'], + ['d' => '1969-12-31 00:00:00.000000 +00:00'], + ['d' => null], + ], + TemporalValues::format( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/logical_types.parquet') + ->values(['d']), + ), + ); + } + + #[DataProvider('engine_provider')] + public function test_reading_fixed_len_decimals(ParquetEngine $engine): void + { + static::assertSame( + [ + [ + 'dec9' => 12345.67, + 'dec18' => 1234567890123456.8, + 'dec38' => 1.2345678901234568E+18, + 'dec50' => 1.2345678901234568E+39, + ], + ['dec9' => -12345.67, 'dec18' => -0.01, 'dec38' => -12345.67, 'dec50' => -12345.67], + ['dec9' => null, 'dec18' => null, 'dec38' => null, 'dec50' => null], + ], + iterator_to_array( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/decimals_fixed_len.parquet') + ->values(), + ), + ); + } + + #[DataProvider('engine_provider')] + public function test_reading_int_backed_decimals(ParquetEngine $engine): void + { + static::assertSame( + [ + ['dec9' => 12345.67, 'dec18' => 1234567890123456.8], + ['dec9' => -12345.67, 'dec18' => -0.01], + ['dec9' => null, 'dec18' => null], + ], + iterator_to_array( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/logical_types.parquet') + ->values(['dec9', 'dec18']), + ), + ); + } + + #[DataProvider('engine_provider')] + public function test_reading_times_in_every_unit(ParquetEngine $engine): void + { + static::assertSame( + [ + ['t_ms' => '03:04:05.678000', 't_us' => '03:04:05.678901', 't_ns' => '03:04:05.678901'], + ['t_ms' => '00:00:00.000000', 't_us' => '23:59:59.999999', 't_ns' => '00:00:00.000000'], + ['t_ms' => null, 't_us' => null, 't_ns' => null], + ], + TemporalValues::format( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/logical_types.parquet') + ->values(['t_ms', 't_us', 't_ns']), + ), + ); + } + + #[DataProvider('engine_provider')] + public function test_reading_timestamps_in_every_unit(ParquetEngine $engine): void + { + static::assertSame( + [ + [ + 'ts_ms' => '2020-01-02 03:04:05.678000 +00:00', + 'ts_us' => '2020-01-02 03:04:05.678901 +00:00', + 'ts_ns' => '2020-01-02 03:04:05.678901 +00:00', + 'ts_us_utc' => '2020-01-02 03:04:05.678901 +00:00', + 'ts_us_far' => '2262-04-11 23:47:16.854775 +00:00', + ], + [ + 'ts_ms' => '1969-12-31 23:59:58.500000 +00:00', + 'ts_us' => '1900-01-01 00:00:00.000001 +00:00', + 'ts_ns' => '1969-12-31 23:59:59.999998 +00:00', + 'ts_us_utc' => '1969-12-31 23:59:58.500000 +00:00', + 'ts_us_far' => '1677-09-21 00:12:43.145225 +00:00', + ], + ['ts_ms' => null, 'ts_us' => null, 'ts_ns' => null, 'ts_us_utc' => null, 'ts_us_far' => null], + ], + TemporalValues::format((new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/logical_types.parquet') + ->values(['ts_ms', 'ts_us', 'ts_ns', 'ts_us_utc', 'ts_us_far'])), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesWritingTest.php new file mode 100644 index 0000000000..b69e676ae7 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesWritingTest.php @@ -0,0 +1,241 @@ +write($path, Schema::with(FlatColumn::date('d')), [ + ['d' => new DateTimeImmutable('2024-01-01 00:00:00', new DateTimeZone('Europe/Warsaw'))], + ['d' => new DateTimeImmutable('1969-12-31 12:00:00', new DateTimeZone('UTC'))], + ['d' => new DateTimeImmutable('1900-02-28 00:00:00', new DateTimeZone('UTC'))], + ]); + + static::assertSame( + [ + ['d' => '2024-01-01 00:00:00.000000 +00:00'], + ['d' => '1969-12-31 00:00:00.000000 +00:00'], + ['d' => '1900-02-28 00:00:00.000000 +00:00'], + ], + TemporalValues::format( + (new Reader(engine: $reader)) + ->read($path) + ->values(), + ), + ); + } + + #[DataProvider('engine_pair_provider')] + public function test_decimals_round_half_away_from_zero(ParquetEngine $writer, ParquetEngine $reader): void + { + $path = TestParquetFile::path($this); + + (new Writer(engine: $writer))->write($path, Schema::with(FlatColumn::decimal('d', 9, 2)), [ + ['d' => 0.125], + ['d' => 2.675], + ['d' => -0.125], + ['d' => 1.005], + ['d' => 9.995], + ['d' => -99.995], + ]); + + static::assertSame( + [['d' => 0.13], ['d' => 2.68], ['d' => -0.13], ['d' => 1.01], ['d' => 10.0], ['d' => -100.0]], + iterator_to_array( + (new Reader(engine: $reader)) + ->read($path) + ->values(), + ), + ); + } + + #[DataProvider('engine_pair_provider')] + public function test_decimals_round_trip(ParquetEngine $writer, ParquetEngine $reader): void + { + $path = TestParquetFile::path($this); + $rows = [ + ['d9' => 12345.67, 'd38' => 12345.67, 'i32' => 12345.67, 'i64' => 12345.67, 'ba' => 12345.67], + ['d9' => -12345.67, 'd38' => -12345.67, 'i32' => -12345.67, 'i64' => -12345.67, 'ba' => -12345.67], + ['d9' => null, 'd38' => 1.2345678901234568E+18, 'i32' => null, 'i64' => null, 'ba' => null], + ]; + + (new Writer(engine: $writer))->write( + $path, + Schema::with( + FlatColumn::decimal('d9', 9, 2), + FlatColumn::decimal('d38', 38, 10), + new FlatColumn( + 'i32', + PhysicalType::INT32, + null, + LogicalType::decimal(2, 9), + Repetition::OPTIONAL, + 9, + 2, + ), + new FlatColumn( + 'i64', + PhysicalType::INT64, + null, + LogicalType::decimal(2, 18), + Repetition::OPTIONAL, + 18, + 2, + ), + new FlatColumn( + 'ba', + PhysicalType::BYTE_ARRAY, + null, + LogicalType::decimal(2, 9), + Repetition::OPTIONAL, + 9, + 2, + ), + ), + $rows, + ); + + static::assertSame( + $rows, + iterator_to_array( + (new Reader(engine: $reader)) + ->read($path) + ->values(), + ), + ); + } + + #[DataProvider('engine_pair_provider')] + public function test_times_round_trip_in_every_unit(ParquetEngine $writer, ParquetEngine $reader): void + { + $path = TestParquetFile::path($this); + $time = (new DateTimeImmutable('2020-01-01 00:00:00 UTC'))->diff( + new DateTimeImmutable('2020-01-01 03:04:05.678901 UTC'), + ); + + (new Writer(engine: $writer))->write( + $path, + Schema::with( + FlatColumn::time('t'), + new FlatColumn( + 't_ms', + PhysicalType::INT32, + logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::MILLISECONDS)), + ), + new FlatColumn( + 't_ns', + PhysicalType::INT64, + logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::NANOSECONDS)), + ), + ), + [['t' => $time, 't_ms' => $time, 't_ns' => $time]], + ); + + static::assertSame( + [['t' => '03:04:05.678901', 't_ms' => '03:04:05.678000', 't_ns' => '03:04:05.678901']], + TemporalValues::format( + (new Reader(engine: $reader)) + ->read($path) + ->values(), + ), + ); + } + + #[DataProvider('engine_pair_provider')] + public function test_timestamps_round_trip_in_every_unit(ParquetEngine $writer, ParquetEngine $reader): void + { + $path = TestParquetFile::path($this); + $instant = new DateTimeImmutable('2020-01-02 04:04:05.678901 +01:00'); + $preEpoch = new DateTimeImmutable('1969-12-31 23:59:58.5 UTC'); + + (new Writer(engine: $writer))->write( + $path, + Schema::with( + FlatColumn::dateTime('ts'), + new FlatColumn( + 'ts_ms_local', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(false, TimeUnit::MILLISECONDS), + ), + ), + new FlatColumn( + 'ts_ns_utc', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(true, TimeUnit::NANOSECONDS), + ), + ), + ), + [ + ['ts' => $instant, 'ts_ms_local' => $instant, 'ts_ns_utc' => $instant], + ['ts' => $preEpoch, 'ts_ms_local' => $preEpoch, 'ts_ns_utc' => $preEpoch], + ], + ); + + static::assertSame( + [ + [ + 'ts' => '2020-01-02 03:04:05.678901 +00:00', + 'ts_ms_local' => '2020-01-02 03:04:05.678000 +00:00', + 'ts_ns_utc' => '2020-01-02 03:04:05.678901 +00:00', + ], + [ + 'ts' => '1969-12-31 23:59:58.500000 +00:00', + 'ts_ms_local' => '1969-12-31 23:59:58.500000 +00:00', + 'ts_ns_utc' => '1969-12-31 23:59:58.500000 +00:00', + ], + ], + TemporalValues::format( + (new Reader(engine: $reader)) + ->read($path) + ->values(), + ), + ); + + $schema = (new Reader()) + ->read($path) + ->metadata() + ->schema(); + + static::assertSame( + [ + 'ts' => [TimeUnit::MICROSECONDS, true], + 'ts_ms_local' => [TimeUnit::MILLISECONDS, false], + 'ts_ns_utc' => [TimeUnit::NANOSECONDS, true], + ], + array_map(static fn(string $column): array => [ + $schema->get($column)->logicalType()?->timestampData()?->unit(), + $schema->get($column)->logicalType()?->timestampData()?->isAdjustedToUTC(), + ], ['ts' => 'ts', 'ts_ms_local' => 'ts_ms_local', 'ts_ns_utc' => 'ts_ns_utc']), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ParquetIntegrationTestCase.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ParquetIntegrationTestCase.php index e3e2a7008c..b85bbc681e 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ParquetIntegrationTestCase.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ParquetIntegrationTestCase.php @@ -27,6 +27,22 @@ public static function engine_provider(): array return $engines; } + /** + * @return array + */ + public static function engine_pair_provider(): array + { + $pairs = ['php/php' => [new PhpParquetEngine(), new PhpParquetEngine()]]; + + if (extension_loaded('arrow')) { + $pairs['php/arrow'] = [new PhpParquetEngine(), new ArrowParquetEngine()]; + $pairs['arrow/php'] = [new ArrowParquetEngine(), new PhpParquetEngine()]; + $pairs['arrow/arrow'] = [new ArrowParquetEngine(), new ArrowParquetEngine()]; + } + + return $pairs; + } + protected function tearDown(): void { TestParquetFile::remove($this); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Binary/ByteConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Binary/ByteConverterTest.php index a5c284a239..a6055c1cae 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Binary/ByteConverterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Binary/ByteConverterTest.php @@ -100,44 +100,9 @@ public static function int8Provider(): array } #[DataProvider('decimalProvider')] - public function test_decimal_roundtrip_big_endian(float $value, int $byteLength, int $precision, int $scale): void + public function test_decimal_roundtrip(float $value, int $byteLength, int $precision, int $scale): void { - $encoded = encode_decimal(ByteOrder::BIG_ENDIAN, $value, $byteLength, $precision, $scale); - $decoded = decode_decimal(ByteOrder::BIG_ENDIAN, $encoded, $precision, $scale); - - static::assertSame($value, $decoded); - } - - #[DataProvider('decimalProvider')] - public function test_decimal_roundtrip_little_endian( - float $value, - int $byteLength, - int $precision, - int $scale, - ): void { - $encoded = encode_decimal(ByteOrder::LITTLE_ENDIAN, $value, $byteLength, $precision, $scale); - $decoded = decode_decimal(ByteOrder::LITTLE_ENDIAN, $encoded, $precision, $scale); - - static::assertSame($value, $decoded); - } - - public function test_decode_decimal_throws_on_precision_overflow(): void - { - $this->expectException(OverflowException::class); - $this->expectExceptionMessage('exceeds maximum precision of 3 digits'); - - $encoded = encode_decimal(ByteOrder::LITTLE_ENDIAN, 12.34, 4, 10, 2); - - decode_decimal(ByteOrder::LITTLE_ENDIAN, $encoded, 3, 2); - } - - public function test_decode_decimal_validates_precision(): void - { - $encoded = encode_decimal(ByteOrder::LITTLE_ENDIAN, 9.99, 4, 10, 2); - - $decoded = decode_decimal(ByteOrder::LITTLE_ENDIAN, $encoded, 3, 2); - - static::assertSame(9.99, $decoded); + static::assertSame($value, decode_decimal(encode_decimal($value, $precision, $scale, $byteLength), $scale)); } public function test_encode_decimal_throws_on_precision_overflow(): void @@ -145,7 +110,7 @@ public function test_encode_decimal_throws_on_precision_overflow(): void $this->expectException(OverflowException::class); $this->expectExceptionMessage('exceeds maximum precision of 10 digits'); - encode_decimal(ByteOrder::LITTLE_ENDIAN, 933162046.43, 5, 10, 2); + encode_decimal(933162046.43, 10, 2, 5); } public function test_encode_decode_f32_big_endian(): void diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Binary/DecimalFunctionsTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Binary/DecimalFunctionsTest.php new file mode 100644 index 0000000000..a79ad65114 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Binary/DecimalFunctionsTest.php @@ -0,0 +1,125 @@ + + */ + public static function round_trip_provider(): Generator + { + $lengths = [1 => [2, 0], 4 => [9, 2], 5 => [11, 2], 8 => [18, 2], 9 => [21, 2], 16 => [38, 2], 21 => [50, 2]]; + + foreach ($lengths as $length => [$precision, $scale]) { + foreach ([0.0, 1.0, -1.0, 12345.67, -12345.67, 1.2345678901234568E+18] as $value) { + if (strlen(ltrim(number_format(abs($value), $scale, '', ''), '0')) <= $precision) { + yield "{$length} bytes, {$value}" => [$value, $precision, $scale, $length]; + } + } + } + } + + #[TestWith(["\x00\x12\xD6\x87", 2, 12345.67])] + #[TestWith(["\xFF\xED\x29\x79", 2, -12345.67])] + #[TestWith(["\x12\xD6\x87", 2, 12345.67])] + #[TestWith(["\xED\x29\x79", 2, -12345.67])] + #[TestWith(['', 2, 0.0])] + public function test_decode_decimal(string $bytes, int $scale, float $expected): void + { + static::assertSame($expected, decode_decimal($bytes, $scale)); + } + + #[TestWith([12345.67, "\x00\x12\xD6\x87", "\x12\xD6\x87"])] + #[TestWith([-12345.67, "\xFF\xED\x29\x79", "\xED\x29\x79"])] + public function test_encode_decimal_bytes(float $value, string $fixedLength, string $minimal): void + { + static::assertSame($fixedLength, encode_decimal($value, 9, 2, 4)); + static::assertSame($minimal, encode_decimal($value, 9, 2, null)); + } + + public function test_encode_over_precision_throws(): void + { + $this->expectException(OverflowException::class); + $this->expectExceptionMessage('Decimal value 1234567890 exceeds maximum precision of 9 digits'); + + encode_decimal(12345678.9, 9, 2, 4); + } + + #[TestWith([0.125, '13'])] + #[TestWith([2.675, '268'])] + #[TestWith([-0.125, '-13'])] + #[TestWith([1.005, '101'])] + public function test_encode_rounds_half_away_from_zero(float $value, string $expected): void + { + static::assertSame($expected, decimal_unscaled($value, 9, 2)); + } + + #[TestWith([INF])] + #[TestWith([-INF])] + #[TestWith([NAN])] + public function test_unscaled_rejects_non_finite_values(float $value): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('is not a finite number'); + + decimal_unscaled($value, 9, 2); + } + + #[TestWith([-1234567890123456.8, 18, 2, '-123456789012345680'])] + #[TestWith([1.2345678901234568E+18, 38, 10, '12345678901234568000000000000'])] + #[TestWith([1.0E-7, 9, 8, '10'])] + #[TestWith([1.0E+25, 38, 0, '10000000000000000000000000'])] + #[TestWith([-0.5, 9, 0, '-1'])] + public function test_unscaled_is_the_shortest_round_trip_repr( + float $value, + int $precision, + int $scale, + string $expected, + ): void { + static::assertSame($expected, decimal_unscaled($value, $precision, $scale)); + } + + #[TestWith([1.2345678901234568E+39, 50, 10, 21])] + #[TestWith([-1.2345678901234568E+39, 50, 10, 21])] + #[TestWith([1.2345678901234568E+18, 38, 10, 12])] + #[TestWith([-1.2345678901234568E+18, 38, 10, 12])] + #[TestWith([-128.0, 38, 17, 9])] + public function test_round_trip_above_int64_with_minimal_length( + float $value, + int $precision, + int $scale, + int $expectedLength, + ): void { + $bytes = encode_decimal($value, $precision, $scale, null); + + static::assertSame($expectedLength, strlen($bytes)); + static::assertSame($value, decode_decimal($bytes, $scale)); + } + + #[DataProvider('round_trip_provider')] + public function test_round_trip(float $value, int $precision, int $scale, int $length): void + { + $bytes = encode_decimal($value, $precision, $scale, $length); + + static::assertSame($length, strlen($bytes)); + static::assertSame($value, decode_decimal($bytes, $scale)); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/PlainValueUnpackerTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/PlainValueUnpackerTest.php index 51dddfd853..3cccbbcd31 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/PlainValueUnpackerTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/PlainValueUnpackerTest.php @@ -137,12 +137,11 @@ public function test_unpack_double(): void public function test_unpack_fixed_len_byte_array_with_decimal_logical_type(): void { - $byteOrder = ByteOrder::LITTLE_ENDIAN; $column = FlatColumn::decimal('test_column', 10, 2); $typeLength = $column->typeLength(); static::assertNotNull($typeLength); - $buffer = encode_decimal($byteOrder, 123.45, $typeLength, 10, 2); - $buffer .= encode_decimal($byteOrder, 678.90, $typeLength, 10, 2); + $buffer = encode_decimal(123.45, 10, 2, $typeLength); + $buffer .= encode_decimal(678.90, 10, 2, $typeLength); $reader = new BinaryBufferReader($buffer); $unpacker = new PlainValueUnpacker($reader); @@ -150,6 +149,30 @@ public function test_unpack_fixed_len_byte_array_with_decimal_logical_type(): vo static::assertEquals([123.45, 678.90], iterator_to_array($unpacker->unpack($column, 2))); } + public function test_unpack_byte_array_with_decimal_logical_type(): void + { + $buffer = + encode_u32(ByteOrder::LITTLE_ENDIAN, [3]) + . "\x12\xD6\x87" + . encode_u32(ByteOrder::LITTLE_ENDIAN, [3]) + . "\xED\x29\x79"; + $column = new FlatColumn('test_column', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::decimal(2, 9)); + + $unpacker = new PlainValueUnpacker(new BinaryBufferReader($buffer)); + + static::assertSame([12345.67, -12345.67], iterator_to_array($unpacker->unpack($column, 2))); + } + + public function test_unpack_fixed_len_byte_array_with_negative_decimal(): void + { + $unpacker = new PlainValueUnpacker(new BinaryBufferReader("\xFF\xED\x29\x79")); + + static::assertSame( + [-12345.67], + iterator_to_array($unpacker->unpack(FlatColumn::decimal('test_column', 9, 2), 1)), + ); + } + public function test_unpack_fixed_len_byte_array_with_null_logical_type_returns_raw_string(): void { $data1 = str_repeat('A', 16); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/PlainValuesPackerTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/PlainValuesPackerTest.php new file mode 100644 index 0000000000..f84b3933d7 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/PlainValuesPackerTest.php @@ -0,0 +1,88 @@ +packValues($column, [12345.67, -12345.67]); + + static::assertSame( + [12345.67, -12345.67], + iterator_to_array((new PlainValueUnpacker(new BinaryBufferReader($buffer)))->unpack($column, 2)), + ); + } + + public function test_fixed_len_byte_array_decimal_above_int64_round_trip(): void + { + $column = new FlatColumn( + 'd', + PhysicalType::FIXED_LEN_BYTE_ARRAY, + null, + LogicalType::decimal(10, 50), + Repetition::OPTIONAL, + typeLength: 21, + ); + + $buffer = ''; + (new PlainValuesPacker(new BinaryBufferWriter($buffer)))->packValues($column, [ + 1.2345678901234568E+39, + -12345.67, + ]); + + static::assertSame( + [1.2345678901234568E+39, -12345.67], + iterator_to_array((new PlainValueUnpacker(new BinaryBufferReader($buffer)))->unpack($column, 2)), + ); + } + + public function test_int32_decimal_round_trip(): void + { + $column = new FlatColumn('d', PhysicalType::INT32, logicalType: LogicalType::decimal(2, 9)); + + $buffer = ''; + (new PlainValuesPacker(new BinaryBufferWriter($buffer)))->packValues($column, [1234567]); + + static::assertSame( + [1234567], + iterator_to_array((new PlainValueUnpacker(new BinaryBufferReader($buffer)))->unpack($column, 1)), + ); + } + + public function test_int32_time_round_trip(): void + { + $column = new FlatColumn( + 't', + PhysicalType::INT32, + logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::MILLISECONDS)), + ); + + $buffer = ''; + (new PlainValuesPacker(new BinaryBufferWriter($buffer)))->packValues($column, [11045678]); + + static::assertSame( + [11045678], + iterator_to_array((new PlainValueUnpacker(new BinaryBufferReader($buffer)))->unpack($column, 1)), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/SchemaConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/SchemaConverterTest.php index c077edc8a8..bad365240a 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/SchemaConverterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/SchemaConverterTest.php @@ -8,10 +8,15 @@ use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\ListElement; +use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Time; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Timestamp; use Flow\Parquet\ParquetFile\Schema\MapKey; use Flow\Parquet\ParquetFile\Schema\MapValue; use Flow\Parquet\ParquetFile\Schema\NestedColumn; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\ParquetFile\Schema\Repetition; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; use PHPUnit\Framework\TestCase; final class SchemaConverterTest extends TestCase @@ -40,6 +45,8 @@ public function test_datetime_column_to_extension(): void $result = SchemaConverter::toExtension($schema); static::assertSame('TIMESTAMP', $result[0]['type']); + static::assertSame('MICROS', $result[0]['unit']); + static::assertTrue($result[0]['utc']); } public function test_decimal_column_to_extension(): void @@ -109,6 +116,38 @@ public function test_full_schema_to_extension(): void static::assertSame('STRUCT', $result[4]['type']); } + public function test_local_millis_timestamp_column_to_extension(): void + { + $result = SchemaConverter::toExtension(Schema::with( + new FlatColumn( + 'ts', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(false, TimeUnit::MILLISECONDS), + ), + ), + )); + + static::assertSame('TIMESTAMP', $result[0]['type']); + static::assertSame('MILLIS', $result[0]['unit']); + static::assertFalse($result[0]['utc']); + } + + public function test_nanos_time_column_to_extension(): void + { + $result = SchemaConverter::toExtension(Schema::with( + new FlatColumn( + 't', + PhysicalType::INT64, + logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::NANOSECONDS)), + ), + )); + + static::assertSame('TIME', $result[0]['type']); + static::assertSame('NANOS', $result[0]['unit']); + } + public function test_int32_column_to_extension(): void { $schema = Schema::with(FlatColumn::int32('count')); @@ -218,6 +257,8 @@ public function test_time_column_to_extension(): void $result = SchemaConverter::toExtension($schema); static::assertSame('TIME', $result[0]['type']); + static::assertSame('MICROS', $result[0]['unit']); + static::assertArrayNotHasKey('utc', $result[0]); } public function test_uuid_column_to_extension(): void diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/FunctionsTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/FunctionsTest.php index d1c5c15f2f..2acddfb062 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/FunctionsTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/FunctionsTest.php @@ -4,9 +4,11 @@ namespace Flow\Parquet\Tests\Unit; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; use function Flow\Parquet\array_merge_recursive; +use function Flow\Parquet\floor_div; final class FunctionsTest extends TestCase { @@ -40,4 +42,15 @@ public function test_array_merge_recursive(): void ]), ); } + + #[TestWith([7, 2, 3])] + #[TestWith([-7, 2, -4])] + #[TestWith([-6, 2, -3])] + #[TestWith([7, -2, -4])] + #[TestWith([-1500, 1000, -2])] + #[TestWith([0, 1000, 0])] + public function test_floor_div(int $dividend, int $divisor, int $expected): void + { + static::assertSame($expected, floor_div($dividend, $divisor)); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/DecimalConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/DecimalConverterTest.php new file mode 100644 index 0000000000..412daf49b1 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/DecimalConverterTest.php @@ -0,0 +1,58 @@ +fromParquetType(1234567)); + } + + #[TestWith([2, 1234567, 12345.67])] + #[TestWith([2, -1, -0.01])] + #[TestWith([0, 42, 42.0])] + #[TestWith([2, 123456789012345678, 1234567890123456.8])] + public function test_from_parquet_type(int $scale, int $value, float $expected): void + { + static::assertSame($expected, (new DecimalConverter(18, $scale))->fromParquetType($value)); + } + + #[TestWith([9, 2, 12345.67, 1234567])] + #[TestWith([9, 2, 0.125, 13])] + public function test_to_parquet_type(int $precision, int $scale, float $value, int $expected): void + { + static::assertSame($expected, (new DecimalConverter($precision, $scale))->toParquetType($value)); + } + + public function test_to_parquet_type_over_precision_throws(): void + { + $this->expectException(OverflowException::class); + + (new DecimalConverter(9, 2))->toParquetType(12345678.9); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int32DateConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int32DateConverterTest.php index e28ce94fc9..e9f4957723 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int32DateConverterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int32DateConverterTest.php @@ -5,11 +5,13 @@ namespace Flow\Parquet\Tests\Unit\ParquetFile\Data\Converter; use DateTimeImmutable; +use DateTimeZone; use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Data\Converter\Int32DateConverter; use Flow\Parquet\ParquetFile\Schema\ConvertedType; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; final class Int32DateConverterTest extends TestCase @@ -25,11 +27,22 @@ public function test_converting_dates(): void public function test_converting_int32_with_deprecated_converted_type(): void { - $converter = new Int32DateConverter(); - - static::assertTrue($converter->isFor( + static::assertInstanceOf(Int32DateConverter::class, Int32DateConverter::forColumn( new FlatColumn('date', PhysicalType::INT32, ConvertedType::DATE, null), Options::default(), )); } + + #[TestWith(['2024-01-01 00:00:00', 'Europe/Warsaw', 19723])] + #[TestWith(['2024-01-01 23:59:59', 'America/New_York', 19723])] + #[TestWith(['1969-12-31 12:00:00', 'UTC', -1])] + #[TestWith(['1969-12-31 00:00:00', 'Asia/Tokyo', -1])] + #[TestWith(['1900-02-28 00:00:00', 'UTC', -25509])] + public function test_to_parquet_type(string $dateTime, string $timeZone, int $expected): void + { + static::assertSame( + $expected, + (new Int32DateConverter())->toParquetType(new DateTimeImmutable($dateTime, new DateTimeZone($timeZone))), + ); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int32DateTimeConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int32DateTimeConverterTest.php deleted file mode 100644 index dd8cb48e72..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int32DateTimeConverterTest.php +++ /dev/null @@ -1,24 +0,0 @@ -fromParquetType($converter->toParquetType($date)), - ); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int64DateTimeConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int64DateTimeConverterTest.php index 1ca13a1664..9c81f1e8e5 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int64DateTimeConverterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int64DateTimeConverterTest.php @@ -6,7 +6,15 @@ use DateTimeImmutable; use DateTimeZone; +use Flow\Parquet\Exception\InvalidArgumentException; +use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Data\Converter\Int64DateTimeConverter; +use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Timestamp; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; final class Int64DateTimeConverterTest extends TestCase @@ -17,8 +25,80 @@ public function test_converting_date_times(): void ->setTimezone(new DateTimeZone('UTC')) ->setTime(14, 56, 24, 54324); - $converter = new Int64DateTimeConverter(); + $converter = new Int64DateTimeConverter(TimeUnit::MICROSECONDS); static::assertEquals($date, $converter->fromParquetType($converter->toParquetType($date))); } + + public function test_for_column_reads_the_unit(): void + { + $converter = Int64DateTimeConverter::forColumn( + new FlatColumn( + 'ts', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(false, TimeUnit::MILLISECONDS), + ), + ), + Options::default(), + ); + + static::assertNotNull($converter); + static::assertSame( + '2020-01-02 03:04:05.678000', + $converter->fromParquetType(1577934245678)->format('Y-m-d H:i:s.u'), + ); + } + + public function test_for_column_rejects_int32_and_untyped_int64(): void + { + static::assertNull(Int64DateTimeConverter::forColumn( + new FlatColumn('ts', PhysicalType::INT32, logicalType: LogicalType::timestamp()), + Options::default(), + )); + static::assertNull(Int64DateTimeConverter::forColumn(FlatColumn::int64('id'), Options::default())); + } + + #[TestWith([TimeUnit::MILLISECONDS, 1577934245678, '2020-01-02 03:04:05.678000 +00:00'])] + #[TestWith([TimeUnit::MICROSECONDS, 1577934245678901, '2020-01-02 03:04:05.678901 +00:00'])] + #[TestWith([TimeUnit::NANOSECONDS, 1577934245678901234, '2020-01-02 03:04:05.678901 +00:00'])] + #[TestWith([TimeUnit::MILLISECONDS, -1500, '1969-12-31 23:59:58.500000 +00:00'])] + #[TestWith([TimeUnit::MICROSECONDS, -1, '1969-12-31 23:59:59.999999 +00:00'])] + #[TestWith([TimeUnit::NANOSECONDS, -1500, '1969-12-31 23:59:59.999998 +00:00'])] + #[TestWith([TimeUnit::MICROSECONDS, 9223372036854775, '2262-04-11 23:47:16.854775 +00:00'])] + #[TestWith([TimeUnit::MICROSECONDS, -9223372036854775, '1677-09-21 00:12:43.145225 +00:00'])] + public function test_from_parquet_type(TimeUnit $unit, int $value, string $expected): void + { + static::assertSame( + $expected, + (new Int64DateTimeConverter($unit)) + ->fromParquetType($value) + ->format('Y-m-d H:i:s.u P'), + ); + } + + #[TestWith([TimeUnit::MILLISECONDS, '2020-01-02 04:04:05.678901 +01:00', 1577934245678])] + #[TestWith([TimeUnit::MICROSECONDS, '2020-01-02 04:04:05.678901 +01:00', 1577934245678901])] + #[TestWith([TimeUnit::NANOSECONDS, '2020-01-02 04:04:05.678901 +01:00', 1577934245678901000])] + #[TestWith([TimeUnit::MILLISECONDS, '1969-12-31 23:59:58.5 UTC', -1500])] + public function test_to_parquet_type(TimeUnit $unit, string $dateTime, int $expected): void + { + static::assertSame( + $expected, + (new Int64DateTimeConverter($unit))->toParquetType(new DateTimeImmutable($dateTime)), + ); + } + + public function test_to_parquet_type_outside_nanos_range_throws(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'DateTime 2262-04-12T00:00:00+00:00 is outside the TIMESTAMP(NANOS) range 1677-09-21 – 2262-04-11', + ); + + (new Int64DateTimeConverter(TimeUnit::NANOSECONDS))->toParquetType( + new DateTimeImmutable('2262-04-12 00:00:00 UTC'), + ); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/TimeConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/TimeConverterTest.php index 2dd5b3ed92..824f7b9435 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/TimeConverterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/TimeConverterTest.php @@ -5,7 +5,14 @@ namespace Flow\Parquet\Tests\Unit\ParquetFile\Data\Converter; use DateTimeImmutable; +use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Data\Converter\TimeConverter; +use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Time; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; final class TimeConverterTest extends TestCase @@ -16,9 +23,49 @@ public function test_converting_diff_between_two_datetimes_to_time(): void $end = (new DateTimeImmutable('2023-01-01 00:00:0 UTC'))->setTime(15, 25, 30, 15040); $diff = $start->diff($end); + $converter = new TimeConverter(TimeUnit::MICROSECONDS); - $microseconds = (new TimeConverter())->toParquetType($diff); + static::assertEquals($diff, $converter->fromParquetType($converter->toParquetType($diff))); + } + + public function test_for_column_on_int32_time_millis(): void + { + $converter = TimeConverter::forColumn( + new FlatColumn( + 't', + PhysicalType::INT32, + logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::MILLISECONDS)), + ), + Options::default(), + ); + + static::assertNotNull($converter); + static::assertSame('03:04:05.678000', $converter->fromParquetType(11045678)->format('%H:%I:%S.%F')); + } + + #[TestWith([TimeUnit::MILLISECONDS, 11045678, '03:04:05.678000'])] + #[TestWith([TimeUnit::MICROSECONDS, 11045678901, '03:04:05.678901'])] + #[TestWith([TimeUnit::NANOSECONDS, 11045678901234, '03:04:05.678901'])] + #[TestWith([TimeUnit::NANOSECONDS, 999, '00:00:00.000000'])] + public function test_from_parquet_type(TimeUnit $unit, int $value, string $expected): void + { + static::assertSame( + $expected, + (new TimeConverter($unit)) + ->fromParquetType($value) + ->format('%H:%I:%S.%F'), + ); + } + + #[TestWith([TimeUnit::MILLISECONDS, 11045678])] + #[TestWith([TimeUnit::MICROSECONDS, 11045678901])] + #[TestWith([TimeUnit::NANOSECONDS, 11045678901000])] + public function test_to_parquet_type(TimeUnit $unit, int $expected): void + { + $interval = (new DateTimeImmutable('2020-01-01 00:00:00 UTC'))->diff( + new DateTimeImmutable('2020-01-01 03:04:05.678901 UTC'), + ); - static::assertEquals($diff, (new TimeConverter())->fromParquetType($microseconds)); + static::assertSame($expected, (new TimeConverter($unit))->toParquetType($interval)); } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/DataConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/DataConverterTest.php index 6486a51be7..c1fad4a288 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/DataConverterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/DataConverterTest.php @@ -4,442 +4,76 @@ namespace Flow\Parquet\Tests\Unit\ParquetFile\Data; +use DateTimeImmutable; use Flow\Parquet\Exception\DataConversionException; use Flow\Parquet\Options; -use Flow\Parquet\ParquetFile\Data\Converter; use Flow\Parquet\ParquetFile\Data\DataConverter; +use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Timestamp; use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; use PHPUnit\Framework\TestCase; -use RuntimeException; -use stdClass; final class DataConverterTest extends TestCase { - public function test_cache_behavior_between_from_and_to_parquet_type_methods(): void + public function test_column_without_logical_type_passes_values_through(): void { - $options = Options::default(); - $mockConverter = new MockConverter(true, 'converted_data'); - $dataConverter = new DataConverter([$mockConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); + $converter = DataConverter::initialize(new Options()); + $column = Schema::with(FlatColumn::int64('id'))->getFlat('id'); - static::assertSame('converted_data', $dataConverter->fromParquetType($column, 'data1')); - static::assertSame('converted_data', $dataConverter->toParquetType($column, 'data2')); - static::assertSame(1, $mockConverter->isForCallCount); // Only called once for caching - static::assertSame(1, $mockConverter->fromParquetTypeCallCount); - static::assertSame(1, $mockConverter->toParquetTypeCallCount); + static::assertNull($converter->resolveConverter($column)); + static::assertSame(42, $converter->fromParquetType($column, 42)); + static::assertSame(42, $converter->toParquetType($column, 42)); + static::assertNull($converter->fromParquetType($column, null)); + static::assertNull($converter->toParquetType($column, null)); } - public function test_complex_data_types_handling(): void + public function test_conversion_failure_is_wrapped_on_every_value(): void { - $options = Options::default(); - $complexDataConverter = new ComplexDataMockConverter(); - $dataConverter = new DataConverter([$complexDataConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); + $converter = DataConverter::initialize(new Options()); + $column = Schema::with(FlatColumn::date('d'))->getFlat('d'); - static::assertSame('complex_from_parquet', $dataConverter->fromParquetType($column, ['key' => 'value'])); - - $objectData = new stdClass(); - $objectData->property = 'value'; - static::assertSame('complex_to_parquet', $dataConverter->toParquetType($column, $objectData)); - } - - public function test_constructor_with_empty_converters_array(): void - { - $options = Options::default(); - $dataConverter = new DataConverter([], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('data', $dataConverter->fromParquetType($column, 'data')); - } - - public function test_from_parquet_type_caches_converter_result(): void - { - $options = Options::default(); - $mockConverter = new MockConverter(true, 'converted_data'); - $dataConverter = new DataConverter([$mockConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('converted_data', $dataConverter->fromParquetType($column, 'data1')); - static::assertSame('converted_data', $dataConverter->fromParquetType($column, 'data2')); - static::assertSame(1, $mockConverter->isForCallCount); - static::assertSame(2, $mockConverter->fromParquetTypeCallCount); - } - - public function test_from_parquet_type_caches_null_when_no_converter_matches(): void - { - $options = Options::default(); - $nonMatchingConverter = new MockConverter(false, 'not_used'); - $dataConverter = new DataConverter([$nonMatchingConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('data1', $dataConverter->fromParquetType($column, 'data1')); - static::assertSame('data2', $dataConverter->fromParquetType($column, 'data2')); - static::assertSame(1, $nonMatchingConverter->isForCallCount); - static::assertSame(0, $nonMatchingConverter->fromParquetTypeCallCount); - } - - public function test_from_parquet_type_preserves_original_exception_as_previous(): void - { - $options = Options::default(); - $throwingConverter = new ThrowingMockConverter(); - $dataConverter = new DataConverter([$throwingConverter], $options); - $column = new FlatColumn('test_column', PhysicalType::INT32); - - try { - $dataConverter->fromParquetType($column, 'data'); - static::fail('Expected DataConversionException to be thrown'); - } catch (DataConversionException $e) { - $previous = $e->getPrevious(); - static::assertInstanceOf(RuntimeException::class, $previous); - static::assertSame('Test exception from converter', $previous->getMessage()); - } - } - - public function test_from_parquet_type_returns_null_for_null_data(): void - { - $options = Options::default(); - $dataConverter = new DataConverter([], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertNull($dataConverter->fromParquetType($column, null)); - } - - public function test_from_parquet_type_returns_original_data_when_no_converter_matches(): void - { - $options = Options::default(); - $dataConverter = new DataConverter([], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('test_data', $dataConverter->fromParquetType($column, 'test_data')); - } - - public function test_from_parquet_type_uses_first_matching_converter(): void - { - $options = Options::default(); - $firstConverter = new MockConverter(true, 'first_converter_result'); - $secondConverter = new MockConverter(true, 'second_converter_result'); - $dataConverter = new DataConverter([$firstConverter, $secondConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('first_converter_result', $dataConverter->fromParquetType($column, 'data')); - static::assertSame(1, $firstConverter->isForCallCount); - static::assertSame(1, $firstConverter->fromParquetTypeCallCount); - static::assertSame(0, $secondConverter->isForCallCount); - static::assertSame(0, $secondConverter->fromParquetTypeCallCount); - } - - public function test_from_parquet_type_uses_matching_converter(): void - { - $options = Options::default(); - $mockConverter = new MockConverter(true, 'converted_from_parquet'); - $dataConverter = new DataConverter([$mockConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('converted_from_parquet', $dataConverter->fromParquetType($column, 'original_data')); - } - - public function test_from_parquet_type_with_different_column_paths_uses_separate_cache_entries(): void - { - $options = Options::default(); - $selectiveConverter = new SelectiveMockConverter('column1', 'converter_result'); - $dataConverter = new DataConverter([$selectiveConverter], $options); - - $column1 = new FlatColumn('column1', PhysicalType::INT32); - $column2 = new FlatColumn('column2', PhysicalType::INT64); - - static::assertSame('converter_result', $dataConverter->fromParquetType($column1, 'data')); - static::assertSame('data', $dataConverter->fromParquetType($column2, 'data')); // No converter matches column2 - static::assertSame(2, $selectiveConverter->isForCallCount); // Called for both columns - static::assertSame(1, $selectiveConverter->fromParquetTypeCallCount); // Only called for column1 - } - - public function test_from_parquet_type_with_multiple_non_matching_converters(): void - { - $options = Options::default(); - $converter1 = new MockConverter(false, 'not_used1'); - $converter2 = new MockConverter(false, 'not_used2'); - $converter3 = new MockConverter(false, 'not_used3'); - $dataConverter = new DataConverter([$converter1, $converter2, $converter3], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('original_data', $dataConverter->fromParquetType($column, 'original_data')); - static::assertSame(1, $converter1->isForCallCount); - static::assertSame(1, $converter2->isForCallCount); - static::assertSame(1, $converter3->isForCallCount); - static::assertSame(0, $converter1->fromParquetTypeCallCount); - static::assertSame(0, $converter2->fromParquetTypeCallCount); - static::assertSame(0, $converter3->fromParquetTypeCallCount); - } - - public function test_from_parquet_type_with_nested_column_path(): void - { - $options = Options::default(); - $selectiveConverter = new SelectiveMockConverter('nested.path.column', 'converted_result'); - $dataConverter = new DataConverter([$selectiveConverter], $options); - - $column = new FlatColumn('nested.path.column', PhysicalType::INT32); - - static::assertSame('converted_result', $dataConverter->fromParquetType($column, 'data')); - } - - public function test_from_parquet_type_wraps_converter_exceptions(): void - { - $options = Options::default(); - $throwingConverter = new ThrowingMockConverter(); - $dataConverter = new DataConverter([$throwingConverter], $options); - $column = new FlatColumn('test_column', PhysicalType::INT32); + static::assertEquals(new DateTimeImmutable('1970-01-02 00:00:00 UTC'), $converter->fromParquetType($column, 1)); $this->expectException(DataConversionException::class); $this->expectExceptionMessage( - "Failed to convert data from parquet type for column 'test_column'. Test exception from converter", + "Failed to convert data from parquet type for column 'd'. Expected int, got string", ); - $dataConverter->fromParquetType($column, 'data'); - } - - public function test_initialize_creates_data_converter_with_default_converters(): void - { - $options = Options::default(); - - $dataConverter = DataConverter::initialize($options); - - static::assertInstanceOf(DataConverter::class, $dataConverter); - } - - public function test_initialize_returns_new_instance_each_time(): void - { - $options = Options::default(); - - $dataConverter1 = DataConverter::initialize($options); - $dataConverter2 = DataConverter::initialize($options); - - static::assertNotSame($dataConverter1, $dataConverter2); - } - - public function test_to_parquet_type_caches_converter_result(): void - { - $options = Options::default(); - $mockConverter = new MockConverter(true, 'converted_data'); - $dataConverter = new DataConverter([$mockConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('converted_data', $dataConverter->toParquetType($column, 'data1')); - static::assertSame('converted_data', $dataConverter->toParquetType($column, 'data2')); - static::assertSame(1, $mockConverter->isForCallCount); - static::assertSame(2, $mockConverter->toParquetTypeCallCount); - } - - public function test_to_parquet_type_caches_null_when_no_converter_matches(): void - { - $options = Options::default(); - $nonMatchingConverter = new MockConverter(false, 'not_used'); - $dataConverter = new DataConverter([$nonMatchingConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('data1', $dataConverter->toParquetType($column, 'data1')); - static::assertSame('data2', $dataConverter->toParquetType($column, 'data2')); - static::assertSame(1, $nonMatchingConverter->isForCallCount); - static::assertSame(0, $nonMatchingConverter->toParquetTypeCallCount); - } - - public function test_to_parquet_type_does_not_wrap_exceptions(): void - { - $options = Options::default(); - $throwingConverter = new ThrowingMockConverter(); - $dataConverter = new DataConverter([$throwingConverter], $options); - $column = new FlatColumn('test_column', PhysicalType::INT32); - - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('Test exception from converter'); - - $dataConverter->toParquetType($column, 'data'); - } - - public function test_to_parquet_type_returns_null_for_null_data(): void - { - $options = Options::default(); - $dataConverter = new DataConverter([], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertNull($dataConverter->toParquetType($column, null)); - } - - public function test_to_parquet_type_returns_original_data_when_no_converter_matches(): void - { - $options = Options::default(); - $dataConverter = new DataConverter([], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('test_data', $dataConverter->toParquetType($column, 'test_data')); - } - - public function test_to_parquet_type_uses_first_matching_converter(): void - { - $options = Options::default(); - $firstConverter = new MockConverter(true, 'first_converter_result'); - $secondConverter = new MockConverter(true, 'second_converter_result'); - $dataConverter = new DataConverter([$firstConverter, $secondConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('first_converter_result', $dataConverter->toParquetType($column, 'data')); - static::assertSame(1, $firstConverter->isForCallCount); - static::assertSame(1, $firstConverter->toParquetTypeCallCount); - static::assertSame(0, $secondConverter->isForCallCount); - static::assertSame(0, $secondConverter->toParquetTypeCallCount); - } - - public function test_to_parquet_type_uses_matching_converter(): void - { - $options = Options::default(); - $mockConverter = new MockConverter(true, 'converted_to_parquet'); - $dataConverter = new DataConverter([$mockConverter], $options); - $column = new FlatColumn('test', PhysicalType::INT32); - - static::assertSame('converted_to_parquet', $dataConverter->toParquetType($column, 'original_data')); - } - - public function test_to_parquet_type_with_different_column_paths_uses_separate_cache_entries(): void - { - $options = Options::default(); - $selectiveConverter = new SelectiveMockConverter('column1', 'converter_result'); - $dataConverter = new DataConverter([$selectiveConverter], $options); - - $column1 = new FlatColumn('column1', PhysicalType::INT32); - $column2 = new FlatColumn('column2', PhysicalType::INT64); - - static::assertSame('converter_result', $dataConverter->toParquetType($column1, 'data')); - static::assertSame('data', $dataConverter->toParquetType($column2, 'data')); // No converter matches column2 - static::assertSame(2, $selectiveConverter->isForCallCount); // Called for both columns - static::assertSame(1, $selectiveConverter->toParquetTypeCallCount); // Only called for column1 - } - - public function test_to_parquet_type_with_nested_column_path(): void - { - $options = Options::default(); - $selectiveConverter = new SelectiveMockConverter('nested.path.column', 'converted_result'); - $dataConverter = new DataConverter([$selectiveConverter], $options); - - $column = new FlatColumn('nested.path.column', PhysicalType::INT32); - - static::assertSame('converted_result', $dataConverter->toParquetType($column, 'data')); - } -} - -/** - * Mock converter for testing purposes. - */ -final class MockConverter implements Converter -{ - public int $fromParquetTypeCallCount = 0; - - public int $isForCallCount = 0; - - public int $toParquetTypeCallCount = 0; - - public function __construct( - private readonly bool $isForResult, - private readonly mixed $conversionResult, - ) {} - - public function fromParquetType(mixed $data): mixed - { - $this->fromParquetTypeCallCount++; - - return $this->conversionResult; - } - - public function isFor(FlatColumn $column, Options $options): bool - { - $this->isForCallCount++; - - return $this->isForResult; - } - - public function toParquetType(mixed $data): mixed - { - $this->toParquetTypeCallCount++; - - return $this->conversionResult; - } -} - -/** - * Mock converter that only matches specific column names. - */ -final class SelectiveMockConverter implements Converter -{ - public int $fromParquetTypeCallCount = 0; - - public int $isForCallCount = 0; - - public int $toParquetTypeCallCount = 0; - - public function __construct( - private readonly string $matchingColumnName, - private readonly mixed $conversionResult, - ) {} - - public function fromParquetType(mixed $data): mixed - { - $this->fromParquetTypeCallCount++; - - return $this->conversionResult; - } - - public function isFor(FlatColumn $column, Options $options): bool - { - $this->isForCallCount++; - - return $column->name() === $this->matchingColumnName; - } - - public function toParquetType(mixed $data): mixed - { - $this->toParquetTypeCallCount++; - - return $this->conversionResult; - } -} - -/** - * Mock converter that handles complex data types. - */ -final class ComplexDataMockConverter implements Converter -{ - public function fromParquetType(mixed $data): mixed - { - return 'complex_from_parquet'; - } - - public function isFor(FlatColumn $column, Options $options): bool - { - return true; - } - - public function toParquetType(mixed $data): mixed - { - return 'complex_to_parquet'; - } -} - -/** - * Mock converter that throws exceptions for testing error handling. - */ -final class ThrowingMockConverter implements Converter -{ - public function fromParquetType(mixed $data): mixed - { - throw new RuntimeException('Test exception from converter'); - } - - public function isFor(FlatColumn $column, Options $options): bool - { - return true; - } + $converter->fromParquetType($column, 'not a date'); + } + + public function test_resolves_a_converter_per_column(): void + { + $converter = DataConverter::initialize(new Options()); + $schema = Schema::with( + new FlatColumn( + 'ts_ms', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(false, TimeUnit::MILLISECONDS), + ), + ), + new FlatColumn( + 'ts_ns', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(false, TimeUnit::NANOSECONDS), + ), + ), + ); - public function toParquetType(mixed $data): mixed - { - throw new RuntimeException('Test exception from converter'); + static::assertEquals( + new DateTimeImmutable('2020-01-02 03:04:05.678000 UTC'), + $converter->fromParquetType($schema->getFlat('ts_ms'), 1577934245678), + ); + static::assertEquals( + new DateTimeImmutable('2020-01-02 03:04:05.678901 UTC'), + $converter->fromParquetType($schema->getFlat('ts_ns'), 1577934245678901234), + ); } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/FlatColumnTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/FlatColumnTest.php index 538f016e75..c2b3a10840 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/FlatColumnTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/FlatColumnTest.php @@ -6,8 +6,16 @@ use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\ParquetFile\Schema; +use Flow\Parquet\ParquetFile\Schema\ConvertedType; use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\ParquetFile\Schema\Repetition; +use Flow\Parquet\ParquetFile\Schema\TimeUnit; +use Flow\Parquet\ThriftModel\LogicalType as ThriftLogicalType; +use Flow\Parquet\ThriftModel\MicroSeconds; +use Flow\Parquet\ThriftModel\SchemaElement; +use Flow\Parquet\ThriftModel\TimestampType; +use Flow\Parquet\ThriftModel\TimeUnit as ThriftTimeUnit; use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; @@ -40,6 +48,54 @@ public function test_decimal_rejects_precision_out_of_range(int $precision): voi FlatColumn::decimal('x', $precision, 2); } + public function test_from_thrift_derives_logical_type_from_converted_type(): void + { + $timestamp = FlatColumn::fromThrift(new SchemaElement([ + 'name' => 'ts', + 'type' => PhysicalType::INT64->value, + 'converted_type' => ConvertedType::TIMESTAMP_MILLIS->value, + 'repetition_type' => Repetition::OPTIONAL->value, + ])) + ->logicalType() + ?->timestampData(); + $decimal = FlatColumn::fromThrift(new SchemaElement([ + 'name' => 'dec', + 'type' => PhysicalType::INT32->value, + 'converted_type' => ConvertedType::DECIMAL->value, + 'repetition_type' => Repetition::OPTIONAL->value, + 'scale' => 2, + 'precision' => 9, + ])) + ->logicalType() + ?->decimalData(); + + static::assertSame(TimeUnit::MILLISECONDS, $timestamp?->unit()); + static::assertTrue($timestamp?->isAdjustedToUTC()); + static::assertSame(2, $decimal?->scale()); + static::assertSame(9, $decimal?->precision()); + } + + public function test_from_thrift_keeps_the_logical_type_over_the_converted_type(): void + { + $timestamp = FlatColumn::fromThrift(new SchemaElement([ + 'name' => 'ts', + 'type' => PhysicalType::INT64->value, + 'converted_type' => ConvertedType::TIMESTAMP_MICROS->value, + 'logicalType' => new ThriftLogicalType([ + 'TIMESTAMP' => new TimestampType([ + 'isAdjustedToUTC' => false, + 'unit' => new ThriftTimeUnit(['MICROS' => new MicroSeconds()]), + ]), + ]), + 'repetition_type' => Repetition::OPTIONAL->value, + ])) + ->logicalType() + ?->timestampData(); + + static::assertFalse($timestamp?->isAdjustedToUTC()); + static::assertSame(TimeUnit::MICROSECONDS, $timestamp?->unit()); + } + public function test_is_map_on_a_non_map_column(): void { static::assertFalse(FlatColumn::int32('int32')->isMap()); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/TimeTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/TimeTest.php new file mode 100644 index 0000000000..7e4e4f2437 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/TimeTest.php @@ -0,0 +1,40 @@ + true, + 'unit' => new ThriftTimeUnit(['NANOS' => new NanoSeconds()]), + ])); + + static::assertTrue($logicalType->isAdjustedToUTC()); + static::assertSame(TimeUnit::NANOSECONDS, $logicalType->unit()); + } + + #[TestWith([TimeUnit::MILLISECONDS, true, false, false])] + #[TestWith([TimeUnit::MICROSECONDS, false, true, false])] + #[TestWith([TimeUnit::NANOSECONDS, false, false, true])] + public function test_unit_flags_follow_the_unit(TimeUnit $unit, bool $millis, bool $micros, bool $nanos): void + { + $logicalType = new Time(false, $unit); + + static::assertSame($unit, $logicalType->unit()); + static::assertSame($millis, $logicalType->millis()); + static::assertSame($micros, $logicalType->micros()); + static::assertSame($nanos, $logicalType->nanos()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/TimestampTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/TimestampTest.php new file mode 100644 index 0000000000..1fab18a59d --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/TimestampTest.php @@ -0,0 +1,40 @@ + true, + 'unit' => new ThriftTimeUnit(['NANOS' => new NanoSeconds()]), + ])); + + static::assertTrue($logicalType->isAdjustedToUTC()); + static::assertSame(TimeUnit::NANOSECONDS, $logicalType->unit()); + } + + #[TestWith([TimeUnit::MILLISECONDS, true, false, false])] + #[TestWith([TimeUnit::MICROSECONDS, false, true, false])] + #[TestWith([TimeUnit::NANOSECONDS, false, false, true])] + public function test_unit_flags_follow_the_unit(TimeUnit $unit, bool $millis, bool $micros, bool $nanos): void + { + $logicalType = new Timestamp(false, $unit); + + static::assertSame($unit, $logicalType->unit()); + static::assertSame($millis, $logicalType->millis()); + static::assertSame($micros, $logicalType->micros()); + static::assertSame($nanos, $logicalType->nanos()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalTypeTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalTypeTest.php new file mode 100644 index 0000000000..592dc59318 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalTypeTest.php @@ -0,0 +1,99 @@ + + */ + public static function converted_types_provider(): Generator + { + yield 'DATE' => [ConvertedType::DATE, LogicalType::DATE, null, null, null, null]; + yield 'DECIMAL' => [ConvertedType::DECIMAL, LogicalType::DECIMAL, null, null, 2, 9]; + yield 'TIME_MILLIS' => [ + ConvertedType::TIME_MILLIS, + LogicalType::TIME, + TimeUnit::MILLISECONDS, + true, + null, + null, + ]; + yield 'TIME_MICROS' => [ + ConvertedType::TIME_MICROS, + LogicalType::TIME, + TimeUnit::MICROSECONDS, + true, + null, + null, + ]; + yield 'TIMESTAMP_MILLIS' => [ + ConvertedType::TIMESTAMP_MILLIS, + LogicalType::TIMESTAMP, + TimeUnit::MILLISECONDS, + true, + null, + null, + ]; + yield 'TIMESTAMP_MICROS' => [ + ConvertedType::TIMESTAMP_MICROS, + LogicalType::TIMESTAMP, + TimeUnit::MICROSECONDS, + true, + null, + null, + ]; + } + + #[DataProvider('converted_types_provider')] + public function test_from_converted_type( + ConvertedType $convertedType, + string $name, + ?TimeUnit $unit, + ?bool $isAdjustedToUTC, + ?int $scale, + ?int $precision, + ): void { + $logicalType = LogicalType::fromConvertedType($convertedType, 2, 9); + + static::assertNotNull($logicalType); + static::assertSame($name, $logicalType->name()); + static::assertSame($unit, $logicalType->timestampData()?->unit() ?? $logicalType->timeData()?->unit()); + static::assertSame( + $isAdjustedToUTC, + $logicalType->timestampData()?->isAdjustedToUTC() ?? $logicalType->timeData()?->isAdjustedToUTC(), + ); + static::assertSame($scale, $logicalType->decimalData()?->scale()); + static::assertSame($precision, $logicalType->decimalData()?->precision()); + } + + #[TestWith([ConvertedType::UTF8])] + #[TestWith([ConvertedType::INT_32])] + #[TestWith([ConvertedType::LIST])] + public function test_from_converted_type_without_logical_counterpart(ConvertedType $convertedType): void + { + static::assertNull(LogicalType::fromConvertedType($convertedType, null, null)); + } + + public function test_time_is_local(): void + { + static::assertFalse(LogicalType::time()->timeData()?->isAdjustedToUTC()); + static::assertSame(TimeUnit::MICROSECONDS, LogicalType::time()->timeData()?->unit()); + } + + public function test_timestamp_is_an_utc_instant(): void + { + static::assertTrue(LogicalType::timestamp()->timestampData()?->isAdjustedToUTC()); + static::assertSame(TimeUnit::MICROSECONDS, LogicalType::timestamp()->timestampData()?->unit()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/TimeUnitTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/TimeUnitTest.php new file mode 100644 index 0000000000..d7ff58fc55 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/TimeUnitTest.php @@ -0,0 +1,39 @@ + new MilliSeconds(), + 'MICROS' => new MicroSeconds(), + default => new NanoSeconds(), + }; + + static::assertSame($expected, TimeUnit::fromThrift(new ThriftTimeUnit([$field => $unit]))); + } + + public function test_from_thrift_without_unit_throws(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Unsupported time unit, expected one of MILLIS, MICROS, NANOS'); + + TimeUnit::fromThrift(new ThriftTimeUnit()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/StatisticsCounterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/StatisticsCounterTest.php index 38a85b23fd..8bc6968b19 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/StatisticsCounterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/StatisticsCounterTest.php @@ -6,6 +6,8 @@ use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\Writer\StatisticsCounter; use Generator; use PHPUnit\Framework\Attributes\DataProvider; @@ -523,6 +525,23 @@ public function test_to_statistics_encodes_byte_array_without_length_prefix(): v static::assertSame('world', $result->maxValue); } + public function test_to_statistics_encodes_byte_array_decimal_without_length_prefix(): void + { + $statistics = new StatisticsCounter( + new FlatColumn('test_column', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::decimal(2, 9)), + ); + + $statistics->add(12345.67); + $statistics->add(-12345.67); + + $result = $statistics->toStatistics(); + + static::assertSame("\xED\x29\x79", $result->min); + static::assertSame("\x12\xD6\x87", $result->max); + static::assertSame("\xED\x29\x79", $result->minValue); + static::assertSame("\x12\xD6\x87", $result->maxValue); + } + public function test_to_statistics_with_int32_encodes_with_packer(): void { $column = FlatColumn::int32('test_column');