diff --git a/cpp/src/parquet/reader_test.cc b/cpp/src/parquet/reader_test.cc index e4d2fd1d590..b5487fa4689 100644 --- a/cpp/src/parquet/reader_test.cc +++ b/cpp/src/parquet/reader_test.cc @@ -1124,6 +1124,11 @@ class TestJSONWithLocalFile : public ::testing::Test { }; TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) { + // simdjson 5.0 changed the format of fractured_json + if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) { + GTEST_SKIP() << "Test requires simdjson >= 5"; + } + std::string json_output = R"###({ "FileName": "nested_lists.snappy.parquet", "Version": "1.0", @@ -1138,15 +1143,9 @@ TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) { "Name": "a.list.element.list.element.list.element", "PhysicalType": "BYTE_ARRAY", "ConvertedType": "UTF8", - "LogicalType": { "Type": "String" } + "LogicalType": {"Type": "String"} }, - { - "Id": "1", - "Name": "b", - "PhysicalType": "INT32", - "ConvertedType": "NONE", - "LogicalType": { "Type": "None" } - } + { "Id": "1", "Name": "b", "PhysicalType": "INT32", "ConvertedType": "NONE", "LogicalType": {"Type": "None"} } ], "RowGroups": [ { @@ -1191,6 +1190,11 @@ TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) { } TEST_F(TestJSONWithLocalFile, JSONOutput) { + // simdjson 5.0 changed the format of fractured_json + if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) { + GTEST_SKIP() << "Test requires simdjson >= 5"; + } + std::string json_output = R"###({ "FileName": "alltypes_plain.parquet", "Version": "1.0", @@ -1200,17 +1204,77 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) { "NumberOfRealColumns": "11", "NumberOfColumns": "11", "Columns": [ - { "ConvertedType": "NONE", "Id": "0" , "LogicalType": { "Type": "None" }, "Name": "id" , "PhysicalType": "INT32" }, - { "ConvertedType": "NONE", "Id": "1" , "LogicalType": { "Type": "None" }, "Name": "bool_col" , "PhysicalType": "BOOLEAN" }, - { "ConvertedType": "NONE", "Id": "2" , "LogicalType": { "Type": "None" }, "Name": "tinyint_col" , "PhysicalType": "INT32" }, - { "ConvertedType": "NONE", "Id": "3" , "LogicalType": { "Type": "None" }, "Name": "smallint_col" , "PhysicalType": "INT32" }, - { "ConvertedType": "NONE", "Id": "4" , "LogicalType": { "Type": "None" }, "Name": "int_col" , "PhysicalType": "INT32" }, - { "ConvertedType": "NONE", "Id": "5" , "LogicalType": { "Type": "None" }, "Name": "bigint_col" , "PhysicalType": "INT64" }, - { "ConvertedType": "NONE", "Id": "6" , "LogicalType": { "Type": "None" }, "Name": "float_col" , "PhysicalType": "FLOAT" }, - { "ConvertedType": "NONE", "Id": "7" , "LogicalType": { "Type": "None" }, "Name": "double_col" , "PhysicalType": "DOUBLE" }, - { "ConvertedType": "NONE", "Id": "8" , "LogicalType": { "Type": "None" }, "Name": "date_string_col", "PhysicalType": "BYTE_ARRAY" }, - { "ConvertedType": "NONE", "Id": "9" , "LogicalType": { "Type": "None" }, "Name": "string_col" , "PhysicalType": "BYTE_ARRAY" }, - { "ConvertedType": "NONE", "Id": "10", "LogicalType": { "Type": "None" }, "Name": "timestamp_col" , "PhysicalType": "INT96" } + { "Id": "0", "Name": "id", "PhysicalType": "INT32", "ConvertedType": "NONE", "LogicalType": {"Type": "None"} }, + { + "Id": "1", + "Name": "bool_col", + "PhysicalType": "BOOLEAN", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "2", + "Name": "tinyint_col", + "PhysicalType": "INT32", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "3", + "Name": "smallint_col", + "PhysicalType": "INT32", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "4", + "Name": "int_col", + "PhysicalType": "INT32", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "5", + "Name": "bigint_col", + "PhysicalType": "INT64", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "6", + "Name": "float_col", + "PhysicalType": "FLOAT", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "7", + "Name": "double_col", + "PhysicalType": "DOUBLE", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "8", + "Name": "date_string_col", + "PhysicalType": "BYTE_ARRAY", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "9", + "Name": "string_col", + "PhysicalType": "BYTE_ARRAY", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + }, + { + "Id": "10", + "Name": "timestamp_col", + "PhysicalType": "INT96", + "ConvertedType": "NONE", + "LogicalType": {"Type": "None"} + } ], "RowGroups": [ { @@ -1219,17 +1283,105 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) { "TotalCompressedBytes": "0", "Rows": "8", "ColumnChunks": [ - { "CompressedSize": "73" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "0" , "StatsSet": "False", "UncompressedSize": "73" , "Values": "8" }, - { "CompressedSize": "24" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "1" , "StatsSet": "False", "UncompressedSize": "24" , "Values": "8" }, - { "CompressedSize": "47" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "2" , "StatsSet": "False", "UncompressedSize": "47" , "Values": "8" }, - { "CompressedSize": "47" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "3" , "StatsSet": "False", "UncompressedSize": "47" , "Values": "8" }, - { "CompressedSize": "47" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "4" , "StatsSet": "False", "UncompressedSize": "47" , "Values": "8" }, - { "CompressedSize": "55" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "5" , "StatsSet": "False", "UncompressedSize": "55" , "Values": "8" }, - { "CompressedSize": "47" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "6" , "StatsSet": "False", "UncompressedSize": "47" , "Values": "8" }, - { "CompressedSize": "55" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "7" , "StatsSet": "False", "UncompressedSize": "55" , "Values": "8" }, - { "CompressedSize": "88" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "8" , "StatsSet": "False", "UncompressedSize": "88" , "Values": "8" }, - { "CompressedSize": "49" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "9" , "StatsSet": "False", "UncompressedSize": "49" , "Values": "8" }, - { "CompressedSize": "139", "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "10", "StatsSet": "False", "UncompressedSize": "139", "Values": "8" } + { + "Id": "0", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "73", + "CompressedSize": "73" + }, + { + "Id": "1", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "24", + "CompressedSize": "24" + }, + { + "Id": "2", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "47", + "CompressedSize": "47" + }, + { + "Id": "3", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "47", + "CompressedSize": "47" + }, + { + "Id": "4", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "47", + "CompressedSize": "47" + }, + { + "Id": "5", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "55", + "CompressedSize": "55" + }, + { + "Id": "6", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "47", + "CompressedSize": "47" + }, + { + "Id": "7", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "55", + "CompressedSize": "55" + }, + { + "Id": "8", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "88", + "CompressedSize": "88" + }, + { + "Id": "9", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "49", + "CompressedSize": "49" + }, + { + "Id": "10", + "Values": "8", + "StatsSet": "False", + "Compression": "UNCOMPRESSED", + "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", + "UncompressedSize": "139", + "CompressedSize": "139" + } ] } ] @@ -1243,6 +1395,11 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) { TEST_F(TestJSONWithLocalFile, JSONOutputFLBA) { // min-max stats for FLBA contains non-utf8 output, so we don't check // the whole json output. + // simdjson 5.0 changed the format of fractured_json + if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) { + GTEST_SKIP() << "Test requires simdjson >= 5"; + } + std::string json_content = ReadFromLocalFile("fixed_length_byte_array.parquet"); std::string json_contains = R"###({ @@ -1259,7 +1416,7 @@ TEST_F(TestJSONWithLocalFile, JSONOutputFLBA) { "Name": "flba_field", "PhysicalType": "FIXED_LEN_BYTE_ARRAY(4)", "ConvertedType": "NONE", - "LogicalType": { "Type": "None" } + "LogicalType": {"Type": "None"} } ],)###"; @@ -1267,11 +1424,18 @@ TEST_F(TestJSONWithLocalFile, JSONOutputFLBA) { } TEST_F(TestJSONWithLocalFile, JSONOutputSortColumns) { + // simdjson 5.0 changed the format of fractured_json + if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) { + GTEST_SKIP() << "Test requires simdjson >= 5"; + } + std::string json_content = ReadFromLocalFile("sort_columns.parquet"); std::string json_contains = R"###("SortColumns": [ - { "column_idx": 0, "descending": 1, "nulls_first": 1 }, { "column_idx": 1, "descending": 0, "nulls_first": 0 } + {"column_idx": 0, "descending": 1, "nulls_first": 1}, + {"column_idx": 1, "descending": 0, "nulls_first": 0} ],)###"; + EXPECT_THAT(json_content, testing::HasSubstr(json_contains)); }