From 361950e4c9bd6234cba2bf90c0b3712537623ab3 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 20:50:58 +0300 Subject: [PATCH 01/12] GH-33295: [C++] Add list_contains kernel --- .../arrow/compute/kernels/scalar_nested.cc | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc index e9c65aff1ce1..c3300b632f5a 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc @@ -28,6 +28,7 @@ #include "arrow/util/bit_block_counter.h" #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_generate.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/logging_internal.h" #include "arrow/util/string.h" #include "arrow/util/unreachable.h" @@ -537,6 +538,146 @@ const FunctionDoc list_element_doc( "is emitted. Null values emit a null in the output."), {"lists", "index"}); +Result> GetListContainsValue(const ExecValue& value) { + if (value.is_scalar()) { + return value.scalar->GetSharedPtr(); + } + if (value.array.length != 1) { + return Status::NotImplemented( + "list_contains not yet implemented for arrays of values"); + } + return value.array.ToArray()->GetScalar(0); +} + +bool IsNaN(const Scalar& value) { + switch (value.type->id()) { + case Type::FLOAT: + return std::isnan(checked_cast(value).value); + case Type::DOUBLE: + return std::isnan(checked_cast(value).value); + default: + return false; + } +} + +// Like "is_in", and unlike "equal", a null value matches null values and a NaN +// value matches NaN values. +Result ListValuesMatch(KernelContext* ctx, std::shared_ptr values, + const std::shared_ptr& value) { + if (!value->is_valid) { + return CallFunction("is_null", {std::move(values)}, ctx->exec_context()); + } + if (is_floating(values->type->id()) && IsNaN(*value)) { + return CallFunction("is_nan", {std::move(values)}, ctx->exec_context()); + } + return CallFunction("equal", {std::move(values), value}, ctx->exec_context()); +} + +// Search `value` in the child values [values_start, values_start + values_length) +// with a single vectorized comparison, then check each list's range in the resulting +// bitmap. `get_range(i)` returns the (child offset, length) pair of list i. +template +Status ListContainsExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out, + int64_t values_start, int64_t values_length, + GetRange&& get_range) { + const ArraySpan& list = batch[0].array; + ArraySpan* out_arr = out->array_span_mutable(); + + // Only null lists emit a null + if (list.MayHaveNulls()) { + arrow::internal::CopyBitmap(list.buffers[0].data, list.offset, list.length, + out_arr->buffers[0].data, out_arr->offset); + } else { + bit_util::SetBitsTo(out_arr->buffers[0].data, out_arr->offset, out_arr->length, true); + } + out_arr->null_count = list.null_count; + + ARROW_ASSIGN_OR_RAISE(auto value, GetListContainsValue(batch[1])); + auto values = list.child_data[0].ToArrayData()->Slice(values_start, values_length); + ARROW_ASSIGN_OR_RAISE(Datum match, ListValuesMatch(ctx, std::move(values), value)); + const ArrayData& match_data = *match.array(); + // Null comparison results never match + std::shared_ptr matches = match_data.buffers[1]; + int64_t matches_offset = match_data.offset; + if (match_data.MayHaveNulls()) { + ARROW_ASSIGN_OR_RAISE( + matches, arrow::internal::BitmapAnd( + ctx->memory_pool(), matches->data(), match_data.offset, + match_data.buffers[0]->data(), match_data.offset, match_data.length, + /*out_offset=*/0)); + matches_offset = 0; + } + matches_offset -= values_start; + + int64_t i = 0; + arrow::internal::GenerateBitsUnrolled( + out_arr->buffers[1].data, out_arr->offset, out_arr->length, [&] { + bool found = false; + if (list.IsValid(i)) { + const auto [start, length] = get_range(i); + found = arrow::internal::CountSetBits(matches->data(), matches_offset + start, + length) > 0; + } + ++i; + return found; + }); + return Status::OK(); +} + +template +Status ListContains(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) { + using offset_type = typename Type::offset_type; + using Range = std::pair; + const ArraySpan& list = batch[0].array; + if (list.length == 0) { + return Status::OK(); + } + const offset_type* offsets = list.GetValues(1); + if constexpr (is_list_view_type::value) { + // List views may reference child values in any order + const offset_type* sizes = list.GetValues(2); + return ListContainsExec(ctx, batch, out, 0, list.child_data[0].length, + [&](int64_t i) { return Range(offsets[i], sizes[i]); }); + } else { + return ListContainsExec( + ctx, batch, out, offsets[0], offsets[list.length] - offsets[0], + [&](int64_t i) { return Range(offsets[i], offsets[i + 1] - offsets[i]); }); + } +} + +Status FixedSizeListContains(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) { + const int64_t width = + checked_cast(*batch[0].type()).list_size(); + const ArraySpan& list = batch[0].array; + return ListContainsExec( + ctx, batch, out, list.offset * width, list.length * width, + [&](int64_t i) { return std::make_pair((list.offset + i) * width, width); }); +} + +void AddListContainsKernels(ScalarFunction* func) { + auto add_kernel = [&](Type::type list_type_id, ArrayKernelExec exec) { + ScalarKernel kernel({InputType(list_type_id), InputType::Any()}, boolean(), exec); + // A null value is searched for rather than propagated + kernel.null_handling = NullHandling::COMPUTED_PREALLOCATE; + DCHECK_OK(func->AddKernel(std::move(kernel))); + }; + add_kernel(Type::LIST, ListContains); + add_kernel(Type::LARGE_LIST, ListContains); + add_kernel(Type::LIST_VIEW, ListContains); + add_kernel(Type::LARGE_LIST_VIEW, ListContains); + add_kernel(Type::FIXED_SIZE_LIST, FixedSizeListContains); +} + +const FunctionDoc list_contains_doc( + "Check whether lists contain a given value", + ("`lists` must have a list-like type and `value` must be a scalar\n" + "comparable with the list value type.\n" + "For each list in `lists`, true is emitted if any of its values is equal\n" + "to `value`, false otherwise. A null `value` matches null list values,\n" + "and a NaN `value` matches NaN list values; otherwise null list values\n" + "never match. Null lists emit a null in the output."), + {"lists", "value"}); + struct StructFieldFunctor { static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) { const auto& options = OptionsWrapper::Get(ctx); @@ -960,6 +1101,11 @@ void RegisterScalarNested(FunctionRegistry* registry) { AddListElementKernels(list_element.get()); DCHECK_OK(registry->AddFunction(std::move(list_element))); + auto list_contains = std::make_shared("list_contains", Arity::Binary(), + list_contains_doc); + AddListContainsKernels(list_contains.get()); + DCHECK_OK(registry->AddFunction(std::move(list_contains))); + auto list_slice = std::make_shared("list_slice", Arity::Unary(), list_slice_doc); AddListSliceKernels(list_slice.get()); From e3844d80ba1b01ea2de7e1be175497da765f187c Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 20:50:58 +0300 Subject: [PATCH 02/12] GH-33295: [C++] Test list_contains kernel --- .../compute/kernels/scalar_nested_test.cc | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc index b5a68d12cb0c..02af24bd0b28 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc @@ -128,6 +128,163 @@ TEST(TestScalarNested, ListElementInvalid) { Raises(StatusCode::Invalid)); } +TEST(TestScalarNested, ListContains) { + auto sample = "[[7, 5, 81], [6, null, 4, 7, 8], [], [5], null, [null]]"; + for (auto ty : NumericTypes()) { + for (auto list_type : + {list(ty), large_list(ty), list_view(ty), large_list_view(ty)}) { + auto input = ArrayFromJSON(list_type, sample); + CheckScalar("list_contains", {input, ScalarFromJSON(ty, "5")}, + ArrayFromJSON(boolean(), "[true, false, false, true, null, false]")); + CheckScalar("list_contains", {input, ScalarFromJSON(ty, "7")}, + ArrayFromJSON(boolean(), "[true, true, false, false, null, false]")); + CheckScalar("list_contains", {input, ScalarFromJSON(ty, "null")}, + ArrayFromJSON(boolean(), "[false, true, false, false, null, true]")); + CheckScalar("list_contains", + {ArrayFromJSON(list_type, "[]"), ScalarFromJSON(ty, "5")}, + ArrayFromJSON(boolean(), "[]")); + } + } + + auto input = + ArrayFromJSON(list(utf8()), R"([["a", "b"], ["a", "c"], ["b", "c", "d"]])"); + CheckScalar("list_contains", {input, ScalarFromJSON(utf8(), R"("a")")}, + ArrayFromJSON(boolean(), "[true, true, false]")); + CheckScalar("list_contains", {input->Slice(1), ScalarFromJSON(utf8(), R"("b")")}, + ArrayFromJSON(boolean(), "[false, true]")); + + // All-null lists contain no non-null value + CheckScalar( + "list_contains", + {ArrayFromJSON(list(int64()), "[[null, null]]"), ScalarFromJSON(int64(), "1")}, + ArrayFromJSON(boolean(), "[false]")); +} + +TEST(TestScalarNested, ListContainsNull) { + // A null value matches lists holding a null + auto input = ArrayFromJSON( + list(int32()), "[[1, null], [null], [1, 2], [1, 1], [2, 1, null, 1], [], null]"); + CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "null")}, + ArrayFromJSON(boolean(), "[true, true, false, false, true, false, null]")); + CheckScalar("list_contains", {input, MakeNullScalar(null())}, + ArrayFromJSON(boolean(), "[true, true, false, false, true, false, null]")); + CheckScalar("list_contains", + {ArrayFromJSON(list(int32()), "[[]]"), ScalarFromJSON(int32(), "null")}, + ArrayFromJSON(boolean(), "[false]")); + + input = ArrayFromJSON(list(utf8()), R"([["x", null], [null], ["x", "y"], [], null])"); + CheckScalar("list_contains", {input, ScalarFromJSON(utf8(), "null")}, + ArrayFromJSON(boolean(), "[true, true, false, false, null]")); + + input = ArrayFromJSON(list(list(int32())), + "[[[1], null], [null], [[1], [null]], [], null]"); + CheckScalar("list_contains", {input, MakeNullScalar(list(int32()))}, + ArrayFromJSON(boolean(), "[true, true, false, false, null]")); + + input = ArrayFromJSON(list(null()), "[[null], [], null]"); + CheckScalar("list_contains", {input, MakeNullScalar(null())}, + ArrayFromJSON(boolean(), "[true, false, null]")); +} + +TEST(TestScalarNested, ListContainsNaN) { + // Unlike "equal", a NaN value matches NaN list values + for (auto ty : {float32(), float64()}) { + auto input = ArrayFromJSON(list(ty), "[[1.5, null], [NaN], [1.5, NaN], [], null]"); + for (auto value_ty : {float32(), float64()}) { + CheckScalar("list_contains", {input, ScalarFromJSON(value_ty, "NaN")}, + ArrayFromJSON(boolean(), "[false, true, true, false, null]")); + CheckScalar("list_contains", {input, ScalarFromJSON(value_ty, "1.5")}, + ArrayFromJSON(boolean(), "[true, false, true, false, null]")); + } + } + CheckScalar( + "list_contains", + {ArrayFromJSON(list(int64()), "[[1], []]"), ScalarFromJSON(float64(), "NaN")}, + ArrayFromJSON(boolean(), "[false, false]")); +} + +TEST(TestScalarNested, ListContainsValueTypes) { + auto check = [](const std::shared_ptr& ty, const std::string& x, + const std::string& y) { + auto input = ArrayFromJSON( + list(ty), "[[" + x + ", null], [" + y + "], [" + x + ", " + y + "], [], null]"); + CheckScalar("list_contains", {input, ScalarFromJSON(ty, x)}, + ArrayFromJSON(boolean(), "[true, false, true, false, null]")); + CheckScalar("list_contains", {input, ScalarFromJSON(ty, y)}, + ArrayFromJSON(boolean(), "[false, true, true, false, null]")); + }; + check(utf8(), R"("x")", R"("y")"); + check(large_binary(), R"("x")", R"("y")"); + check(boolean(), "true", "false"); + check(date32(), "18262", "18628"); + check(decimal128(38, 2), R"("1.50")", R"("2.50")"); + check(timestamp(TimeUnit::NANO), "1", "2"); +} + +TEST(TestScalarNested, ListContainsImplicitCast) { + auto input = ArrayFromJSON(list(int64()), + "[[2, 2, 3, null, null], null, [], [null], [1], [0, -1]]"); + CheckScalar("list_contains", {input, ScalarFromJSON(float64(), "2.0")}, + ArrayFromJSON(boolean(), "[true, null, false, false, false, false]")); + CheckScalar("list_contains", {input, ScalarFromJSON(float64(), "1.5")}, + ArrayFromJSON(boolean(), "[false, null, false, false, false, false]")); + + // No overflow of the list values + input = ArrayFromJSON(list(int8()), "[[44, 1], null, [], [null]]"); + CheckScalar("list_contains", {input, ScalarFromJSON(int64(), "300")}, + ArrayFromJSON(boolean(), "[false, null, false, false]")); + + input = ArrayFromJSON(list(float64()), "[[1.0, 2.0], [3.0], null, []]"); + CheckScalar("list_contains", {input, ScalarFromJSON(int64(), "1")}, + ArrayFromJSON(boolean(), "[true, false, null, false]")); + + input = ArrayFromJSON(list(decimal128(38, 2)), R"([["1.50"], ["2.50"]])"); + CheckScalar("list_contains", {input, ScalarFromJSON(decimal128(3, 2), R"("1.50")")}, + ArrayFromJSON(boolean(), "[true, false]")); +} + +TEST(TestScalarNested, ListContainsChunked) { + auto array = ArrayFromJSON(list(int32()), "[[1, 2], [3], null, [], [2, null]]"); + auto input = std::make_shared( + ArrayVector{array->Slice(3), array->Slice(0, 0), array->Slice(1, 2)}); + ASSERT_OK_AND_ASSIGN( + Datum result, CallFunction("list_contains", {input, ScalarFromJSON(int32(), "2")})); + AssertDatumsEqual( + ChunkedArrayFromJSON(boolean(), {"[false, true]", "[]", "[false, null]"}), result, + /*verbose=*/true); +} + +TEST(TestScalarNested, ListContainsFixedSizeList) { + auto input = ArrayFromJSON(fixed_size_list(int32(), 2), + "[[1, 2], [3, null], null, [2, 4], [null, null]]"); + CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "2")}, + ArrayFromJSON(boolean(), "[true, false, null, true, false]")); + CheckScalar("list_contains", {input->Slice(2), ScalarFromJSON(int32(), "4")}, + ArrayFromJSON(boolean(), "[null, true, false]")); +} + +TEST(TestScalarNested, ListContainsListViewOutOfOrder) { + // Views: [3, 4], [1, 2], [2, 3], [4], [] + auto values = ArrayFromJSON(int32(), "[1, 2, 3, 4]"); + auto offsets = ArrayFromJSON(int32(), "[2, 0, 1, 3, 0]"); + auto sizes = ArrayFromJSON(int32(), "[2, 2, 2, 1, 0]"); + ASSERT_OK_AND_ASSIGN(auto input, ListViewArray::FromArrays(*offsets, *sizes, *values)); + CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "3")}, + ArrayFromJSON(boolean(), "[true, false, true, false, false]")); + CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "1")}, + ArrayFromJSON(boolean(), "[false, true, false, false, false]")); +} + +TEST(TestScalarNested, ListContainsInvalid) { + auto input = ArrayFromJSON(list(int32()), "[[1, 2], [3]]"); + EXPECT_THAT(CallFunction("list_contains", {input, ArrayFromJSON(int32(), "[1, 3]")}), + Raises(StatusCode::NotImplemented)); + EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(utf8(), R"("a")")}), + Raises(StatusCode::NotImplemented)); + EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(boolean(), "true")}), + Raises(StatusCode::NotImplemented)); +} + using VarLenListLikeTypeFactory = std::shared_ptr (*)(std::shared_ptr); static const VarLenListLikeTypeFactory kVarLenListTypeFactories[] = { From 459673d0506677387a3d0a659c056decb30e2ec1 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 20:50:58 +0300 Subject: [PATCH 03/12] GH-33295: [Docs] Document list_contains --- docs/source/cpp/compute.rst | 7 +++++++ docs/source/python/api/compute.rst | 1 + 2 files changed, 8 insertions(+) diff --git a/docs/source/cpp/compute.rst b/docs/source/cpp/compute.rst index f24fea20a9d8..3ed91fda86df 100644 --- a/docs/source/cpp/compute.rst +++ b/docs/source/cpp/compute.rst @@ -1924,6 +1924,8 @@ Structural transforms +---------------------+------------+-------------------------------------+------------------+------------------------------+--------+ | Function name | Arity | Input types | Output type | Options class | Notes | +=====================+============+=====================================+==================+==============================+========+ +| list_contains | Binary | List-like (Arg 0), Any (Arg 1) | Boolean | | \(7) | ++---------------------+------------+-------------------------------------+------------------+------------------------------+--------+ | list_element | Binary | List-like (Arg 0), Integral (Arg 1) | List value type | | \(1) | +---------------------+------------+-------------------------------------+------------------+------------------------------+--------+ | list_flatten | Unary | List-like | List value type | | \(2) | @@ -1982,6 +1984,11 @@ Structural transforms index *n* and the type code at index *n* is 2. * The indices ``2`` and ``7`` are invalid. +* \(7) Output is true for each list containing a value equal to the second + argument, which must be a scalar. Unlike ``equal``, a null second argument + matches null list values and a NaN second argument matches NaN list values; + otherwise null list values never match. Null lists emit a null in the output. + .. _cpp-compute-vector-replace-functions: Replace functions diff --git a/docs/source/python/api/compute.rst b/docs/source/python/api/compute.rst index 6a4b04468d4c..f83fce2be84e 100644 --- a/docs/source/python/api/compute.rst +++ b/docs/source/python/api/compute.rst @@ -569,6 +569,7 @@ Structural Transforms fill_null fill_null_backward fill_null_forward + list_contains list_element list_flatten list_parent_indices From 60610ce1ee7d6afbe99086a937ad75a69351dedb Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 20:53:13 +0300 Subject: [PATCH 04/12] GH-33295: [Python] Test list_contains --- python/pyarrow/tests/test_compute.py | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 797fbc220ec3..7bdc293480bf 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -3899,6 +3899,38 @@ def test_list_element(): assert result.equals(expected) +@pytest.mark.parametrize("list_type", [ + pa.list_(pa.int64()), pa.large_list(pa.int64()), + pa.list_view(pa.int64()), pa.large_list_view(pa.int64()), +]) +def test_list_contains(list_type): + lists = pa.array([[1, 2, None], [3], [], None], list_type) + assert pc.list_contains(lists, 2).to_pylist() == [True, False, False, None] + assert pc.list_contains(lists, 2.0).to_pylist() == [True, False, False, None] + assert pc.list_contains(lists, 4).to_pylist() == [False, False, False, None] + # A null value matches null list values + assert pc.list_contains(lists, None).to_pylist() == [True, False, False, None] + + +def test_list_contains_nan(): + lists = pa.array([[1.5, None], [float("nan")], []]) + result = pc.list_contains(lists, float("nan")) + assert result.to_pylist() == [False, True, False] + + +def test_list_contains_fixed_size_list(): + lists = pa.array([["a", "b"], ["c", None], None], pa.list_(pa.string(), 2)) + assert pc.list_contains(lists, "c").to_pylist() == [False, True, None] + + +def test_list_contains_invalid(): + lists = pa.array([[1, 2], [3]]) + with pytest.raises(pa.ArrowNotImplementedError): + pc.list_contains(lists, "a") + with pytest.raises(pa.ArrowNotImplementedError): + pc.list_contains(lists, pa.array([1, 2])) + + def test_count_distinct(): samples = [datetime.datetime(year=y, month=1, day=1) for y in range(1992, 2092)] arr = pa.array(samples, pa.timestamp("ns")) From 7349c0e430761b55a927ccfc2272c3a54d603a74 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 21:32:28 +0300 Subject: [PATCH 05/12] GH-33295: [C++] Support arrays of values in list_contains --- .../arrow/compute/kernels/scalar_nested.cc | 207 +++++++++++++----- .../compute/kernels/scalar_nested_test.cc | 56 ++++- docs/source/cpp/compute.rst | 5 +- python/pyarrow/tests/test_compute.py | 9 +- 4 files changed, 213 insertions(+), 64 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc index c3300b632f5a..2d1c4225eee5 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc @@ -20,6 +20,7 @@ #include #include "arrow/array/array_base.h" #include "arrow/array/builder_nested.h" +#include "arrow/array/builder_primitive.h" #include "arrow/compute/api_scalar.h" #include "arrow/compute/kernels/common_internal.h" #include "arrow/compute/registry_internal.h" @@ -538,17 +539,6 @@ const FunctionDoc list_element_doc( "is emitted. Null values emit a null in the output."), {"lists", "index"}); -Result> GetListContainsValue(const ExecValue& value) { - if (value.is_scalar()) { - return value.scalar->GetSharedPtr(); - } - if (value.array.length != 1) { - return Status::NotImplemented( - "list_contains not yet implemented for arrays of values"); - } - return value.array.ToArray()->GetScalar(0); -} - bool IsNaN(const Scalar& value) { switch (value.type->id()) { case Type::FLOAT: @@ -573,30 +563,69 @@ Result ListValuesMatch(KernelContext* ctx, std::shared_ptr val return CallFunction("equal", {std::move(values), value}, ctx->exec_context()); } -// Search `value` in the child values [values_start, values_start + values_length) -// with a single vectorized comparison, then check each list's range in the resulting -// bitmap. `get_range(i)` returns the (child offset, length) pair of list i. -template -Status ListContainsExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out, - int64_t values_start, int64_t values_length, - GetRange&& get_range) { - const ArraySpan& list = batch[0].array; - ArraySpan* out_arr = out->array_span_mutable(); +// Element-wise version of the above, matching each of `values` with the value at the +// same index. +Result ListValuesMatch(KernelContext* ctx, std::shared_ptr values, + std::shared_ptr value) { + ExecContext* exec_ctx = ctx->exec_context(); + if (value->GetNullCount() == value->length) { + return CallFunction("is_null", {std::move(values)}, exec_ctx); + } + ARROW_ASSIGN_OR_RAISE(Datum match, CallFunction("equal", {values, value}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE(Datum values_null, CallFunction("is_null", {values}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE(Datum value_null, CallFunction("is_null", {value}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE(Datum both_null, + CallFunction("and", {values_null, value_null}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE(match, CallFunction("or_kleene", {match, both_null}, exec_ctx)); + if (is_floating(values->type->id()) && is_floating(value->type->id())) { + ARROW_ASSIGN_OR_RAISE(Datum values_nan, CallFunction("is_nan", {values}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE(Datum value_nan, CallFunction("is_nan", {value}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE(Datum both_nan, + CallFunction("and_kleene", {values_nan, value_nan}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE(match, CallFunction("or_kleene", {match, both_nan}, exec_ctx)); + } + return match; +} - // Only null lists emit a null - if (list.MayHaveNulls()) { - arrow::internal::CopyBitmap(list.buffers[0].data, list.offset, list.length, - out_arr->buffers[0].data, out_arr->offset); +// Returns the (child offset, length) pair of list i +template +std::pair GetListRange(const ArraySpan& list, int64_t i) { + if constexpr (std::is_same_v) { + const int64_t width = checked_cast(*list.type).list_size(); + return {(list.offset + i) * width, width}; } else { - bit_util::SetBitsTo(out_arr->buffers[0].data, out_arr->offset, out_arr->length, true); + using offset_type = typename Type::offset_type; + const offset_type* offsets = list.GetValues(1); + if constexpr (is_list_view_type::value) { + return {offsets[i], list.GetValues(2)[i]}; + } else { + return {offsets[i], offsets[i + 1] - offsets[i]}; + } } - out_arr->null_count = list.null_count; +} - ARROW_ASSIGN_OR_RAISE(auto value, GetListContainsValue(batch[1])); - auto values = list.child_data[0].ToArrayData()->Slice(values_start, values_length); - ARROW_ASSIGN_OR_RAISE(Datum match, ListValuesMatch(ctx, std::move(values), value)); +// Returns the (child offset, length) pair of the child values referenced by the lists +template +std::pair GetListValuesRange(const ArraySpan& list) { + if constexpr (std::is_same_v) { + const int64_t width = checked_cast(*list.type).list_size(); + return {list.offset * width, list.length * width}; + } else if constexpr (is_list_view_type::value) { + // List views may reference child values in any order + return {0, list.child_data[0].length}; + } else { + using offset_type = typename Type::offset_type; + const offset_type* offsets = list.GetValues(1); + return {offsets[0], offsets[list.length] - offsets[0]}; + } +} + +// Emit whether each valid list has a match in the range of `match` returned by +// `get_match_range(i)`. Null matches never count. +template +Status EmitListMatches(KernelContext* ctx, const ArraySpan& list, const Datum& match, + ArraySpan* out_arr, GetMatchRange&& get_match_range) { const ArrayData& match_data = *match.array(); - // Null comparison results never match std::shared_ptr matches = match_data.buffers[1]; int64_t matches_offset = match_data.offset; if (match_data.MayHaveNulls()) { @@ -607,14 +636,13 @@ Status ListContainsExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* o /*out_offset=*/0)); matches_offset = 0; } - matches_offset -= values_start; int64_t i = 0; arrow::internal::GenerateBitsUnrolled( out_arr->buffers[1].data, out_arr->offset, out_arr->length, [&] { bool found = false; if (list.IsValid(i)) { - const auto [start, length] = get_range(i); + const auto [start, length] = get_match_range(i); found = arrow::internal::CountSetBits(matches->data(), matches_offset + start, length) > 0; } @@ -624,34 +652,95 @@ Status ListContainsExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* o return Status::OK(); } +// Search a single `value` in the referenced child values with one vectorized +// comparison, then check each list's range in the resulting bitmap. +template +Status ListContainsScalar(KernelContext* ctx, const ArraySpan& list, + const std::shared_ptr& value, ArraySpan* out_arr) { + const auto [values_start, values_length] = GetListValuesRange(list); + auto values = list.child_data[0].ToArrayData()->Slice(values_start, values_length); + ARROW_ASSIGN_OR_RAISE(Datum match, ListValuesMatch(ctx, std::move(values), value)); + return EmitListMatches(ctx, list, match, out_arr, [&](int64_t i) { + const auto [start, length] = GetListRange(list, i); + return std::make_pair(start - values_start, length); + }); +} + +// Gather the child values of each valid list next to copies of its value, compare +// them element-wise, then check each list's range in the resulting bitmap. +template +Status ListContainsArray(KernelContext* ctx, const ArraySpan& list, + const ArraySpan& value, ArraySpan* out_arr) { + Int64Builder values_indices(ctx->memory_pool()); + Int64Builder value_indices(ctx->memory_pool()); + for (int64_t i = 0; i < list.length; ++i) { + if (list.IsValid(i)) { + const auto [start, length] = GetListRange(list, i); + RETURN_NOT_OK(values_indices.Reserve(length)); + RETURN_NOT_OK(value_indices.Reserve(length)); + for (int64_t j = start; j < start + length; ++j) { + values_indices.UnsafeAppend(j); + value_indices.UnsafeAppend(i); + } + } + } + ARROW_ASSIGN_OR_RAISE(auto values_taken, values_indices.Finish()); + ARROW_ASSIGN_OR_RAISE(auto value_taken, value_indices.Finish()); + ExecContext* exec_ctx = ctx->exec_context(); + ARROW_ASSIGN_OR_RAISE( + Datum values, + CallFunction("take", {list.child_data[0].ToArrayData(), values_taken}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE( + Datum repeated_value, + CallFunction("take", {value.ToArrayData(), value_taken}, exec_ctx)); + ARROW_ASSIGN_OR_RAISE(Datum match, + ListValuesMatch(ctx, values.array(), repeated_value.array())); + + // Lists are gathered in order, so each list's range follows the previous one + int64_t match_start = 0; + return EmitListMatches(ctx, list, match, out_arr, [&](int64_t i) { + const int64_t length = GetListRange(list, i).second; + match_start += length; + return std::make_pair(match_start - length, length); + }); +} + template Status ListContains(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) { - using offset_type = typename Type::offset_type; - using Range = std::pair; - const ArraySpan& list = batch[0].array; - if (list.length == 0) { + if (batch.length == 0) { return Status::OK(); } - const offset_type* offsets = list.GetValues(1); - if constexpr (is_list_view_type::value) { - // List views may reference child values in any order - const offset_type* sizes = list.GetValues(2); - return ListContainsExec(ctx, batch, out, 0, list.child_data[0].length, - [&](int64_t i) { return Range(offsets[i], sizes[i]); }); - } else { - return ListContainsExec( - ctx, batch, out, offsets[0], offsets[list.length] - offsets[0], - [&](int64_t i) { return Range(offsets[i], offsets[i + 1] - offsets[i]); }); + if (batch[0].is_scalar()) { + // A scalar list is only paired with an array of values here, so broadcast it + ARROW_ASSIGN_OR_RAISE(auto lists, MakeArrayFromScalar(*batch[0].scalar, batch.length, + ctx->memory_pool())); + ExecSpan array_batch = batch; + array_batch.values[0].SetArray(*lists->data()); + return ListContains(ctx, array_batch, out); } -} -Status FixedSizeListContains(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) { - const int64_t width = - checked_cast(*batch[0].type()).list_size(); const ArraySpan& list = batch[0].array; - return ListContainsExec( - ctx, batch, out, list.offset * width, list.length * width, - [&](int64_t i) { return std::make_pair((list.offset + i) * width, width); }); + ArraySpan* out_arr = out->array_span_mutable(); + + // Only null lists emit a null + if (list.MayHaveNulls()) { + arrow::internal::CopyBitmap(list.buffers[0].data, list.offset, list.length, + out_arr->buffers[0].data, out_arr->offset); + } else { + bit_util::SetBitsTo(out_arr->buffers[0].data, out_arr->offset, out_arr->length, true); + } + out_arr->null_count = list.null_count; + + const ExecValue& value = batch[1]; + if (value.is_scalar()) { + return ListContainsScalar(ctx, list, value.scalar->GetSharedPtr(), out_arr); + } + if (value.array.length == 1) { + // Also covers scalar inputs, which are promoted to arrays + ARROW_ASSIGN_OR_RAISE(auto single_value, value.array.ToArray()->GetScalar(0)); + return ListContainsScalar(ctx, list, single_value, out_arr); + } + return ListContainsArray(ctx, list, value.array, out_arr); } void AddListContainsKernels(ScalarFunction* func) { @@ -665,17 +754,17 @@ void AddListContainsKernels(ScalarFunction* func) { add_kernel(Type::LARGE_LIST, ListContains); add_kernel(Type::LIST_VIEW, ListContains); add_kernel(Type::LARGE_LIST_VIEW, ListContains); - add_kernel(Type::FIXED_SIZE_LIST, FixedSizeListContains); + add_kernel(Type::FIXED_SIZE_LIST, ListContains); } const FunctionDoc list_contains_doc( "Check whether lists contain a given value", - ("`lists` must have a list-like type and `value` must be a scalar\n" - "comparable with the list value type.\n" + ("`lists` must have a list-like type and `value` must be comparable\n" + "with the list value type.\n" "For each list in `lists`, true is emitted if any of its values is equal\n" - "to `value`, false otherwise. A null `value` matches null list values,\n" - "and a NaN `value` matches NaN list values; otherwise null list values\n" - "never match. Null lists emit a null in the output."), + "to the corresponding `value`, false otherwise. A null `value` matches\n" + "null list values, and a NaN `value` matches NaN list values; otherwise\n" + "null list values never match. Null lists emit a null in the output."), {"lists", "value"}); struct StructFieldFunctor { diff --git a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc index 02af24bd0b28..2fe9c4c1d978 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc @@ -275,10 +275,62 @@ TEST(TestScalarNested, ListContainsListViewOutOfOrder) { ArrayFromJSON(boolean(), "[false, true, false, false, false]")); } +TEST(TestScalarNested, ListContainsArrayOfValues) { + // Each list is searched for the value at the same index + for (auto list_type : {list(int32()), large_list(int32()), list_view(int32()), + large_list_view(int32())}) { + auto input = + ArrayFromJSON(list_type, "[[1, 2], [3, null], [], null, [4, 5], [null], [6]]"); + CheckScalar("list_contains", + {input, ArrayFromJSON(int32(), "[2, null, 1, 3, 4, null, 7]")}, + ArrayFromJSON(boolean(), "[true, true, false, null, true, true, false]")); + } + CheckScalar("list_contains", + {ArrayFromJSON(fixed_size_list(int32(), 2), + "[[1, 2], [3, null], null, [4, 5], [null, 6]]"), + ArrayFromJSON(int32(), "[2, null, 1, 5, 7]")}, + ArrayFromJSON(boolean(), "[true, true, null, true, false]")); + + CheckScalar("list_contains", + {ArrayFromJSON(list(int64()), "[[1, 2], [3], [4]]"), + ArrayFromJSON(float64(), "[2.0, 3.5, 4.0]")}, + ArrayFromJSON(boolean(), "[true, false, true]")); + CheckScalar("list_contains", + {ArrayFromJSON(list(float64()), "[[1.5, NaN], [NaN], [1.5], [null]]"), + ArrayFromJSON(float64(), "[NaN, 1.5, NaN, NaN]")}, + ArrayFromJSON(boolean(), "[true, false, false, false]")); + CheckScalar("list_contains", + {ArrayFromJSON(list(int32()), "[[1, null], [2], []]"), + ArrayFromJSON(null(), "[null, null, null]")}, + ArrayFromJSON(boolean(), "[true, false, false]")); + + // Views: [3, 4], [1, 2], [2, 3], [4], [] + ASSERT_OK_AND_ASSIGN( + auto input, ListViewArray::FromArrays(*ArrayFromJSON(int32(), "[2, 0, 1, 3, 0]"), + *ArrayFromJSON(int32(), "[2, 2, 2, 1, 0]"), + *ArrayFromJSON(int32(), "[1, 2, 3, 4]"))); + CheckScalar("list_contains", {input, ArrayFromJSON(int32(), "[3, 1, 1, 4, 1]")}, + ArrayFromJSON(boolean(), "[true, true, false, true, false]")); + + // A scalar list is searched for each value + CheckScalar("list_contains", + {ScalarFromJSON(list(int32()), "[1, null]"), + ArrayFromJSON(int32(), "[1, 2, null]")}, + ArrayFromJSON(boolean(), "[true, false, true]")); + CheckScalar("list_contains", + {ScalarFromJSON(fixed_size_list(int32(), 2), "[1, 2]"), + ArrayFromJSON(int32(), "[2, 3]")}, + ArrayFromJSON(boolean(), "[true, false]")); + CheckScalar("list_contains", + {ScalarFromJSON(list(int32()), "null"), ArrayFromJSON(int32(), "[1, 2]")}, + ArrayFromJSON(boolean(), "[null, null]")); +} + TEST(TestScalarNested, ListContainsInvalid) { auto input = ArrayFromJSON(list(int32()), "[[1, 2], [3]]"); - EXPECT_THAT(CallFunction("list_contains", {input, ArrayFromJSON(int32(), "[1, 3]")}), - Raises(StatusCode::NotImplemented)); + EXPECT_THAT( + CallFunction("list_contains", {input, ArrayFromJSON(utf8(), R"(["a", "b"])")}), + Raises(StatusCode::NotImplemented)); EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(utf8(), R"("a")")}), Raises(StatusCode::NotImplemented)); EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(boolean(), "true")}), diff --git a/docs/source/cpp/compute.rst b/docs/source/cpp/compute.rst index 3ed91fda86df..5e29107a9f1c 100644 --- a/docs/source/cpp/compute.rst +++ b/docs/source/cpp/compute.rst @@ -1985,8 +1985,9 @@ Structural transforms * The indices ``2`` and ``7`` are invalid. * \(7) Output is true for each list containing a value equal to the second - argument, which must be a scalar. Unlike ``equal``, a null second argument - matches null list values and a NaN second argument matches NaN list values; + argument. If the second argument is an array, each list is searched for the + value at the same index. Unlike ``equal``, a null second argument matches + null list values and a NaN second argument matches NaN list values; otherwise null list values never match. Null lists emit a null in the output. .. _cpp-compute-vector-replace-functions: diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 7bdc293480bf..042456a12439 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -3923,12 +3923,19 @@ def test_list_contains_fixed_size_list(): assert pc.list_contains(lists, "c").to_pylist() == [False, True, None] +def test_list_contains_array(): + lists = pa.array([[1, 2], [3, None], [4], None]) + values = pa.array([2, None, 5, 1]) + result = pc.list_contains(lists, values) + assert result.to_pylist() == [True, True, False, None] + + def test_list_contains_invalid(): lists = pa.array([[1, 2], [3]]) with pytest.raises(pa.ArrowNotImplementedError): pc.list_contains(lists, "a") with pytest.raises(pa.ArrowNotImplementedError): - pc.list_contains(lists, pa.array([1, 2])) + pc.list_contains(lists, pa.array(["a", "b"])) def test_count_distinct(): From 7004d3330cca642ffdf022c568ad6fd106612019 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 21:33:42 +0300 Subject: [PATCH 06/12] GH-33295: [C++] Type-check null values in list_contains --- cpp/src/arrow/compute/kernels/scalar_nested.cc | 14 ++++++++++++++ .../arrow/compute/kernels/scalar_nested_test.cc | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc index 2d1c4225eee5..529f02d6087a 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc @@ -550,11 +550,24 @@ bool IsNaN(const Scalar& value) { } } +// Null values are matched without "equal", but must be comparable all the same +Status CheckNullValueType(KernelContext* ctx, const DataType& values_type, + const DataType& value_type) { + if (value_type.id() == Type::NA || value_type.Equals(values_type)) { + return Status::OK(); + } + ARROW_ASSIGN_OR_RAISE(auto equal, + ctx->exec_context()->func_registry()->GetFunction("equal")); + std::vector types{&values_type, &value_type}; + return equal->DispatchBest(&types).status(); +} + // Like "is_in", and unlike "equal", a null value matches null values and a NaN // value matches NaN values. Result ListValuesMatch(KernelContext* ctx, std::shared_ptr values, const std::shared_ptr& value) { if (!value->is_valid) { + RETURN_NOT_OK(CheckNullValueType(ctx, *values->type, *value->type)); return CallFunction("is_null", {std::move(values)}, ctx->exec_context()); } if (is_floating(values->type->id()) && IsNaN(*value)) { @@ -569,6 +582,7 @@ Result ListValuesMatch(KernelContext* ctx, std::shared_ptr val std::shared_ptr value) { ExecContext* exec_ctx = ctx->exec_context(); if (value->GetNullCount() == value->length) { + RETURN_NOT_OK(CheckNullValueType(ctx, *values->type, *value->type)); return CallFunction("is_null", {std::move(values)}, exec_ctx); } ARROW_ASSIGN_OR_RAISE(Datum match, CallFunction("equal", {values, value}, exec_ctx)); diff --git a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc index 2fe9c4c1d978..d99393d90435 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc @@ -168,6 +168,8 @@ TEST(TestScalarNested, ListContainsNull) { ArrayFromJSON(boolean(), "[true, true, false, false, true, false, null]")); CheckScalar("list_contains", {input, MakeNullScalar(null())}, ArrayFromJSON(boolean(), "[true, true, false, false, true, false, null]")); + CheckScalar("list_contains", {input, ScalarFromJSON(int64(), "null")}, + ArrayFromJSON(boolean(), "[true, true, false, false, true, false, null]")); CheckScalar("list_contains", {ArrayFromJSON(list(int32()), "[[]]"), ScalarFromJSON(int32(), "null")}, ArrayFromJSON(boolean(), "[false]")); @@ -335,6 +337,12 @@ TEST(TestScalarNested, ListContainsInvalid) { Raises(StatusCode::NotImplemented)); EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(boolean(), "true")}), Raises(StatusCode::NotImplemented)); + // Null values must be comparable too + EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(utf8(), "null")}), + Raises(StatusCode::NotImplemented)); + EXPECT_THAT( + CallFunction("list_contains", {input, ArrayFromJSON(utf8(), "[null, null]")}), + Raises(StatusCode::NotImplemented)); } using VarLenListLikeTypeFactory = From cb3cea89104e05920210888f910134630003a2b3 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 21:34:39 +0300 Subject: [PATCH 07/12] GH-33295: [C++] Match float16 NaN values in list_contains --- cpp/src/arrow/compute/kernels/scalar_nested.cc | 4 ++++ cpp/src/arrow/compute/kernels/scalar_nested_test.cc | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc index 529f02d6087a..9367080d6658 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc @@ -30,6 +30,7 @@ #include "arrow/util/bit_util.h" #include "arrow/util/bitmap_generate.h" #include "arrow/util/bitmap_ops.h" +#include "arrow/util/float16.h" #include "arrow/util/logging_internal.h" #include "arrow/util/string.h" #include "arrow/util/unreachable.h" @@ -541,6 +542,9 @@ const FunctionDoc list_element_doc( bool IsNaN(const Scalar& value) { switch (value.type->id()) { + case Type::HALF_FLOAT: + return util::Float16::FromBits(checked_cast(value).value) + .is_nan(); case Type::FLOAT: return std::isnan(checked_cast(value).value); case Type::DOUBLE: diff --git a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc index d99393d90435..b0f5811d55af 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc @@ -203,6 +203,15 @@ TEST(TestScalarNested, ListContainsNaN) { "list_contains", {ArrayFromJSON(list(int64()), "[[1], []]"), ScalarFromJSON(float64(), "NaN")}, ArrayFromJSON(boolean(), "[false, false]")); + + auto input = + ArrayFromJSON(list(float16()), "[[1.5, null], [NaN], [1.5, NaN], [], null]"); + CheckScalar("list_contains", {input, ScalarFromJSON(float16(), "NaN")}, + ArrayFromJSON(boolean(), "[false, true, true, false, null]")); + CheckScalar("list_contains", + {ArrayFromJSON(list(float32()), "[[1.5], [NaN]]"), + ScalarFromJSON(float16(), "NaN")}, + ArrayFromJSON(boolean(), "[false, true]")); } TEST(TestScalarNested, ListContainsValueTypes) { From 441fdb3910681ab86b4d0d155fea8eae63c39f0f Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 21:35:45 +0300 Subject: [PATCH 08/12] GH-33295: [C++] Only compare child values referenced by list views in list_contains --- .../arrow/compute/kernels/scalar_nested.cc | 21 ++++++++++++++++--- .../compute/kernels/scalar_nested_test.cc | 7 +++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc index 9367080d6658..68b628dd0547 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc @@ -17,7 +17,9 @@ // Vector kernels involving nested types +#include #include +#include #include "arrow/array/array_base.h" #include "arrow/array/builder_nested.h" #include "arrow/array/builder_primitive.h" @@ -630,7 +632,19 @@ std::pair GetListValuesRange(const ArraySpan& list) { return {list.offset * width, list.length * width}; } else if constexpr (is_list_view_type::value) { // List views may reference child values in any order - return {0, list.child_data[0].length}; + int64_t start = std::numeric_limits::max(); + int64_t end = 0; + for (int64_t i = 0; i < list.length; ++i) { + const auto [view_start, view_length] = GetListRange(list, i); + if (view_length > 0 && list.IsValid(i)) { + start = std::min(start, view_start); + end = std::max(end, view_start + view_length); + } + } + if (start >= end) { + return {0, 0}; + } + return {start, end - start}; } else { using offset_type = typename Type::offset_type; const offset_type* offsets = list.GetValues(1); @@ -660,9 +674,10 @@ Status EmitListMatches(KernelContext* ctx, const ArraySpan& list, const Datum& m out_arr->buffers[1].data, out_arr->offset, out_arr->length, [&] { bool found = false; if (list.IsValid(i)) { + // Empty list views may point outside of the compared values const auto [start, length] = get_match_range(i); - found = arrow::internal::CountSetBits(matches->data(), matches_offset + start, - length) > 0; + found = length > 0 && arrow::internal::CountSetBits( + matches->data(), matches_offset + start, length) > 0; } ++i; return found; diff --git a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc index b0f5811d55af..c9d32d1ac663 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc @@ -284,6 +284,13 @@ TEST(TestScalarNested, ListContainsListViewOutOfOrder) { ArrayFromJSON(boolean(), "[true, false, true, false, false]")); CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "1")}, ArrayFromJSON(boolean(), "[false, true, false, false, false]")); + // Only the child values referenced by the (sliced) views are compared + CheckScalar("list_contains", {input->Slice(2), ScalarFromJSON(int32(), "3")}, + ArrayFromJSON(boolean(), "[true, false, false]")); + CheckScalar("list_contains", {input->Slice(3), ScalarFromJSON(int32(), "1")}, + ArrayFromJSON(boolean(), "[false, false]")); + CheckScalar("list_contains", {input->Slice(4), ScalarFromJSON(int32(), "1")}, + ArrayFromJSON(boolean(), "[false]")); } TEST(TestScalarNested, ListContainsArrayOfValues) { From acc42f1e31593ed90336f267ceec3d49ac328271 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 21:37:13 +0300 Subject: [PATCH 09/12] GH-33295: [C++] Stop at the first match in list_contains --- .../arrow/compute/kernels/scalar_nested.cc | 47 +++++++++++-------- .../compute/kernels/scalar_nested_test.cc | 21 +++++++++ 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc index 68b628dd0547..177e52265d18 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc @@ -652,37 +652,42 @@ std::pair GetListValuesRange(const ArraySpan& list) { } } +// Whether any of the `length` match bits from `offset` is set and valid +bool AnyMatch(const ArraySpan& match, int64_t offset, int64_t length) { + // Empty list views may point outside of the compared values + if (length == 0) { + return false; + } + const uint8_t* validity = match.MayHaveNulls() ? match.buffers[0].data : nullptr; + arrow::internal::OptionalBinaryBitBlockCounter counter(match.buffers[1].data, + match.offset + offset, validity, + match.offset + offset, length); + for (int64_t position = 0; position < length;) { + const auto block = counter.NextAndBlock(); + if (block.popcount > 0) { + return true; + } + position += block.length; + } + return false; +} + // Emit whether each valid list has a match in the range of `match` returned by // `get_match_range(i)`. Null matches never count. template -Status EmitListMatches(KernelContext* ctx, const ArraySpan& list, const Datum& match, - ArraySpan* out_arr, GetMatchRange&& get_match_range) { - const ArrayData& match_data = *match.array(); - std::shared_ptr matches = match_data.buffers[1]; - int64_t matches_offset = match_data.offset; - if (match_data.MayHaveNulls()) { - ARROW_ASSIGN_OR_RAISE( - matches, arrow::internal::BitmapAnd( - ctx->memory_pool(), matches->data(), match_data.offset, - match_data.buffers[0]->data(), match_data.offset, match_data.length, - /*out_offset=*/0)); - matches_offset = 0; - } - +void EmitListMatches(const ArraySpan& list, const ArraySpan& match, ArraySpan* out_arr, + GetMatchRange&& get_match_range) { int64_t i = 0; arrow::internal::GenerateBitsUnrolled( out_arr->buffers[1].data, out_arr->offset, out_arr->length, [&] { bool found = false; if (list.IsValid(i)) { - // Empty list views may point outside of the compared values const auto [start, length] = get_match_range(i); - found = length > 0 && arrow::internal::CountSetBits( - matches->data(), matches_offset + start, length) > 0; + found = AnyMatch(match, start, length); } ++i; return found; }); - return Status::OK(); } // Search a single `value` in the referenced child values with one vectorized @@ -693,10 +698,11 @@ Status ListContainsScalar(KernelContext* ctx, const ArraySpan& list, const auto [values_start, values_length] = GetListValuesRange(list); auto values = list.child_data[0].ToArrayData()->Slice(values_start, values_length); ARROW_ASSIGN_OR_RAISE(Datum match, ListValuesMatch(ctx, std::move(values), value)); - return EmitListMatches(ctx, list, match, out_arr, [&](int64_t i) { + EmitListMatches(list, ArraySpan(*match.array()), out_arr, [&](int64_t i) { const auto [start, length] = GetListRange(list, i); return std::make_pair(start - values_start, length); }); + return Status::OK(); } // Gather the child values of each valid list next to copies of its value, compare @@ -731,11 +737,12 @@ Status ListContainsArray(KernelContext* ctx, const ArraySpan& list, // Lists are gathered in order, so each list's range follows the previous one int64_t match_start = 0; - return EmitListMatches(ctx, list, match, out_arr, [&](int64_t i) { + EmitListMatches(list, ArraySpan(*match.array()), out_arr, [&](int64_t i) { const int64_t length = GetListRange(list, i).second; match_start += length; return std::make_pair(match_start - length, length); }); + return Status::OK(); } template diff --git a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc index c9d32d1ac663..2759b41d9e14 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc @@ -160,6 +160,27 @@ TEST(TestScalarNested, ListContains) { ArrayFromJSON(boolean(), "[false]")); } +TEST(TestScalarNested, ListContainsLongLists) { + // Matches and nulls beyond the first bitmap words, at unaligned offsets + std::string values = "["; + for (int i = 0; i < 300; ++i) { + values += i == 0 ? "" : ", "; + values += i == 100 ? "null" : i == 290 ? "7" : "1"; + } + values += "]"; + ASSERT_OK_AND_ASSIGN(auto input, + ListArray::FromArrays(*ArrayFromJSON(int32(), "[0, 3, 150, 300]"), + *ArrayFromJSON(int32(), values))); + CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "7")}, + ArrayFromJSON(boolean(), "[false, false, true]")); + CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "null")}, + ArrayFromJSON(boolean(), "[false, true, false]")); + CheckScalar("list_contains", {input, ArrayFromJSON(int32(), "[7, 7, null]")}, + ArrayFromJSON(boolean(), "[false, false, false]")); + CheckScalar("list_contains", {input, ArrayFromJSON(int32(), "[1, null, 7]")}, + ArrayFromJSON(boolean(), "[true, true, true]")); +} + TEST(TestScalarNested, ListContainsNull) { // A null value matches lists holding a null auto input = ArrayFromJSON( From 2d1b61375e43c44c48a47bc42f3ccdea4c5de17c Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 21:56:45 +0300 Subject: [PATCH 10/12] GH-33295: [C++] Share one code path for all list types and values in list_contains --- .../arrow/compute/kernels/scalar_nested.cc | 203 ++++++------------ 1 file changed, 69 insertions(+), 134 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc index 177e52265d18..30cf508200ed 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc @@ -17,9 +17,7 @@ // Vector kernels involving nested types -#include #include -#include #include "arrow/array/array_base.h" #include "arrow/array/builder_nested.h" #include "arrow/array/builder_primitive.h" @@ -568,28 +566,21 @@ Status CheckNullValueType(KernelContext* ctx, const DataType& values_type, return equal->DispatchBest(&types).status(); } -// Like "is_in", and unlike "equal", a null value matches null values and a NaN -// value matches NaN values. -Result ListValuesMatch(KernelContext* ctx, std::shared_ptr values, - const std::shared_ptr& value) { - if (!value->is_valid) { - RETURN_NOT_OK(CheckNullValueType(ctx, *values->type, *value->type)); - return CallFunction("is_null", {std::move(values)}, ctx->exec_context()); - } - if (is_floating(values->type->id()) && IsNaN(*value)) { - return CallFunction("is_nan", {std::move(values)}, ctx->exec_context()); - } - return CallFunction("equal", {std::move(values), value}, ctx->exec_context()); -} - -// Element-wise version of the above, matching each of `values` with the value at the -// same index. -Result ListValuesMatch(KernelContext* ctx, std::shared_ptr values, - std::shared_ptr value) { +// Match `values` with a scalar `value`, or element-wise with an array of them. Like +// "is_in", and unlike "equal", a null value matches null values and a NaN value +// matches NaN values. +Result ListValuesMatch(KernelContext* ctx, const Datum& values, + const Datum& value) { ExecContext* exec_ctx = ctx->exec_context(); - if (value->GetNullCount() == value->length) { - RETURN_NOT_OK(CheckNullValueType(ctx, *values->type, *value->type)); - return CallFunction("is_null", {std::move(values)}, exec_ctx); + if (value.null_count() == value.length()) { + RETURN_NOT_OK(CheckNullValueType(ctx, *values.type(), *value.type())); + return CallFunction("is_null", {values}, exec_ctx); + } + if (value.is_scalar()) { + if (is_floating(values.type()->id()) && IsNaN(*value.scalar())) { + return CallFunction("is_nan", {values}, exec_ctx); + } + return CallFunction("equal", {values, value}, exec_ctx); } ARROW_ASSIGN_OR_RAISE(Datum match, CallFunction("equal", {values, value}, exec_ctx)); ARROW_ASSIGN_OR_RAISE(Datum values_null, CallFunction("is_null", {values}, exec_ctx)); @@ -597,7 +588,7 @@ Result ListValuesMatch(KernelContext* ctx, std::shared_ptr val ARROW_ASSIGN_OR_RAISE(Datum both_null, CallFunction("and", {values_null, value_null}, exec_ctx)); ARROW_ASSIGN_OR_RAISE(match, CallFunction("or_kleene", {match, both_null}, exec_ctx)); - if (is_floating(values->type->id()) && is_floating(value->type->id())) { + if (is_floating(values.type()->id()) && is_floating(value.type()->id())) { ARROW_ASSIGN_OR_RAISE(Datum values_nan, CallFunction("is_nan", {values}, exec_ctx)); ARROW_ASSIGN_OR_RAISE(Datum value_nan, CallFunction("is_nan", {value}, exec_ctx)); ARROW_ASSIGN_OR_RAISE(Datum both_nan, @@ -607,57 +598,41 @@ Result ListValuesMatch(KernelContext* ctx, std::shared_ptr val return match; } -// Returns the (child offset, length) pair of list i +// Returns the number of child values of list i, with none for null list views template -std::pair GetListRange(const ArraySpan& list, int64_t i) { +int64_t GetListLength(const ArraySpan& list, int64_t i) { if constexpr (std::is_same_v) { - const int64_t width = checked_cast(*list.type).list_size(); - return {(list.offset + i) * width, width}; + return checked_cast(*list.type).list_size(); + } else if constexpr (is_list_view_type::value) { + return list.IsValid(i) ? list.GetValues(2)[i] : 0; } else { - using offset_type = typename Type::offset_type; - const offset_type* offsets = list.GetValues(1); - if constexpr (is_list_view_type::value) { - return {offsets[i], list.GetValues(2)[i]}; - } else { - return {offsets[i], offsets[i + 1] - offsets[i]}; - } + const auto* offsets = list.GetValues(1); + return offsets[i + 1] - offsets[i]; } } -// Returns the (child offset, length) pair of the child values referenced by the lists +// Returns the child values of all lists, one list after the other template -std::pair GetListValuesRange(const ArraySpan& list) { +Result> GetListValues(KernelContext* ctx, + const ArraySpan& list) { if constexpr (std::is_same_v) { - const int64_t width = checked_cast(*list.type).list_size(); - return {list.offset * width, list.length * width}; + const int64_t width = GetListLength(list, 0); + return list.child_data[0].ToArrayData()->Slice(list.offset * width, + list.length * width); } else if constexpr (is_list_view_type::value) { // List views may reference child values in any order - int64_t start = std::numeric_limits::max(); - int64_t end = 0; - for (int64_t i = 0; i < list.length; ++i) { - const auto [view_start, view_length] = GetListRange(list, i); - if (view_length > 0 && list.IsValid(i)) { - start = std::min(start, view_start); - end = std::max(end, view_start + view_length); - } - } - if (start >= end) { - return {0, 0}; - } - return {start, end - start}; + typename TypeTraits::ArrayType list_view(list.ToArrayData()); + ARROW_ASSIGN_OR_RAISE(auto values, list_view.Flatten(ctx->memory_pool())); + return values->data(); } else { - using offset_type = typename Type::offset_type; - const offset_type* offsets = list.GetValues(1); - return {offsets[0], offsets[list.length] - offsets[0]}; + const auto* offsets = list.GetValues(1); + return list.child_data[0].ToArrayData()->Slice(offsets[0], + offsets[list.length] - offsets[0]); } } // Whether any of the `length` match bits from `offset` is set and valid bool AnyMatch(const ArraySpan& match, int64_t offset, int64_t length) { - // Empty list views may point outside of the compared values - if (length == 0) { - return false; - } const uint8_t* validity = match.MayHaveNulls() ? match.buffers[0].data : nullptr; arrow::internal::OptionalBinaryBitBlockCounter counter(match.buffers[1].data, match.offset + offset, validity, @@ -672,79 +647,24 @@ bool AnyMatch(const ArraySpan& match, int64_t offset, int64_t length) { return false; } -// Emit whether each valid list has a match in the range of `match` returned by -// `get_match_range(i)`. Null matches never count. -template -void EmitListMatches(const ArraySpan& list, const ArraySpan& match, ArraySpan* out_arr, - GetMatchRange&& get_match_range) { - int64_t i = 0; - arrow::internal::GenerateBitsUnrolled( - out_arr->buffers[1].data, out_arr->offset, out_arr->length, [&] { - bool found = false; - if (list.IsValid(i)) { - const auto [start, length] = get_match_range(i); - found = AnyMatch(match, start, length); - } - ++i; - return found; - }); -} - -// Search a single `value` in the referenced child values with one vectorized -// comparison, then check each list's range in the resulting bitmap. +// Repeat the value of each list for each of its child values template -Status ListContainsScalar(KernelContext* ctx, const ArraySpan& list, - const std::shared_ptr& value, ArraySpan* out_arr) { - const auto [values_start, values_length] = GetListValuesRange(list); - auto values = list.child_data[0].ToArrayData()->Slice(values_start, values_length); - ARROW_ASSIGN_OR_RAISE(Datum match, ListValuesMatch(ctx, std::move(values), value)); - EmitListMatches(list, ArraySpan(*match.array()), out_arr, [&](int64_t i) { - const auto [start, length] = GetListRange(list, i); - return std::make_pair(start - values_start, length); - }); - return Status::OK(); -} - -// Gather the child values of each valid list next to copies of its value, compare -// them element-wise, then check each list's range in the resulting bitmap. -template -Status ListContainsArray(KernelContext* ctx, const ArraySpan& list, - const ArraySpan& value, ArraySpan* out_arr) { - Int64Builder values_indices(ctx->memory_pool()); - Int64Builder value_indices(ctx->memory_pool()); +Result RepeatListValues(KernelContext* ctx, const ArraySpan& list, + int64_t values_length, const ArraySpan& value) { + Int64Builder indices(ctx->memory_pool()); + RETURN_NOT_OK(indices.Reserve(values_length)); for (int64_t i = 0; i < list.length; ++i) { - if (list.IsValid(i)) { - const auto [start, length] = GetListRange(list, i); - RETURN_NOT_OK(values_indices.Reserve(length)); - RETURN_NOT_OK(value_indices.Reserve(length)); - for (int64_t j = start; j < start + length; ++j) { - values_indices.UnsafeAppend(j); - value_indices.UnsafeAppend(i); - } + const int64_t length = GetListLength(list, i); + for (int64_t j = 0; j < length; ++j) { + indices.UnsafeAppend(i); } } - ARROW_ASSIGN_OR_RAISE(auto values_taken, values_indices.Finish()); - ARROW_ASSIGN_OR_RAISE(auto value_taken, value_indices.Finish()); - ExecContext* exec_ctx = ctx->exec_context(); - ARROW_ASSIGN_OR_RAISE( - Datum values, - CallFunction("take", {list.child_data[0].ToArrayData(), values_taken}, exec_ctx)); - ARROW_ASSIGN_OR_RAISE( - Datum repeated_value, - CallFunction("take", {value.ToArrayData(), value_taken}, exec_ctx)); - ARROW_ASSIGN_OR_RAISE(Datum match, - ListValuesMatch(ctx, values.array(), repeated_value.array())); - - // Lists are gathered in order, so each list's range follows the previous one - int64_t match_start = 0; - EmitListMatches(list, ArraySpan(*match.array()), out_arr, [&](int64_t i) { - const int64_t length = GetListRange(list, i).second; - match_start += length; - return std::make_pair(match_start - length, length); - }); - return Status::OK(); + ARROW_ASSIGN_OR_RAISE(auto repeat_indices, indices.Finish()); + return CallFunction("take", {value.ToArrayData(), repeat_indices}, ctx->exec_context()); } +// Match the child values of all lists with a single vectorized comparison, then check +// each list's range in the resulting bitmap. template Status ListContains(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) { if (batch.length == 0) { @@ -771,16 +691,31 @@ Status ListContains(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) } out_arr->null_count = list.null_count; - const ExecValue& value = batch[1]; - if (value.is_scalar()) { - return ListContainsScalar(ctx, list, value.scalar->GetSharedPtr(), out_arr); - } - if (value.array.length == 1) { + ARROW_ASSIGN_OR_RAISE(auto values, GetListValues(ctx, list)); + Datum value; + if (batch[1].is_scalar()) { + value = batch[1].scalar->GetSharedPtr(); + } else if (batch[1].array.length == 1) { // Also covers scalar inputs, which are promoted to arrays - ARROW_ASSIGN_OR_RAISE(auto single_value, value.array.ToArray()->GetScalar(0)); - return ListContainsScalar(ctx, list, single_value, out_arr); + ARROW_ASSIGN_OR_RAISE(value, batch[1].array.ToArray()->GetScalar(0)); + } else { + ARROW_ASSIGN_OR_RAISE( + value, RepeatListValues(ctx, list, values->length, batch[1].array)); } - return ListContainsArray(ctx, list, value.array, out_arr); + ARROW_ASSIGN_OR_RAISE(Datum match, ListValuesMatch(ctx, values, value)); + + const ArraySpan match_span(*match.array()); + int64_t start = 0; + int64_t i = 0; + arrow::internal::GenerateBitsUnrolled( + out_arr->buffers[1].data, out_arr->offset, out_arr->length, [&] { + const int64_t length = GetListLength(list, i); + const bool found = list.IsValid(i) && AnyMatch(match_span, start, length); + start += length; + ++i; + return found; + }); + return Status::OK(); } void AddListContainsKernels(ScalarFunction* func) { From 95d6b8719e88be04bec042941b5b665125071348 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Mon, 28 Sep 2026 21:58:28 +0300 Subject: [PATCH 11/12] GH-33295: [C++] Condense list_contains tests --- .../compute/kernels/scalar_nested_test.cc | 245 +++++++----------- 1 file changed, 100 insertions(+), 145 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc index 2759b41d9e14..143a7f35bcf5 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc @@ -128,36 +128,35 @@ TEST(TestScalarNested, ListElementInvalid) { Raises(StatusCode::Invalid)); } +void CheckListContains(Datum lists, Datum value, const std::string& expected) { + CheckScalar("list_contains", {std::move(lists), std::move(value)}, + ArrayFromJSON(boolean(), expected)); +} + TEST(TestScalarNested, ListContains) { auto sample = "[[7, 5, 81], [6, null, 4, 7, 8], [], [5], null, [null]]"; for (auto ty : NumericTypes()) { for (auto list_type : {list(ty), large_list(ty), list_view(ty), large_list_view(ty)}) { auto input = ArrayFromJSON(list_type, sample); - CheckScalar("list_contains", {input, ScalarFromJSON(ty, "5")}, - ArrayFromJSON(boolean(), "[true, false, false, true, null, false]")); - CheckScalar("list_contains", {input, ScalarFromJSON(ty, "7")}, - ArrayFromJSON(boolean(), "[true, true, false, false, null, false]")); - CheckScalar("list_contains", {input, ScalarFromJSON(ty, "null")}, - ArrayFromJSON(boolean(), "[false, true, false, false, null, true]")); - CheckScalar("list_contains", - {ArrayFromJSON(list_type, "[]"), ScalarFromJSON(ty, "5")}, - ArrayFromJSON(boolean(), "[]")); + CheckListContains(input, ScalarFromJSON(ty, "5"), + "[true, false, false, true, null, false]"); + CheckListContains(input, ScalarFromJSON(ty, "7"), + "[true, true, false, false, null, false]"); + CheckListContains(input, ScalarFromJSON(ty, "null"), + "[false, true, false, false, null, true]"); + CheckListContains(ArrayFromJSON(list_type, "[]"), ScalarFromJSON(ty, "5"), "[]"); } } auto input = ArrayFromJSON(list(utf8()), R"([["a", "b"], ["a", "c"], ["b", "c", "d"]])"); - CheckScalar("list_contains", {input, ScalarFromJSON(utf8(), R"("a")")}, - ArrayFromJSON(boolean(), "[true, true, false]")); - CheckScalar("list_contains", {input->Slice(1), ScalarFromJSON(utf8(), R"("b")")}, - ArrayFromJSON(boolean(), "[false, true]")); + CheckListContains(input, ScalarFromJSON(utf8(), R"("a")"), "[true, true, false]"); + CheckListContains(input->Slice(1), ScalarFromJSON(utf8(), R"("b")"), "[false, true]"); // All-null lists contain no non-null value - CheckScalar( - "list_contains", - {ArrayFromJSON(list(int64()), "[[null, null]]"), ScalarFromJSON(int64(), "1")}, - ArrayFromJSON(boolean(), "[false]")); + CheckListContains(ArrayFromJSON(list(int64()), "[[null, null]]"), + ScalarFromJSON(int64(), "1"), "[false]"); } TEST(TestScalarNested, ListContainsLongLists) { @@ -171,42 +170,35 @@ TEST(TestScalarNested, ListContainsLongLists) { ASSERT_OK_AND_ASSIGN(auto input, ListArray::FromArrays(*ArrayFromJSON(int32(), "[0, 3, 150, 300]"), *ArrayFromJSON(int32(), values))); - CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "7")}, - ArrayFromJSON(boolean(), "[false, false, true]")); - CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "null")}, - ArrayFromJSON(boolean(), "[false, true, false]")); - CheckScalar("list_contains", {input, ArrayFromJSON(int32(), "[7, 7, null]")}, - ArrayFromJSON(boolean(), "[false, false, false]")); - CheckScalar("list_contains", {input, ArrayFromJSON(int32(), "[1, null, 7]")}, - ArrayFromJSON(boolean(), "[true, true, true]")); + CheckListContains(input, ScalarFromJSON(int32(), "7"), "[false, false, true]"); + CheckListContains(input, ScalarFromJSON(int32(), "null"), "[false, true, false]"); + CheckListContains(input, ArrayFromJSON(int32(), "[7, 7, null]"), + "[false, false, false]"); + CheckListContains(input, ArrayFromJSON(int32(), "[1, null, 7]"), "[true, true, true]"); } TEST(TestScalarNested, ListContainsNull) { // A null value matches lists holding a null auto input = ArrayFromJSON( list(int32()), "[[1, null], [null], [1, 2], [1, 1], [2, 1, null, 1], [], null]"); - CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "null")}, - ArrayFromJSON(boolean(), "[true, true, false, false, true, false, null]")); - CheckScalar("list_contains", {input, MakeNullScalar(null())}, - ArrayFromJSON(boolean(), "[true, true, false, false, true, false, null]")); - CheckScalar("list_contains", {input, ScalarFromJSON(int64(), "null")}, - ArrayFromJSON(boolean(), "[true, true, false, false, true, false, null]")); - CheckScalar("list_contains", - {ArrayFromJSON(list(int32()), "[[]]"), ScalarFromJSON(int32(), "null")}, - ArrayFromJSON(boolean(), "[false]")); + for (auto value : {ScalarFromJSON(int32(), "null"), MakeNullScalar(null()), + ScalarFromJSON(int64(), "null")}) { + CheckListContains(input, value, "[true, true, false, false, true, false, null]"); + } + CheckListContains(ArrayFromJSON(list(int32()), "[[]]"), ScalarFromJSON(int32(), "null"), + "[false]"); input = ArrayFromJSON(list(utf8()), R"([["x", null], [null], ["x", "y"], [], null])"); - CheckScalar("list_contains", {input, ScalarFromJSON(utf8(), "null")}, - ArrayFromJSON(boolean(), "[true, true, false, false, null]")); + CheckListContains(input, ScalarFromJSON(utf8(), "null"), + "[true, true, false, false, null]"); input = ArrayFromJSON(list(list(int32())), "[[[1], null], [null], [[1], [null]], [], null]"); - CheckScalar("list_contains", {input, MakeNullScalar(list(int32()))}, - ArrayFromJSON(boolean(), "[true, true, false, false, null]")); + CheckListContains(input, MakeNullScalar(list(int32())), + "[true, true, false, false, null]"); input = ArrayFromJSON(list(null()), "[[null], [], null]"); - CheckScalar("list_contains", {input, MakeNullScalar(null())}, - ArrayFromJSON(boolean(), "[true, false, null]")); + CheckListContains(input, MakeNullScalar(null()), "[true, false, null]"); } TEST(TestScalarNested, ListContainsNaN) { @@ -214,25 +206,21 @@ TEST(TestScalarNested, ListContainsNaN) { for (auto ty : {float32(), float64()}) { auto input = ArrayFromJSON(list(ty), "[[1.5, null], [NaN], [1.5, NaN], [], null]"); for (auto value_ty : {float32(), float64()}) { - CheckScalar("list_contains", {input, ScalarFromJSON(value_ty, "NaN")}, - ArrayFromJSON(boolean(), "[false, true, true, false, null]")); - CheckScalar("list_contains", {input, ScalarFromJSON(value_ty, "1.5")}, - ArrayFromJSON(boolean(), "[true, false, true, false, null]")); + CheckListContains(input, ScalarFromJSON(value_ty, "NaN"), + "[false, true, true, false, null]"); + CheckListContains(input, ScalarFromJSON(value_ty, "1.5"), + "[true, false, true, false, null]"); } } - CheckScalar( - "list_contains", - {ArrayFromJSON(list(int64()), "[[1], []]"), ScalarFromJSON(float64(), "NaN")}, - ArrayFromJSON(boolean(), "[false, false]")); + CheckListContains(ArrayFromJSON(list(int64()), "[[1], []]"), + ScalarFromJSON(float64(), "NaN"), "[false, false]"); auto input = ArrayFromJSON(list(float16()), "[[1.5, null], [NaN], [1.5, NaN], [], null]"); - CheckScalar("list_contains", {input, ScalarFromJSON(float16(), "NaN")}, - ArrayFromJSON(boolean(), "[false, true, true, false, null]")); - CheckScalar("list_contains", - {ArrayFromJSON(list(float32()), "[[1.5], [NaN]]"), - ScalarFromJSON(float16(), "NaN")}, - ArrayFromJSON(boolean(), "[false, true]")); + CheckListContains(input, ScalarFromJSON(float16(), "NaN"), + "[false, true, true, false, null]"); + CheckListContains(ArrayFromJSON(list(float32()), "[[1.5], [NaN]]"), + ScalarFromJSON(float16(), "NaN"), "[false, true]"); } TEST(TestScalarNested, ListContainsValueTypes) { @@ -240,10 +228,8 @@ TEST(TestScalarNested, ListContainsValueTypes) { const std::string& y) { auto input = ArrayFromJSON( list(ty), "[[" + x + ", null], [" + y + "], [" + x + ", " + y + "], [], null]"); - CheckScalar("list_contains", {input, ScalarFromJSON(ty, x)}, - ArrayFromJSON(boolean(), "[true, false, true, false, null]")); - CheckScalar("list_contains", {input, ScalarFromJSON(ty, y)}, - ArrayFromJSON(boolean(), "[false, true, true, false, null]")); + CheckListContains(input, ScalarFromJSON(ty, x), "[true, false, true, false, null]"); + CheckListContains(input, ScalarFromJSON(ty, y), "[false, true, true, false, null]"); }; check(utf8(), R"("x")", R"("y")"); check(large_binary(), R"("x")", R"("y")"); @@ -256,23 +242,18 @@ TEST(TestScalarNested, ListContainsValueTypes) { TEST(TestScalarNested, ListContainsImplicitCast) { auto input = ArrayFromJSON(list(int64()), "[[2, 2, 3, null, null], null, [], [null], [1], [0, -1]]"); - CheckScalar("list_contains", {input, ScalarFromJSON(float64(), "2.0")}, - ArrayFromJSON(boolean(), "[true, null, false, false, false, false]")); - CheckScalar("list_contains", {input, ScalarFromJSON(float64(), "1.5")}, - ArrayFromJSON(boolean(), "[false, null, false, false, false, false]")); + CheckListContains(input, ScalarFromJSON(float64(), "2.0"), + "[true, null, false, false, false, false]"); + CheckListContains(input, ScalarFromJSON(float64(), "1.5"), + "[false, null, false, false, false, false]"); // No overflow of the list values - input = ArrayFromJSON(list(int8()), "[[44, 1], null, [], [null]]"); - CheckScalar("list_contains", {input, ScalarFromJSON(int64(), "300")}, - ArrayFromJSON(boolean(), "[false, null, false, false]")); - - input = ArrayFromJSON(list(float64()), "[[1.0, 2.0], [3.0], null, []]"); - CheckScalar("list_contains", {input, ScalarFromJSON(int64(), "1")}, - ArrayFromJSON(boolean(), "[true, false, null, false]")); - - input = ArrayFromJSON(list(decimal128(38, 2)), R"([["1.50"], ["2.50"]])"); - CheckScalar("list_contains", {input, ScalarFromJSON(decimal128(3, 2), R"("1.50")")}, - ArrayFromJSON(boolean(), "[true, false]")); + CheckListContains(ArrayFromJSON(list(int8()), "[[44, 1], null, [], [null]]"), + ScalarFromJSON(int64(), "300"), "[false, null, false, false]"); + CheckListContains(ArrayFromJSON(list(float64()), "[[1.0, 2.0], [3.0], null, []]"), + ScalarFromJSON(int64(), "1"), "[true, false, null, false]"); + CheckListContains(ArrayFromJSON(list(decimal128(38, 2)), R"([["1.50"], ["2.50"]])"), + ScalarFromJSON(decimal128(3, 2), R"("1.50")"), "[true, false]"); } TEST(TestScalarNested, ListContainsChunked) { @@ -289,97 +270,71 @@ TEST(TestScalarNested, ListContainsChunked) { TEST(TestScalarNested, ListContainsFixedSizeList) { auto input = ArrayFromJSON(fixed_size_list(int32(), 2), "[[1, 2], [3, null], null, [2, 4], [null, null]]"); - CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "2")}, - ArrayFromJSON(boolean(), "[true, false, null, true, false]")); - CheckScalar("list_contains", {input->Slice(2), ScalarFromJSON(int32(), "4")}, - ArrayFromJSON(boolean(), "[null, true, false]")); + CheckListContains(input, ScalarFromJSON(int32(), "2"), + "[true, false, null, true, false]"); + CheckListContains(input->Slice(2), ScalarFromJSON(int32(), "4"), "[null, true, false]"); } TEST(TestScalarNested, ListContainsListViewOutOfOrder) { // Views: [3, 4], [1, 2], [2, 3], [4], [] - auto values = ArrayFromJSON(int32(), "[1, 2, 3, 4]"); - auto offsets = ArrayFromJSON(int32(), "[2, 0, 1, 3, 0]"); - auto sizes = ArrayFromJSON(int32(), "[2, 2, 2, 1, 0]"); - ASSERT_OK_AND_ASSIGN(auto input, ListViewArray::FromArrays(*offsets, *sizes, *values)); - CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "3")}, - ArrayFromJSON(boolean(), "[true, false, true, false, false]")); - CheckScalar("list_contains", {input, ScalarFromJSON(int32(), "1")}, - ArrayFromJSON(boolean(), "[false, true, false, false, false]")); + ASSERT_OK_AND_ASSIGN( + auto input, ListViewArray::FromArrays(*ArrayFromJSON(int32(), "[2, 0, 1, 3, 0]"), + *ArrayFromJSON(int32(), "[2, 2, 2, 1, 0]"), + *ArrayFromJSON(int32(), "[1, 2, 3, 4]"))); + CheckListContains(input, ScalarFromJSON(int32(), "3"), + "[true, false, true, false, false]"); + CheckListContains(input, ScalarFromJSON(int32(), "1"), + "[false, true, false, false, false]"); + CheckListContains(input, ArrayFromJSON(int32(), "[3, 1, 1, 4, 1]"), + "[true, true, false, true, false]"); // Only the child values referenced by the (sliced) views are compared - CheckScalar("list_contains", {input->Slice(2), ScalarFromJSON(int32(), "3")}, - ArrayFromJSON(boolean(), "[true, false, false]")); - CheckScalar("list_contains", {input->Slice(3), ScalarFromJSON(int32(), "1")}, - ArrayFromJSON(boolean(), "[false, false]")); - CheckScalar("list_contains", {input->Slice(4), ScalarFromJSON(int32(), "1")}, - ArrayFromJSON(boolean(), "[false]")); + CheckListContains(input->Slice(2), ScalarFromJSON(int32(), "3"), + "[true, false, false]"); + CheckListContains(input->Slice(3), ScalarFromJSON(int32(), "1"), "[false, false]"); + CheckListContains(input->Slice(4), ScalarFromJSON(int32(), "1"), "[false]"); } TEST(TestScalarNested, ListContainsArrayOfValues) { // Each list is searched for the value at the same index for (auto list_type : {list(int32()), large_list(int32()), list_view(int32()), large_list_view(int32())}) { - auto input = - ArrayFromJSON(list_type, "[[1, 2], [3, null], [], null, [4, 5], [null], [6]]"); - CheckScalar("list_contains", - {input, ArrayFromJSON(int32(), "[2, null, 1, 3, 4, null, 7]")}, - ArrayFromJSON(boolean(), "[true, true, false, null, true, true, false]")); + CheckListContains( + ArrayFromJSON(list_type, "[[1, 2], [3, null], [], null, [4, 5], [null], [6]]"), + ArrayFromJSON(int32(), "[2, null, 1, 3, 4, null, 7]"), + "[true, true, false, null, true, true, false]"); } - CheckScalar("list_contains", - {ArrayFromJSON(fixed_size_list(int32(), 2), - "[[1, 2], [3, null], null, [4, 5], [null, 6]]"), - ArrayFromJSON(int32(), "[2, null, 1, 5, 7]")}, - ArrayFromJSON(boolean(), "[true, true, null, true, false]")); - - CheckScalar("list_contains", - {ArrayFromJSON(list(int64()), "[[1, 2], [3], [4]]"), - ArrayFromJSON(float64(), "[2.0, 3.5, 4.0]")}, - ArrayFromJSON(boolean(), "[true, false, true]")); - CheckScalar("list_contains", - {ArrayFromJSON(list(float64()), "[[1.5, NaN], [NaN], [1.5], [null]]"), - ArrayFromJSON(float64(), "[NaN, 1.5, NaN, NaN]")}, - ArrayFromJSON(boolean(), "[true, false, false, false]")); - CheckScalar("list_contains", - {ArrayFromJSON(list(int32()), "[[1, null], [2], []]"), - ArrayFromJSON(null(), "[null, null, null]")}, - ArrayFromJSON(boolean(), "[true, false, false]")); - - // Views: [3, 4], [1, 2], [2, 3], [4], [] - ASSERT_OK_AND_ASSIGN( - auto input, ListViewArray::FromArrays(*ArrayFromJSON(int32(), "[2, 0, 1, 3, 0]"), - *ArrayFromJSON(int32(), "[2, 2, 2, 1, 0]"), - *ArrayFromJSON(int32(), "[1, 2, 3, 4]"))); - CheckScalar("list_contains", {input, ArrayFromJSON(int32(), "[3, 1, 1, 4, 1]")}, - ArrayFromJSON(boolean(), "[true, true, false, true, false]")); + CheckListContains(ArrayFromJSON(fixed_size_list(int32(), 2), + "[[1, 2], [3, null], null, [4, 5], [null, 6]]"), + ArrayFromJSON(int32(), "[2, null, 1, 5, 7]"), + "[true, true, null, true, false]"); + CheckListContains(ArrayFromJSON(list(int64()), "[[1, 2], [3], [4]]"), + ArrayFromJSON(float64(), "[2.0, 3.5, 4.0]"), "[true, false, true]"); + CheckListContains(ArrayFromJSON(list(float64()), "[[1.5, NaN], [NaN], [1.5], [null]]"), + ArrayFromJSON(float64(), "[NaN, 1.5, NaN, NaN]"), + "[true, false, false, false]"); + CheckListContains(ArrayFromJSON(list(int32()), "[[1, null], [2], []]"), + ArrayFromJSON(null(), "[null, null, null]"), "[true, false, false]"); // A scalar list is searched for each value - CheckScalar("list_contains", - {ScalarFromJSON(list(int32()), "[1, null]"), - ArrayFromJSON(int32(), "[1, 2, null]")}, - ArrayFromJSON(boolean(), "[true, false, true]")); - CheckScalar("list_contains", - {ScalarFromJSON(fixed_size_list(int32(), 2), "[1, 2]"), - ArrayFromJSON(int32(), "[2, 3]")}, - ArrayFromJSON(boolean(), "[true, false]")); - CheckScalar("list_contains", - {ScalarFromJSON(list(int32()), "null"), ArrayFromJSON(int32(), "[1, 2]")}, - ArrayFromJSON(boolean(), "[null, null]")); + CheckListContains(ScalarFromJSON(list(int32()), "[1, null]"), + ArrayFromJSON(int32(), "[1, 2, null]"), "[true, false, true]"); + CheckListContains(ScalarFromJSON(fixed_size_list(int32(), 2), "[1, 2]"), + ArrayFromJSON(int32(), "[2, 3]"), "[true, false]"); + CheckListContains(ScalarFromJSON(list(int32()), "null"), + ArrayFromJSON(int32(), "[1, 2]"), "[null, null]"); } TEST(TestScalarNested, ListContainsInvalid) { auto input = ArrayFromJSON(list(int32()), "[[1, 2], [3]]"); - EXPECT_THAT( - CallFunction("list_contains", {input, ArrayFromJSON(utf8(), R"(["a", "b"])")}), - Raises(StatusCode::NotImplemented)); - EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(utf8(), R"("a")")}), - Raises(StatusCode::NotImplemented)); - EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(boolean(), "true")}), - Raises(StatusCode::NotImplemented)); // Null values must be comparable too - EXPECT_THAT(CallFunction("list_contains", {input, ScalarFromJSON(utf8(), "null")}), - Raises(StatusCode::NotImplemented)); - EXPECT_THAT( - CallFunction("list_contains", {input, ArrayFromJSON(utf8(), "[null, null]")}), - Raises(StatusCode::NotImplemented)); + for (Datum value : + {Datum(ArrayFromJSON(utf8(), R"(["a", "b"])")), + Datum(ScalarFromJSON(utf8(), R"("a")")), Datum(ScalarFromJSON(boolean(), "true")), + Datum(ScalarFromJSON(utf8(), "null")), + Datum(ArrayFromJSON(utf8(), "[null, null]"))}) { + EXPECT_THAT(CallFunction("list_contains", {input, value}), + Raises(StatusCode::NotImplemented)); + } } using VarLenListLikeTypeFactory = From 972bc4cad4a0695285a6d7a049f809f4f2fe25fa Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Wed, 30 Sep 2026 09:27:20 +0300 Subject: [PATCH 12/12] GH-33295: [C++] Speed up the list_contains scan for short lists --- .../arrow/compute/kernels/scalar_nested.cc | 64 +++++++++++++------ .../compute/kernels/scalar_nested_test.cc | 22 +++++++ 2 files changed, 67 insertions(+), 19 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_nested.cc b/cpp/src/arrow/compute/kernels/scalar_nested.cc index 30cf508200ed..8990ec5b463e 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested.cc @@ -33,6 +33,7 @@ #include "arrow/util/float16.h" #include "arrow/util/logging_internal.h" #include "arrow/util/string.h" +#include "arrow/util/ubsan.h" #include "arrow/util/unreachable.h" namespace arrow { @@ -598,16 +599,20 @@ Result ListValuesMatch(KernelContext* ctx, const Datum& values, return match; } -// Returns the number of child values of list i, with none for null list views +// Returns a function giving the number of child values of list i, with none for null +// list views template -int64_t GetListLength(const ArraySpan& list, int64_t i) { +auto GetListLengths(const ArraySpan& list) { if constexpr (std::is_same_v) { - return checked_cast(*list.type).list_size(); + const int64_t width = checked_cast(*list.type).list_size(); + return [width](int64_t) { return width; }; } else if constexpr (is_list_view_type::value) { - return list.IsValid(i) ? list.GetValues(2)[i] : 0; + const auto* sizes = list.GetValues(2); + return + [&list, sizes](int64_t i) -> int64_t { return list.IsValid(i) ? sizes[i] : 0; }; } else { const auto* offsets = list.GetValues(1); - return offsets[i + 1] - offsets[i]; + return [offsets](int64_t i) -> int64_t { return offsets[i + 1] - offsets[i]; }; } } @@ -616,7 +621,7 @@ template Result> GetListValues(KernelContext* ctx, const ArraySpan& list) { if constexpr (std::is_same_v) { - const int64_t width = GetListLength(list, 0); + const int64_t width = GetListLengths(list)(0); return list.child_data[0].ToArrayData()->Slice(list.offset * width, list.length * width); } else if constexpr (is_list_view_type::value) { @@ -631,14 +636,21 @@ Result> GetListValues(KernelContext* ctx, } } -// Whether any of the `length` match bits from `offset` is set and valid -bool AnyMatch(const ArraySpan& match, int64_t offset, int64_t length) { - const uint8_t* validity = match.MayHaveNulls() ? match.buffers[0].data : nullptr; - arrow::internal::OptionalBinaryBitBlockCounter counter(match.buffers[1].data, - match.offset + offset, validity, - match.offset + offset, length); +// Whether any of the `length` bits from `offset` is set in `bits`, a bitmap of `size` +// bytes +bool AnySet(const uint8_t* bits, int64_t size, int64_t offset, int64_t length) { + const int64_t byte_offset = offset / 8; + const int64_t bit_offset = offset % 8; + if (bit_offset + length <= 64 && byte_offset + 8 <= size) { + // Short ranges fit in a single word + const uint64_t word = + bit_util::FromLittleEndian(util::SafeLoadAs(bits + byte_offset)); + const uint64_t mask = length == 64 ? ~uint64_t{0} : (uint64_t{1} << length) - 1; + return ((word >> bit_offset) & mask) != 0; + } + arrow::internal::BitBlockCounter counter(bits, offset, length); for (int64_t position = 0; position < length;) { - const auto block = counter.NextAndBlock(); + const auto block = counter.NextWord(); if (block.popcount > 0) { return true; } @@ -651,10 +663,11 @@ bool AnyMatch(const ArraySpan& match, int64_t offset, int64_t length) { template Result RepeatListValues(KernelContext* ctx, const ArraySpan& list, int64_t values_length, const ArraySpan& value) { + const auto list_length = GetListLengths(list); Int64Builder indices(ctx->memory_pool()); RETURN_NOT_OK(indices.Reserve(values_length)); for (int64_t i = 0; i < list.length; ++i) { - const int64_t length = GetListLength(list, i); + const int64_t length = list_length(i); for (int64_t j = 0; j < length; ++j) { indices.UnsafeAppend(i); } @@ -704,15 +717,28 @@ Status ListContains(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) } ARROW_ASSIGN_OR_RAISE(Datum match, ListValuesMatch(ctx, values, value)); - const ArraySpan match_span(*match.array()); - int64_t start = 0; + // Null matches never count + const ArrayData& match_data = *match.array(); + std::shared_ptr matches = match_data.buffers[1]; + int64_t matches_offset = match_data.offset; + if (match_data.MayHaveNulls()) { + ARROW_ASSIGN_OR_RAISE( + matches, arrow::internal::BitmapAnd( + ctx->memory_pool(), matches->data(), match_data.offset, + match_data.buffers[0]->data(), match_data.offset, match_data.length, + /*out_offset=*/0)); + matches_offset = 0; + } + + // The output bits of null lists don't matter, so they are not special-cased + const auto list_length = GetListLengths(list); + int64_t start = matches_offset; int64_t i = 0; arrow::internal::GenerateBitsUnrolled( out_arr->buffers[1].data, out_arr->offset, out_arr->length, [&] { - const int64_t length = GetListLength(list, i); - const bool found = list.IsValid(i) && AnyMatch(match_span, start, length); + const int64_t length = list_length(i++); + const bool found = AnySet(matches->data(), matches->size(), start, length); start += length; - ++i; return found; }); return Status::OK(); diff --git a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc index 143a7f35bcf5..e4d6d69085ed 100644 --- a/cpp/src/arrow/compute/kernels/scalar_nested_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_nested_test.cc @@ -177,6 +177,28 @@ TEST(TestScalarNested, ListContainsLongLists) { CheckListContains(input, ArrayFromJSON(int32(), "[1, null, 7]"), "[true, true, true]"); } +TEST(TestScalarNested, ListContainsManyShortLists) { + // Short lists at every bit alignment, with null lists and null values + std::string lists = "["; + std::string contains_5 = "["; + std::string contains_null = "["; + for (int i = 0; i < 200; ++i) { + const std::string sep = i == 0 ? "" : ", "; + const bool is_null = i % 11 == 0; + lists += sep + (is_null ? "null" + : i % 7 == 0 ? "[5, 1]" + : i % 3 == 0 ? "[null, 2, 3]" + : "[1]"); + contains_5 += sep + (is_null ? "null" : i % 7 == 0 ? "true" : "false"); + contains_null += sep + (is_null ? "null" + : i % 7 != 0 && i % 3 == 0 ? "true" + : "false"); + } + auto input = ArrayFromJSON(list(int32()), lists + "]"); + CheckListContains(input, ScalarFromJSON(int32(), "5"), contains_5 + "]"); + CheckListContains(input, ScalarFromJSON(int32(), "null"), contains_null + "]"); +} + TEST(TestScalarNested, ListContainsNull) { // A null value matches lists holding a null auto input = ArrayFromJSON(