Skip to content

Commit 4b94187

Browse files
fangchenliclaude
fangchenli
and
claude
committed
GH-50478: [C++][Compute] Support string_view/binary_view in scalar string predicate kernels
Add STRING_VIEW/BINARY_VIEW support to the scalar string predicate and measurement kernels that read a string and emit a fixed-width type: match_substring(+regex), match_like, starts_with, ends_with, find_substring(+regex), count_substring(+regex), binary_length, utf8_length, string_is_ascii, and the ascii_is_*/utf8_is_* classifiers. The matcher/predicate logic already operates on std::string_view, so the change feeds view elements into the existing logic: MatchSubstringImpl gets a per-element ArrayIterator branch for the view layout (the packed-offset fast path stays for base binary/string), and the applicator-based kernels only need registration entries plus a StringOffsetType helper mapping views to Int32 output. Kernels that emit strings/lists are left for a follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 5815dc0 commit 4b94187

6 files changed

Lines changed: 265 additions & 32 deletions

File tree

‎cpp/src/arrow/compute/kernels/scalar_string_ascii.cc‎

Lines changed: 100 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -889,6 +889,12 @@ void AddAsciiStringLength(FunctionRegistry* registry) {
889889
ty);
890890
DCHECK_OK(func->AddKernel({ty}, int64(), std::move(exec)));
891891
}
892+
// View element length is int32-sized, so both view types emit int32.
893+
for (const auto& ty : BinaryViewTypes()) {
894+
auto exec = GenerateVarBinaryViewBase<applicator::ScalarUnaryNotNull, Int32Type,
895+
BinaryLength>(*ty);
896+
DCHECK_OK(func->AddKernel({ty}, int32(), std::move(exec)));
897+
}
892898
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
893899
BinaryLength::FixedSizeExec));
894900
DCHECK_OK(registry->AddFunction(std::move(func)));
@@ -1337,27 +1343,45 @@ struct RegexSubstringMatcher {
13371343

13381344
template <typename Type, typename Matcher>
13391345
struct MatchSubstringImpl {
1340-
using offset_type = typename Type::offset_type;
1341-
13421346
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out,
13431347
const Matcher* matcher) {
1344-
StringBoolTransform<Type>(
1345-
ctx, batch,
1346-
[&matcher](const void* raw_offsets, const uint8_t* data, int64_t length,
1347-
int64_t output_offset, uint8_t* output) {
1348-
const offset_type* offsets = reinterpret_cast<const offset_type*>(raw_offsets);
1349-
FirstTimeBitmapWriter bitmap_writer(output, output_offset, length);
1350-
for (int64_t i = 0; i < length; ++i) {
1351-
const char* current_data = reinterpret_cast<const char*>(data + offsets[i]);
1352-
int64_t current_length = offsets[i + 1] - offsets[i];
1353-
if (matcher->Match(std::string_view(current_data, current_length))) {
1354-
bitmap_writer.Set();
1348+
if constexpr (is_binary_view_like_type<Type>::value) {
1349+
// Views have no packed offset buffer, so iterate per element. Null slots
1350+
// are matched too (masked by output validity); reading their zero-filled
1351+
// view header is safe, as StringPredicateFunctor also assumes.
1352+
const ArraySpan& input = batch[0].array;
1353+
ArraySpan* out_arr = out->array_span_mutable();
1354+
ArrayIterator<Type> input_it(input);
1355+
FirstTimeBitmapWriter bitmap_writer(out_arr->buffers[1].data, out_arr->offset,
1356+
input.length);
1357+
for (int64_t i = 0; i < input.length; ++i) {
1358+
if (matcher->Match(input_it())) {
1359+
bitmap_writer.Set();
1360+
}
1361+
bitmap_writer.Next();
1362+
}
1363+
bitmap_writer.Finish();
1364+
} else {
1365+
using offset_type = typename Type::offset_type;
1366+
StringBoolTransform<Type>(
1367+
ctx, batch,
1368+
[&matcher](const void* raw_offsets, const uint8_t* data, int64_t length,
1369+
int64_t output_offset, uint8_t* output) {
1370+
const offset_type* offsets =
1371+
reinterpret_cast<const offset_type*>(raw_offsets);
1372+
FirstTimeBitmapWriter bitmap_writer(output, output_offset, length);
1373+
for (int64_t i = 0; i < length; ++i) {
1374+
const char* current_data = reinterpret_cast<const char*>(data + offsets[i]);
1375+
int64_t current_length = offsets[i + 1] - offsets[i];
1376+
if (matcher->Match(std::string_view(current_data, current_length))) {
1377+
bitmap_writer.Set();
1378+
}
1379+
bitmap_writer.Next();
13551380
}
1356-
bitmap_writer.Next();
1357-
}
1358-
bitmap_writer.Finish();
1359-
},
1360-
out);
1381+
bitmap_writer.Finish();
1382+
},
1383+
out);
1384+
}
13611385
return Status::OK();
13621386
}
13631387
};
@@ -1611,6 +1635,19 @@ const FunctionDoc match_like_doc(
16111635
{"strings"}, "MatchSubstringOptions", /*options_required=*/true);
16121636
#endif
16131637

1638+
// Register the view kernels for a match-substring-style predicate. Registered per
1639+
// view type (not via GenerateVarBinaryViewBase) so utf8_view -> StringViewType keeps
1640+
// the is_utf8 distinction used by ignore_case/regex folding.
1641+
template <template <typename...> class ExecTemplate, typename... Matcher>
1642+
void AddMatchSubstringViewKernels(ScalarFunction* func) {
1643+
DCHECK_OK(func->AddKernel({binary_view()}, boolean(),
1644+
ExecTemplate<BinaryViewType, Matcher...>::Exec,
1645+
MatchSubstringState::Init));
1646+
DCHECK_OK(func->AddKernel({utf8_view()}, boolean(),
1647+
ExecTemplate<StringViewType, Matcher...>::Exec,
1648+
MatchSubstringState::Init));
1649+
}
1650+
16141651
void AddAsciiStringMatchSubstring(FunctionRegistry* registry) {
16151652
{
16161653
auto func = std::make_shared<ScalarFunction>("match_substring", Arity::Unary(),
@@ -1620,6 +1657,7 @@ void AddAsciiStringMatchSubstring(FunctionRegistry* registry) {
16201657
DCHECK_OK(
16211658
func->AddKernel({ty}, boolean(), std::move(exec), MatchSubstringState::Init));
16221659
}
1660+
AddMatchSubstringViewKernels<MatchSubstring, PlainSubstringMatcher>(func.get());
16231661
DCHECK_OK(registry->AddFunction(std::move(func)));
16241662
}
16251663
{
@@ -1631,6 +1669,7 @@ void AddAsciiStringMatchSubstring(FunctionRegistry* registry) {
16311669
DCHECK_OK(
16321670
func->AddKernel({ty}, boolean(), std::move(exec), MatchSubstringState::Init));
16331671
}
1672+
AddMatchSubstringViewKernels<MatchSubstring, PlainStartsWithMatcher>(func.get());
16341673
DCHECK_OK(registry->AddFunction(std::move(func)));
16351674
}
16361675
{
@@ -1641,6 +1680,7 @@ void AddAsciiStringMatchSubstring(FunctionRegistry* registry) {
16411680
DCHECK_OK(
16421681
func->AddKernel({ty}, boolean(), std::move(exec), MatchSubstringState::Init));
16431682
}
1683+
AddMatchSubstringViewKernels<MatchSubstring, PlainEndsWithMatcher>(func.get());
16441684
DCHECK_OK(registry->AddFunction(std::move(func)));
16451685
}
16461686
#ifdef ARROW_WITH_RE2
@@ -1652,6 +1692,7 @@ void AddAsciiStringMatchSubstring(FunctionRegistry* registry) {
16521692
DCHECK_OK(
16531693
func->AddKernel({ty}, boolean(), std::move(exec), MatchSubstringState::Init));
16541694
}
1695+
AddMatchSubstringViewKernels<MatchSubstring, RegexSubstringMatcher>(func.get());
16551696
DCHECK_OK(registry->AddFunction(std::move(func)));
16561697
}
16571698
{
@@ -1662,6 +1703,7 @@ void AddAsciiStringMatchSubstring(FunctionRegistry* registry) {
16621703
DCHECK_OK(
16631704
func->AddKernel({ty}, boolean(), std::move(exec), MatchSubstringState::Init));
16641705
}
1706+
AddMatchSubstringViewKernels<MatchLike>(func.get());
16651707
DCHECK_OK(registry->AddFunction(std::move(func)));
16661708
}
16671709
#endif
@@ -1670,6 +1712,17 @@ void AddAsciiStringMatchSubstring(FunctionRegistry* registry) {
16701712
// ----------------------------------------------------------------------
16711713
// Substring find - lfind/index/etc.
16721714

1715+
// The output offset type for find/count kernels. Like TypeTraits<T>::OffsetType,
1716+
// but also defined for the view types, whose per-element length is int32-sized.
1717+
template <typename T, typename Enable = void>
1718+
struct StringOffsetType {
1719+
using type = typename TypeTraits<T>::OffsetType;
1720+
};
1721+
template <typename T>
1722+
struct StringOffsetType<T, enable_if_binary_view_like<T>> {
1723+
using type = Int32Type;
1724+
};
1725+
16731726
struct FindSubstring {
16741727
const PlainSubstringMatcher matcher_;
16751728

@@ -1716,7 +1769,7 @@ struct FindSubstringRegex {
17161769

17171770
template <typename InputType>
17181771
struct FindSubstringExec {
1719-
using OffsetType = typename TypeTraits<InputType>::OffsetType;
1772+
using OffsetType = typename StringOffsetType<InputType>::type;
17201773
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
17211774
const MatchSubstringOptions& options = MatchSubstringState::Get(ctx);
17221775
if (options.ignore_case) {
@@ -1746,7 +1799,7 @@ const FunctionDoc find_substring_doc(
17461799
#ifdef ARROW_WITH_RE2
17471800
template <typename InputType>
17481801
struct FindSubstringRegexExec {
1749-
using OffsetType = typename TypeTraits<InputType>::OffsetType;
1802+
using OffsetType = typename StringOffsetType<InputType>::type;
17501803
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
17511804
const MatchSubstringOptions& options = MatchSubstringState::Get(ctx);
17521805
ARROW_ASSIGN_OR_RAISE(auto matcher, FindSubstringRegex::Make(options, false));
@@ -1774,6 +1827,13 @@ void AddAsciiStringFindSubstring(FunctionRegistry* registry) {
17741827
GenerateVarBinaryToVarBinary<FindSubstringExec>(ty),
17751828
MatchSubstringState::Init));
17761829
}
1830+
// Per view type to keep the is_utf8 distinction; view length fits int32.
1831+
DCHECK_OK(func->AddKernel({binary_view()}, int32(),
1832+
FindSubstringExec<BinaryViewType>::Exec,
1833+
MatchSubstringState::Init));
1834+
DCHECK_OK(func->AddKernel({utf8_view()}, int32(),
1835+
FindSubstringExec<StringViewType>::Exec,
1836+
MatchSubstringState::Init));
17771837
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
17781838
FindSubstringExec<FixedSizeBinaryType>::Exec,
17791839
MatchSubstringState::Init));
@@ -1789,6 +1849,12 @@ void AddAsciiStringFindSubstring(FunctionRegistry* registry) {
17891849
GenerateVarBinaryToVarBinary<FindSubstringRegexExec>(ty),
17901850
MatchSubstringState::Init));
17911851
}
1852+
DCHECK_OK(func->AddKernel({binary_view()}, int32(),
1853+
FindSubstringRegexExec<BinaryViewType>::Exec,
1854+
MatchSubstringState::Init));
1855+
DCHECK_OK(func->AddKernel({utf8_view()}, int32(),
1856+
FindSubstringRegexExec<StringViewType>::Exec,
1857+
MatchSubstringState::Init));
17921858
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
17931859
FindSubstringRegexExec<FixedSizeBinaryType>::Exec,
17941860
MatchSubstringState::Init));
@@ -1862,7 +1928,7 @@ struct CountSubstringRegex {
18621928

18631929
template <typename InputType>
18641930
struct CountSubstringRegexExec {
1865-
using OffsetType = typename TypeTraits<InputType>::OffsetType;
1931+
using OffsetType = typename StringOffsetType<InputType>::type;
18661932
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
18671933
const MatchSubstringOptions& options = MatchSubstringState::Get(ctx);
18681934
ARROW_ASSIGN_OR_RAISE(
@@ -1876,7 +1942,7 @@ struct CountSubstringRegexExec {
18761942

18771943
template <typename InputType>
18781944
struct CountSubstringExec {
1879-
using OffsetType = typename TypeTraits<InputType>::OffsetType;
1945+
using OffsetType = typename StringOffsetType<InputType>::type;
18801946
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
18811947
const MatchSubstringOptions& options = MatchSubstringState::Get(ctx);
18821948
if (options.ignore_case) {
@@ -1923,6 +1989,12 @@ void AddAsciiStringCountSubstring(FunctionRegistry* registry) {
19231989
GenerateVarBinaryToVarBinary<CountSubstringExec>(ty),
19241990
MatchSubstringState::Init));
19251991
}
1992+
DCHECK_OK(func->AddKernel({binary_view()}, int32(),
1993+
CountSubstringExec<BinaryViewType>::Exec,
1994+
MatchSubstringState::Init));
1995+
DCHECK_OK(func->AddKernel({utf8_view()}, int32(),
1996+
CountSubstringExec<StringViewType>::Exec,
1997+
MatchSubstringState::Init));
19261998
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
19271999
CountSubstringExec<FixedSizeBinaryType>::Exec,
19282000
MatchSubstringState::Init));
@@ -1938,6 +2010,12 @@ void AddAsciiStringCountSubstring(FunctionRegistry* registry) {
19382010
GenerateVarBinaryToVarBinary<CountSubstringRegexExec>(ty),
19392011
MatchSubstringState::Init));
19402012
}
2013+
DCHECK_OK(func->AddKernel({binary_view()}, int32(),
2014+
CountSubstringRegexExec<BinaryViewType>::Exec,
2015+
MatchSubstringState::Init));
2016+
DCHECK_OK(func->AddKernel({utf8_view()}, int32(),
2017+
CountSubstringRegexExec<StringViewType>::Exec,
2018+
MatchSubstringState::Init));
19412019
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
19422020
CountSubstringRegexExec<FixedSizeBinaryType>::Exec,
19432021
MatchSubstringState::Init));

‎cpp/src/arrow/compute/kernels/scalar_string_benchmark.cc‎

Lines changed: 44 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "arrow/array.h"
2323
#include "arrow/chunked_array.h"
2424
#include "arrow/compute/api_scalar.h"
25+
#include "arrow/compute/cast.h"
2526
#include "arrow/compute/exec.h"
2627
#include "arrow/datum.h"
2728
#include "arrow/testing/gtest_util.h"
@@ -39,25 +40,34 @@ namespace compute {
3940

4041
constexpr auto kSeed = 0x94378165;
4142

42-
static void UnaryStringBenchmark(benchmark::State& state, const std::string& func_name,
43-
const FunctionOptions* options = nullptr) {
44-
const int64_t array_length = 1 << 20;
45-
const int64_t value_min_size = 0;
46-
const int64_t value_max_size = 32;
47-
const double null_probability = 0.01;
43+
// Shared dataset for the string benchmarks: ~1M only-ASCII values.
44+
static std::shared_ptr<Array> MakeBenchmarkStrings() {
45+
constexpr int64_t kArrayLength = 1 << 20;
46+
constexpr int64_t kValueMinSize = 0;
47+
constexpr int64_t kValueMaxSize = 32;
48+
constexpr double kNullProbability = 0.01;
4849
random::RandomArrayGenerator rng(kSeed);
50+
return rng.String(kArrayLength, kValueMinSize, kValueMaxSize, kNullProbability);
51+
}
4952

50-
// NOTE: this produces only-Ascii data
51-
auto values =
52-
rng.String(array_length, value_min_size, value_max_size, null_probability);
53+
static void UnaryStringBenchmark(benchmark::State& state, const std::string& func_name,
54+
const FunctionOptions* options = nullptr,
55+
const std::shared_ptr<DataType>& input_type = utf8()) {
56+
std::shared_ptr<Array> values = MakeBenchmarkStrings();
57+
const int64_t array_length = values->length();
58+
// Report throughput based on the raw string bytes, before any conversion.
59+
const int64_t data_nbytes = values->data()->buffers[2]->size();
60+
if (input_type->id() != Type::STRING) {
61+
values = Cast(*values, input_type).ValueOrDie();
62+
}
5363
// Make sure lookup tables are initialized before measuring
5464
ABORT_NOT_OK(CallFunction(func_name, {values}, options));
5565

5666
for (auto _ : state) {
5767
ABORT_NOT_OK(CallFunction(func_name, {values}, options));
5868
}
5969
state.SetItemsProcessed(state.iterations() * array_length);
60-
state.SetBytesProcessed(state.iterations() * values->data()->buffers[2]->size());
70+
state.SetBytesProcessed(state.iterations() * data_nbytes);
6171
}
6272

6373
static void AsciiLower(benchmark::State& state) {
@@ -77,6 +87,28 @@ static void MatchSubstring(benchmark::State& state) {
7787
UnaryStringBenchmark(state, "match_substring", &options);
7888
}
7989

90+
// Predicate run directly on a string_view array (the path added by this PR).
91+
static void MatchSubstringView(benchmark::State& state) {
92+
MatchSubstringOptions options("abac");
93+
UnaryStringBenchmark(state, "match_substring", &options, utf8_view());
94+
}
95+
96+
// Baseline: the pre-PR workaround of casting the view column to utf8 first. The
97+
// gap vs. MatchSubstringView is the cast that direct view support avoids.
98+
static void MatchSubstringViewCast(benchmark::State& state) {
99+
std::shared_ptr<Array> utf8_values = MakeBenchmarkStrings();
100+
const int64_t array_length = utf8_values->length();
101+
const int64_t data_nbytes = utf8_values->data()->buffers[2]->size();
102+
std::shared_ptr<Array> view_values = Cast(*utf8_values, utf8_view()).ValueOrDie();
103+
MatchSubstringOptions options("abac");
104+
for (auto _ : state) {
105+
ASSIGN_OR_ABORT(auto casted, Cast(*view_values, utf8()));
106+
ABORT_NOT_OK(CallFunction("match_substring", {casted}, &options));
107+
}
108+
state.SetItemsProcessed(state.iterations() * array_length);
109+
state.SetBytesProcessed(state.iterations() * data_nbytes);
110+
}
111+
80112
static void SplitPattern(benchmark::State& state) {
81113
SplitPatternOptions options("a");
82114
UnaryStringBenchmark(state, "split_pattern", &options);
@@ -242,6 +274,8 @@ BENCHMARK(AsciiLower);
242274
BENCHMARK(AsciiUpper);
243275
BENCHMARK(IsAlphaNumericAscii);
244276
BENCHMARK(MatchSubstring);
277+
BENCHMARK(MatchSubstringView);
278+
BENCHMARK(MatchSubstringViewCast);
245279
BENCHMARK(SplitPattern);
246280
BENCHMARK(TrimSingleAscii);
247281
BENCHMARK(TrimManyAscii);

‎cpp/src/arrow/compute/kernels/scalar_string_internal.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,8 @@ void AddUnaryStringPredicate(std::string name, FunctionRegistry* registry,
240240
auto exec = GenerateVarBinaryToVarBinary<StringPredicateFunctor, Predicate>(ty);
241241
ARROW_DCHECK_OK(func->AddKernel({ty}, boolean(), std::move(exec)));
242242
}
243+
ARROW_DCHECK_OK(func->AddKernel(
244+
{utf8_view()}, boolean(), StringPredicateFunctor<StringViewType, Predicate>::Exec));
243245
ARROW_DCHECK_OK(registry->AddFunction(std::move(func)));
244246
}
245247

0 commit comments

Comments
 (0)