Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 42 additions & 1 deletion cpp/src/gandiva/gdv_function_stubs_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <gtest/gtest.h>

#include <limits>
#include <memory>

#include "arrow/util/logging.h"
#include "gandiva/execution_context.h"
Expand Down Expand Up @@ -649,6 +650,20 @@ TEST(TestGdvFnStubs, TestUpper) {

ctx.Reset();

// Truncated trailing multibyte glyph in an exact-sized buffer (no trailing NUL);
// the lead byte claims more bytes than remain. Use new[] so the over-read trips
// ASAN instead of landing on std::string's guaranteed terminator.
{
const int32_t trunc_len = 4;
std::unique_ptr<char[]> trunc(new char[trunc_len]{'a', 'b', 'c', '\xc3'});
out_str = gdv_fn_upper_utf8(ctx_ptr, trunc.get(), trunc_len, &out_len);
EXPECT_EQ(out_len, 0);
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr(
"unexpected byte \\c3 encountered while decoding utf8 string"));
ctx.Reset();
}

// Max Len Test
out_len = -1;
int32_t bad_len = std::numeric_limits<int32_t>::max() / 2 + 1;
Expand Down Expand Up @@ -752,6 +767,19 @@ TEST(TestGdvFnStubs, TestLower) {
"unexpected byte \\c3 encountered while decoding utf8 string"));
ctx.Reset();

// Truncated trailing 3-byte lead in an exact-sized buffer (no trailing NUL); only
// the lead byte is present, so decoding must not read the missing continuation bytes.
{
const int32_t trunc_len = 4;
std::unique_ptr<char[]> trunc(new char[trunc_len]{'a', 'b', 'c', '\xe0'});
out_str = gdv_fn_lower_utf8(ctx_ptr, trunc.get(), trunc_len, &out_len);
EXPECT_EQ(out_len, 0);
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr(
"unexpected byte \\e0 encountered while decoding utf8 string"));
ctx.Reset();
}

std::string e(
"åbÑg\xe0\xa0"
"åBUå");
Expand Down Expand Up @@ -805,7 +833,7 @@ TEST(TestGdvFnStubs, TestInitCap) {
EXPECT_EQ(std::string(out_str, out_len), "{Õhp,Pqśv}Ń+");
EXPECT_FALSE(ctx.has_error());

out_str = gdv_fn_initcap_utf8(ctx_ptr, "sɦasasdsɦsd\"sdsdɦ", 19, &out_len);
out_str = gdv_fn_initcap_utf8(ctx_ptr, "sɦasasdsɦsd\"sdsdɦ", 20, &out_len);
EXPECT_EQ(std::string(out_str, out_len), "Sɦasasdsɦsd\"Sdsdɦ");
EXPECT_FALSE(ctx.has_error());

Expand Down Expand Up @@ -842,6 +870,19 @@ TEST(TestGdvFnStubs, TestInitCap) {
"unexpected byte \\c3 encountered while decoding utf8 string"));
ctx.Reset();

// Truncated trailing multibyte glyph in an exact-sized buffer (no trailing NUL);
// the lead byte claims more bytes than remain and must not be decoded past the end.
{
const int32_t trunc_len = 4;
std::unique_ptr<char[]> trunc(new char[trunc_len]{'a', 'b', 'c', '\xc3'});
out_str = gdv_fn_initcap_utf8(ctx_ptr, trunc.get(), trunc_len, &out_len);
EXPECT_EQ(out_len, 0);
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr(
"unexpected byte \\c3 encountered while decoding utf8 string"));
ctx.Reset();
}

// Max Len Test
out_len = -1;
int32_t bad_len = std::numeric_limits<int32_t>::max() / 2 + 1;
Expand Down
24 changes: 24 additions & 0 deletions cpp/src/gandiva/gdv_string_function_stubs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,14 @@ const char* gdv_fn_lower_utf8(int64_t context, const char* data, int32_t data_le
continue;
}

// A truncated trailing glyph must be rejected before decoding, otherwise
// UTF8Decode walks continuation bytes past data[data_len].
if (char_len == 0 || i + char_len > data_len) {
gdv_fn_set_error_for_invalid_utf8(context, data[i]);
*out_len = 0;
return "";
}

// Control reaches here when we encounter a multibyte character
const auto* in_char = (const uint8_t*)(data + i);

Expand Down Expand Up @@ -353,6 +361,14 @@ const char* gdv_fn_upper_utf8(int64_t context, const char* data, int32_t data_le
continue;
}

// A truncated trailing glyph must be rejected before decoding, otherwise
// UTF8Decode walks continuation bytes past data[data_len].
if (char_len == 0 || i + char_len > data_len) {
gdv_fn_set_error_for_invalid_utf8(context, data[i]);
*out_len = 0;
return "";
}

// Control reaches here when we encounter a multibyte character
const auto* in_char = (const uint8_t*)(data + i);

Expand Down Expand Up @@ -571,6 +587,14 @@ const char* gdv_fn_initcap_utf8(int64_t context, const char* data, int32_t data_

char_len = gdv_fn_utf8_char_length(data[i]);

// A truncated trailing glyph must be rejected before decoding, otherwise
// UTF8Decode walks continuation bytes past data[data_len].
if (char_len == 0 || i + char_len > data_len) {
gdv_fn_set_error_for_invalid_utf8(context, data[i]);
*out_len = 0;
return "";
}

// Control reaches here when we encounter a multibyte character
const auto* in_char = (const uint8_t*)(data + i);

Expand Down
Loading