GH-37476: [C++][Python] Preserve unsigned dictionary index types when building from values

cpp/src/arrow/array/array_dict_test.cc

@@ -1040,6 +1040,177 @@ TYPED_TEST(TestDictionaryBuilderIndexByteWidth, MakeBuilder) {
   AssertIndexByteWidth<TypeParam, NullType>();
 }
 
+// ----------------------------------------------------------------------
+// GH-37476: the requested index type's signedness must be preserved
+
+TEST(TestDictionaryBuilderIndexType, TypePreservesUnsignedIndexType) {
+  for (auto index_type : {uint8(), uint16(), uint32(), uint64()}) {
+    ARROW_SCOPED_TRACE("index_type = ", index_type->ToString());
+    auto dict_type = dictionary(index_type, utf8());
+    ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type));
+
+    auto builder_type = boxed_builder->type();
+    const auto& dict_builder_type = checked_cast<const DictionaryType&>(*builder_type);
+    AssertTypeEqual(*index_type, *dict_builder_type.index_type());
+  }
+}
+
+TEST(TestDictionaryBuilderIndexType, TypePreservesSignedIndexType) {
+  for (auto index_type : {int8(), int16(), int32(), int64()}) {
+    ARROW_SCOPED_TRACE("index_type = ", index_type->ToString());
+    auto dict_type = dictionary(index_type, utf8());
+    ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type));
+
+    auto builder_type = boxed_builder->type();
+    const auto& dict_builder_type = checked_cast<const DictionaryType&>(*builder_type);
+    AssertTypeEqual(*index_type, *dict_builder_type.index_type());
+  }
+}
+
+TEST(TestDictionaryBuilderIndexType, FinishPreservesUnsignedIndexType) {
+  for (auto index_type : {uint8(), uint16(), uint32(), uint64()}) {
+    ARROW_SCOPED_TRACE("index_type = ", index_type->ToString());
+    auto dict_type = dictionary(index_type, utf8());
+    ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type));
+    auto& builder = checked_cast<DictionaryBuilder<StringType>&>(*boxed_builder);
+
+    ASSERT_OK(builder.Append("a"));
+    ASSERT_OK(builder.Append("b"));
+    ASSERT_OK(builder.AppendNull());
+    ASSERT_OK(builder.Append("a"));
+
+    std::shared_ptr<Array> result;
+    ASSERT_OK(builder.Finish(&result));
+    ASSERT_OK(result->ValidateFull());
+
+    auto ex_dict = ArrayFromJSON(utf8(), R"(["a", "b"])");
+    auto ex_indices = ArrayFromJSON(index_type, "[0, 1, null, 0]");
+    DictionaryArray expected(dict_type, ex_indices, ex_dict);
+    AssertTypeEqual(*dict_type, *result->type());
+    AssertArraysEqual(expected, *result);
+  }
+}
+
+TEST(TestDictionaryBuilderIndexType, UnsignedIndexWidthStillAdapts) {
+  // The index width remains adaptive, exactly as it is for signed index types:
+  // only the signedness of the requested type is preserved.
+  auto dict_type = dictionary(uint8(), utf8());
+  ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type));
+  auto& builder = checked_cast<DictionaryBuilder<StringType>&>(*boxed_builder);
+
+  // More distinct values than a uint8 index start width accommodates.
+  for (int i = 0; i < 200; ++i) {
+    ASSERT_OK(builder.Append(std::to_string(i)));
+  }
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+  ASSERT_OK(result->ValidateFull());
+
+  const auto& result_type = checked_cast<const DictionaryType&>(*result->type());
+  // Widened, but still unsigned rather than falling back to a signed type.
+  AssertTypeEqual(*uint16(), *result_type.index_type());
+}
+
+TEST(TestDictionaryBuilderIndexType, NullValueTypeUnsignedIndexType) {
+  auto dict_type = dictionary(uint32(), null());
+  ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type));
+  auto& builder = checked_cast<DictionaryBuilder<NullType>&>(*boxed_builder);
+
+  ASSERT_OK(builder.AppendNull());
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+  ASSERT_OK(result->ValidateFull());
+
+  const auto& result_type = checked_cast<const DictionaryType&>(*result->type());
+  AssertTypeEqual(*uint32(), *result_type.index_type());
+}
+
+TEST(TestDictionaryBuilderIndexType, FinishDeltaPreservesUnsignedIndexType) {
+  auto dict_type = dictionary(uint32(), utf8());
+  ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type));
+  auto& builder = checked_cast<DictionaryBuilder<StringType>&>(*boxed_builder);
+
+  ASSERT_OK(builder.Append("a"));
+  ASSERT_OK(builder.Append("b"));
+
+  std::shared_ptr<Array> result_indices, result_delta;
+  ASSERT_OK(builder.FinishDelta(&result_indices, &result_delta));
+  ASSERT_OK(result_indices->ValidateFull());
+
+  // The delta indices must carry the requested index type too, not the signed
+  // type that the adaptive indices builder produces internally.
+  AssertTypeEqual(*uint32(), *result_indices->type());
+  AssertArraysEqual(*ArrayFromJSON(uint32(), "[0, 1]"), *result_indices);
+}
+
+TEST(TestDictionaryBuilderIndexType, SuppliedDictionaryPreservesUnsignedIndexType) {
+  // The supplied-dictionary constructor starts the adaptive indices builder at its
+  // default width rather than at the requested one, so the requested *width* is not
+  // honoured on this path. That predates this change: a requested int32 reports int8
+  // too. What must hold is that the *signedness* survives, i.e. uint8 and not int8.
+  auto dict_values = ArrayFromJSON(utf8(), R"(["a", "b"])");
+
+  ASSERT_OK_AND_ASSIGN(auto unsigned_builder,
+                       MakeDictionaryBuilder(dictionary(uint32(), utf8()), dict_values));
+  auto unsigned_type = unsigned_builder->type();
+  AssertTypeEqual(*uint8(),
+                  *checked_cast<const DictionaryType&>(*unsigned_type).index_type());
+
+  ASSERT_OK_AND_ASSIGN(auto signed_builder,
+                       MakeDictionaryBuilder(dictionary(int32(), utf8()), dict_values));
+  auto signed_type = signed_builder->type();
+  AssertTypeEqual(*int8(),
+                  *checked_cast<const DictionaryType&>(*signed_type).index_type());
+}
+
+TEST(TestDictionaryBuilderIndexType, FixedSizeBinaryUnsignedIndexType) {
+  auto dict_type = dictionary(uint16(), fixed_size_binary(2));
+  ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type));
+  auto& builder = checked_cast<DictionaryBuilder<FixedSizeBinaryType>&>(*boxed_builder);
+
+  ASSERT_OK(builder.Append("ab"));
+  ASSERT_OK(builder.Append("cd"));
+  ASSERT_OK(builder.Append("ab"));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+  ASSERT_OK(result->ValidateFull());
+
+  const auto& result_type = checked_cast<const DictionaryType&>(*result->type());
+  AssertTypeEqual(*uint16(), *result_type.index_type());
+}
+
+TEST(TestDictionaryBuilderIndexType, OrderedAndUnsignedIndexType) {
+  // The ordered flag (GH-49689) and the index type must both survive.
+  auto dict_type = dictionary(uint32(), utf8(), /*ordered=*/true);
+  ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type));
+  auto& builder = checked_cast<DictionaryBuilder<StringType>&>(*boxed_builder);
+
+  ASSERT_OK(builder.Append("a"));
+  ASSERT_OK(builder.Append("b"));
+
+  std::shared_ptr<Array> result;
+  ASSERT_OK(builder.Finish(&result));
+  ASSERT_OK(result->ValidateFull());
+
+  const auto& result_type = checked_cast<const DictionaryType&>(*result->type());
+  AssertTypeEqual(*uint32(), *result_type.index_type());
+  ASSERT_TRUE(result_type.ordered());
+}
+
+TEST(TestDictionaryBuilderIndexType, ExactIndexTypeStillHonoursUnsigned) {
+  // The exact-index builder already honoured unsigned index types; guard it.
+  auto dict_type = dictionary(uint16(), utf8());
+  std::unique_ptr<ArrayBuilder> boxed_builder;
+  ASSERT_OK(MakeBuilderExactIndex(default_memory_pool(), dict_type, &boxed_builder));
+
+  auto builder_type = boxed_builder->type();
+  const auto& dict_builder_type = checked_cast<const DictionaryType&>(*builder_type);
+  AssertTypeEqual(*uint16(), *dict_builder_type.index_type());
+}
+
 // ----------------------------------------------------------------------
 // DictionaryArray tests
 

cpp/src/arrow/array/builder_dict.h

@@ -134,6 +134,33 @@ class ARROW_EXPORT DictionaryMemoTable {
 
 namespace internal {
 
+/// \brief Return the unsigned integer type of the same width when an unsigned
+/// dictionary index type was requested.
+///
+/// The adaptive indices builder only ever produces signed integer types. Dictionary
+/// indices are non-negative, so the signed and unsigned integer types of a given
+/// width have identical memory layout and reporting one as the other is
+/// value-preserving (GH-37476). The width itself stays adaptive, exactly as it is
+/// for signed index types; only the signedness of the requested type is preserved.
+inline std::shared_ptr<DataType> MaybeUnsignedIndexType(
+    const std::shared_ptr<DataType>& index_type, bool unsigned_index) {
+  if (!unsigned_index) {
+    return index_type;
+  }
+  switch (index_type->id()) {
+    case Type::INT8:
+      return ::arrow::uint8();
+    case Type::INT16:
+      return ::arrow::uint16();
+    case Type::INT32:
+      return ::arrow::uint32();
+    case Type::INT64:
+      return ::arrow::uint64();
+    default:
+      return index_type;
+  }
+}
+
 /// \brief Array builder for created encoded DictionaryArray from
 /// dense array
 ///
@@ -154,14 +181,16 @@ class DictionaryBuilderBase : public ArrayBuilder {
                                     const std::shared_ptr<DataType>&>
                             value_type,
                         MemoryPool* pool = default_memory_pool(),
-                        int64_t alignment = kDefaultBufferAlignment, bool ordered = false)
+                        int64_t alignment = kDefaultBufferAlignment, bool ordered = false,
+                        bool unsigned_index = false)
       : ArrayBuilder(pool, alignment),
         memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
         delta_offset_(0),
         byte_width_(-1),
         indices_builder_(start_int_size, pool, alignment),
         value_type_(value_type),
-        ordered_(ordered) {}
+        ordered_(ordered),
+        unsigned_index_(unsigned_index) {}
 
   template <typename T1 = T>
   explicit DictionaryBuilderBase(
@@ -199,14 +228,16 @@ class DictionaryBuilderBase : public ArrayBuilder {
                                     const std::shared_ptr<DataType>&>
                             value_type,
                         MemoryPool* pool = default_memory_pool(),
-                        int64_t alignment = kDefaultBufferAlignment, bool ordered = false)
+                        int64_t alignment = kDefaultBufferAlignment, bool ordered = false,
+                        bool unsigned_index = false)
       : ArrayBuilder(pool, alignment),
         memo_table_(new internal::DictionaryMemoTable(pool, value_type)),
         delta_offset_(0),
         byte_width_(static_cast<const T1&>(*value_type).byte_width()),
         indices_builder_(start_int_size, pool, alignment),
         value_type_(value_type),
-        ordered_(ordered) {}
+        ordered_(ordered),
+        unsigned_index_(unsigned_index) {}
 
   template <typename T1 = T>
   explicit DictionaryBuilderBase(
@@ -244,14 +275,15 @@ class DictionaryBuilderBase : public ArrayBuilder {
   explicit DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
                                  MemoryPool* pool = default_memory_pool(),
                                  int64_t alignment = kDefaultBufferAlignment,
-                                 bool ordered = false)
+                                 bool ordered = false, bool unsigned_index = false)
       : ArrayBuilder(pool, alignment),
         memo_table_(new internal::DictionaryMemoTable(pool, dictionary)),
         delta_offset_(0),
         byte_width_(-1),
         indices_builder_(pool, alignment),
         value_type_(dictionary->type()),
-        ordered_(ordered) {}
+        ordered_(ordered),
+        unsigned_index_(unsigned_index) {}
 
   ~DictionaryBuilderBase() override = default;
 
@@ -486,6 +518,7 @@ class DictionaryBuilderBase : public ArrayBuilder {
     std::shared_ptr<ArrayData> indices_data;
     std::shared_ptr<ArrayData> delta_data;
     ARROW_RETURN_NOT_OK(FinishWithDictOffset(delta_offset_, &indices_data, &delta_data));
+    indices_data->type = MaybeUnsignedIndexType(indices_data->type, unsigned_index_);
     *out_indices = MakeArray(indices_data);
     *out_delta = MakeArray(delta_data);
     return Status::OK();
@@ -498,7 +531,9 @@ class DictionaryBuilderBase : public ArrayBuilder {
   Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }
 
   std::shared_ptr<DataType> type() const override {
-    return ::arrow::dictionary(indices_builder_.type(), value_type_, ordered_);
+    return ::arrow::dictionary(
+        MaybeUnsignedIndexType(indices_builder_.type(), unsigned_index_), value_type_,
+        ordered_);
   }
 
  protected:
@@ -570,6 +605,7 @@ class DictionaryBuilderBase : public ArrayBuilder {
   BuilderType indices_builder_;
   std::shared_ptr<DataType> value_type_;
   bool ordered_ = false;
+  bool unsigned_index_ = false;
 };
 
 template <typename BuilderType>
@@ -581,10 +617,12 @@ class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
           start_int_size,
       const std::shared_ptr<DataType>& value_type,
       MemoryPool* pool = default_memory_pool(),
-      int64_t alignment = kDefaultBufferAlignment, bool ordered = false)
+      int64_t alignment = kDefaultBufferAlignment, bool ordered = false,
+      bool unsigned_index = false)
       : ArrayBuilder(pool, alignment),
         indices_builder_(start_int_size, pool, alignment),
-        ordered_(ordered) {}
+        ordered_(ordered),
+        unsigned_index_(unsigned_index) {}
 
   explicit DictionaryBuilderBase(const std::shared_ptr<DataType>& value_type,
                                  MemoryPool* pool = default_memory_pool(),
@@ -623,10 +661,11 @@ class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
   explicit DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
                                  MemoryPool* pool = default_memory_pool(),
                                  int64_t alignment = kDefaultBufferAlignment,
-                                 bool ordered = false)
+                                 bool ordered = false, bool unsigned_index = false)
       : ArrayBuilder(pool, alignment),
         indices_builder_(pool, alignment),
-        ordered_(ordered) {}
+        ordered_(ordered),
+        unsigned_index_(unsigned_index) {}
 
   /// \brief Append a scalar null value
   Status AppendNull() final {
@@ -678,7 +717,8 @@ class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
 
   Status FinishInternal(std::shared_ptr<ArrayData>* out) override {
     ARROW_RETURN_NOT_OK(indices_builder_.FinishInternal(out));
-    (*out)->type = dictionary((*out)->type, null(), ordered_);
+    (*out)->type = dictionary(MaybeUnsignedIndexType((*out)->type, unsigned_index_),
+                              null(), ordered_);
     (*out)->dictionary = NullArray(0).data();
     return Status::OK();
   }
@@ -690,12 +730,15 @@ class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
   Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }
 
   std::shared_ptr<DataType> type() const override {
-    return ::arrow::dictionary(indices_builder_.type(), null(), ordered_);
+    return ::arrow::dictionary(
+        MaybeUnsignedIndexType(indices_builder_.type(), unsigned_index_), null(),
+        ordered_);
   }
 
  protected:
   BuilderType indices_builder_;
   bool ordered_ = false;
+  bool unsigned_index_ = false;
 };
 
 }  // namespace internal

cpp/src/arrow/builder.cc

@@ -170,9 +170,10 @@ struct DictionaryBuilderCase {
     using AdaptiveBuilderType = DictionaryBuilder<ValueType>;
     using ExactBuilderType =
         internal::DictionaryBuilderBase<TypeErasedIntBuilder, ValueType>;
+    const bool unsigned_index = is_unsigned_integer(index_type->id());
     if (dictionary != nullptr) {
-      out->reset(
-          new AdaptiveBuilderType(dictionary, pool, kDefaultBufferAlignment, ordered));
+      out->reset(new AdaptiveBuilderType(dictionary, pool, kDefaultBufferAlignment,
+                                         ordered, unsigned_index));
     } else if (exact_index_type) {
       if (!is_integer(index_type->id())) {
         return Status::TypeError("MakeBuilder: invalid index type ", *index_type);
@@ -182,7 +183,8 @@ struct DictionaryBuilderCase {
     } else {
       auto start_int_size = index_type->byte_width();
       out->reset(new AdaptiveBuilderType(start_int_size, value_type, pool,
-                                         kDefaultBufferAlignment, ordered));
+                                         kDefaultBufferAlignment, ordered,
+                                         unsigned_index));
     }
     return Status::OK();
   }