YT-26680: Optimize rows digest computer (8-33x faster)

apollo1321 · apollo1321 · commit 6e8395fbd253 · 2025-11-18T02:46:27.000+03:00
Before (MD5 \+ sort for each row):

```
--------------------------------------------------------------------------------------------------
Benchmark                                                        Time             CPU   Iterations
--------------------------------------------------------------------------------------------------
BM_TRowsDigestComputer_ProcessRowsFewColumns_mean              203 ns          203 ns            5
BM_TRowsDigestComputer_ProcessRowsManyColumns_mean            7678 ns         7677 ns            5
BM_TRowsDigestComputer_ProcessRowsRandomOrder_mean            2794 ns         2794 ns            5
BM_TRowsDigestComputer_ProcessRowsMixedTypes_mean              496 ns          496 ns            5
BM_TRowsDigestComputer_ProcessRowsLongStrings_mean            1707 ns         1707 ns            5
BM_TRowsDigestComputer_ProcessRowsDynamicColumns_mean          885 ns          885 ns            5
BM_TRowsDigestComputer_ProcessRowsSparse_mean                  435 ns          435 ns            5
```

After (XXH3 \+ (binary search \+ insert for each new column) \+ batching buffer):

```
--------------------------------------------------------------------------------------------------
Benchmark                                                        Time             CPU   Iterations
--------------------------------------------------------------------------------------------------
BM_TRowsDigestComputer_ProcessRowsFewColumns_mean             23.8 ns         23.8 ns            5
BM_TRowsDigestComputer_ProcessRowsManyColumns_mean             232 ns          232 ns            5
BM_TRowsDigestComputer_ProcessRowsRandomOrder_mean            99.5 ns         99.5 ns            5
BM_TRowsDigestComputer_ProcessRowsMixedTypes_mean             38.4 ns         38.4 ns            5
BM_TRowsDigestComputer_ProcessRowsLongStrings_mean            89.0 ns         89.0 ns            5
BM_TRowsDigestComputer_ProcessRowsDynamicColumns_mean         56.9 ns         56.9 ns            5
BM_TRowsDigestComputer_ProcessRowsSparse_mean                 35.0 ns         35.0 ns            5
```

Without batching (XXH3 \+ (binary search \+ insert for each new column)):

```
--------------------------------------------------------------------------------------------------
Benchmark                                                        Time             CPU   Iterations
--------------------------------------------------------------------------------------------------
BM_TRowsDigestComputer_ProcessRowsFewColumns_mean             42.4 ns         42.4 ns            5
BM_TRowsDigestComputer_ProcessRowsManyColumns_mean             554 ns          554 ns            5
BM_TRowsDigestComputer_ProcessRowsRandomOrder_mean             228 ns          228 ns            5
BM_TRowsDigestComputer_ProcessRowsMixedTypes_mean             73.3 ns         73.3 ns            5
BM_TRowsDigestComputer_ProcessRowsLongStrings_mean            94.7 ns         94.7 ns            5
BM_TRowsDigestComputer_ProcessRowsDynamicColumns_mean          117 ns          117 ns            5
BM_TRowsDigestComputer_ProcessRowsSparse_mean                 65.9 ns         65.9 ns            5
```

XXH64 is slower than XXH3  by about 5-46% (XXH64 \+ (binary search \+ insert for each new column)):

```
--------------------------------------------------------------------------------------------------
Benchmark                                                        Time             CPU   Iterations
--------------------------------------------------------------------------------------------------
BM_TRowsDigestComputer_ProcessRowsFewColumns_mean             47.2 ns         47.2 ns            5
BM_TRowsDigestComputer_ProcessRowsManyColumns_mean             712 ns          712 ns            5
BM_TRowsDigestComputer_ProcessRowsRandomOrder_mean             289 ns          289 ns            5
BM_TRowsDigestComputer_ProcessRowsMixedTypes_mean             87.8 ns         87.8 ns            5
BM_TRowsDigestComputer_ProcessRowsLongStrings_mean             129 ns          129 ns            5
BM_TRowsDigestComputer_ProcessRowsDynamicColumns_mean          144 ns          144 ns            5
BM_TRowsDigestComputer_ProcessRowsSparse_mean                 79.0 ns         79.0 ns            5
```
commit_hash:5e99e75ed9baf15ffdd32bd0a7a132a7911f0829
diff --git a/yt/yt/client/api/rowset.cpp b/yt/yt/client/api/rowset.cpp
@@ -172,7 +172,7 @@ class TSchemafulRowsetWriter
         return VoidFuture;
     }
 
-    std::optional<TMD5Hash> GetDigest() const override
+    std::optional<TRowsDigest> GetDigest() const override
     {
         return std::nullopt;
     }
diff --git a/yt/yt/client/formats/schemaful_writer.cpp b/yt/yt/client/formats/schemaful_writer.cpp
@@ -114,7 +114,7 @@ TFuture<void> TSchemafulWriter::GetReadyEvent()
     return Result_;
 }
 
-std::optional<TMD5Hash> TSchemafulWriter::GetDigest() const
+std::optional<TRowsDigest> TSchemafulWriter::GetDigest() const
 {
     return std::nullopt;
 }
diff --git a/yt/yt/client/formats/schemaful_writer.h b/yt/yt/client/formats/schemaful_writer.h
@@ -34,7 +34,7 @@ class TSchemafulWriter
 
     TFuture<void> GetReadyEvent() override;
 
-    std::optional<NCrypto::TMD5Hash> GetDigest() const override;
+    std::optional<NTableClient::TRowsDigest> GetDigest() const override;
 
 private:
     const NConcurrency::IAsyncOutputStreamPtr Stream_;
diff --git a/yt/yt/client/queue_client/producer_client.cpp b/yt/yt/client/queue_client/producer_client.cpp
@@ -2,6 +2,8 @@
 
 #include "private.h"
 
+#include <yt/yt/client/table_client/public.h>
+
 #include <yt/yt/client/api/client.h>
 #include <yt/yt/client/api/transaction.h>
 
@@ -175,7 +177,7 @@ class TProducerSession
             }));
     }
 
-    std::optional<TMD5Hash> GetDigest() const override
+    std::optional<TRowsDigest> GetDigest() const override
     {
         return std::nullopt;
     }
diff --git a/yt/yt/client/table_client/adapters.cpp b/yt/yt/client/table_client/adapters.cpp
@@ -128,7 +128,7 @@ class TSchemalessApiFromWriterAdapter
         return Schema_;
     }
 
-    std::optional<TMD5Hash> GetDigest() const override
+    std::optional<TRowsDigest> GetDigest() const override
     {
         return std::nullopt;
     }
diff --git a/yt/yt/client/table_client/pipe.cpp b/yt/yt/client/table_client/pipe.cpp
@@ -234,7 +234,7 @@ class TSchemafulPipe::TWriter
         return Data_->WriterReadyEvent;
     }
 
-    std::optional<TMD5Hash> GetDigest() const override
+    std::optional<TRowsDigest> GetDigest() const override
     {
         return std::nullopt;
     }
diff --git a/yt/yt/client/table_client/public.h b/yt/yt/client/table_client/public.h
@@ -480,4 +480,8 @@ YT_DEFINE_STRONG_TYPEDEF(TSignedWriteFragmentResultPtr, NSignature::TSignaturePt
 
 ////////////////////////////////////////////////////////////////////////////////
 
+YT_DEFINE_STRONG_TYPEDEF(TRowsDigest, ui64);
+
+////////////////////////////////////////////////////////////////////////////////
+
 } // namespace NYT::NTableClient
diff --git a/yt/yt/client/table_client/schemaless_dynamic_table_writer.cpp b/yt/yt/client/table_client/schemaless_dynamic_table_writer.cpp
@@ -65,7 +65,7 @@ class TSchemalessDynamicTableWriter
         return NameTable_;
     }
 
-    std::optional<TMD5Hash> GetDigest() const override
+    std::optional<TRowsDigest> GetDigest() const override
     {
         return std::nullopt;
     }
diff --git a/yt/yt/client/table_client/unversioned_writer.h b/yt/yt/client/table_client/unversioned_writer.h
@@ -32,14 +32,16 @@ struct IUnversionedRowsetWriter
     [[nodiscard]] virtual bool Write(TRange<TUnversionedRow> rows) = 0;
 
     /*!
+     * TODO(apollo1321): Should be removed from base class and implemented as wrapper.
+     *
      * Returns the digest of the written rows.
      *
      * Useful for checking the determinism of user jobs.
      * Returns nullopt when hash is not computed.
      *
      * Must not be called concurrently with Write method.
      */
-    virtual std::optional<NCrypto::TMD5Hash> GetDigest() const = 0;
+    virtual std::optional<TRowsDigest> GetDigest() const = 0;
 };
 
 DEFINE_REFCOUNTED_TYPE(IUnversionedRowsetWriter)
diff --git a/yt/yt/client/table_client/wire_protocol.cpp b/yt/yt/client/table_client/wire_protocol.cpp
@@ -1282,7 +1282,7 @@ class TWireProtocolRowsetWriter
         return CompressedBlocks_;
     }
 
-    std::optional<TMD5Hash> GetDigest() const override
+    std::optional<TRowsDigest> GetDigest() const override
     {
         return std::nullopt;
     }
diff --git a/yt/yt/library/formats/schemaful_dsv_writer.cpp b/yt/yt/library/formats/schemaful_dsv_writer.cpp
@@ -247,7 +247,7 @@ class TSchemafulWriterForSchemafulDsv
         return Result_;
     }
 
-    std::optional<NCrypto::TMD5Hash> GetDigest() const override
+    std::optional<TRowsDigest> GetDigest() const override
     {
         return std::nullopt;
     }
diff --git a/yt/yt/library/formats/schemaless_writer_adapter.cpp b/yt/yt/library/formats/schemaless_writer_adapter.cpp
@@ -336,7 +336,7 @@ TFuture<void> TSchemalessFormatWriterBase::Flush()
     return MakeFuture(Error_);
 }
 
-std::optional<TMD5Hash> TSchemalessFormatWriterBase::GetDigest() const {
+std::optional<TRowsDigest> TSchemalessFormatWriterBase::GetDigest() const {
     return std::nullopt;
 }
 
diff --git a/yt/yt/library/formats/schemaless_writer_adapter.h b/yt/yt/library/formats/schemaless_writer_adapter.h
@@ -38,7 +38,7 @@ class TSchemalessFormatWriterBase
 
     TFuture<void> Flush() override;
 
-    std::optional<NCrypto::TMD5Hash> GetDigest() const override;
+    std::optional<NTableClient::TRowsDigest> GetDigest() const override;
 
 protected:
     const NTableClient::TNameTablePtr NameTable_;
diff --git a/yt/yt/library/formats/web_json_writer.cpp b/yt/yt/library/formats/web_json_writer.cpp
@@ -515,7 +515,7 @@ class TWriterForWebJson
     i64 GetWrittenSize() const override;
     TFuture<void> Close() override;
     TFuture<void> Flush() override;
-    std::optional<TMD5Hash> GetDigest() const override;
+    std::optional<TRowsDigest> GetDigest() const override;
 
 private:
     const TWebJsonFormatConfigPtr Config_;
@@ -760,7 +760,7 @@ void TWriterForWebJson<TValueWriter>::DoClose()
 }
 
 template <typename TValueWriter>
-std::optional<TMD5Hash> TWriterForWebJson<TValueWriter>::GetDigest() const
+std::optional<TRowsDigest> TWriterForWebJson<TValueWriter>::GetDigest() const
 {
     return std::nullopt;
 }

Original file line number	Diff line number	Diff line change
`@@ -172,7 +172,7 @@ class TSchemafulRowsetWriter`
`172`	`172`	`return VoidFuture;`
`173`	`173`	`}`
`174`	`174`
`175`		`- std::optional<TMD5Hash> GetDigest() const override`
	`175`	`+ std::optional<TRowsDigest> GetDigest() const override`
`176`	`176`	`{`
`177`	`177`	`return std::nullopt;`
`178`	`178`	`}`
Original file line number	Diff line number	Diff line change
`@@ -114,7 +114,7 @@ TFuture<void> TSchemafulWriter::GetReadyEvent()`
`114`	`114`	`return Result_;`
`115`	`115`	`}`
`116`	`116`
`117`		`-std::optional<TMD5Hash> TSchemafulWriter::GetDigest() const`
	`117`	`+std::optional<TRowsDigest> TSchemafulWriter::GetDigest() const`
`118`	`118`	`{`
`119`	`119`	`return std::nullopt;`
`120`	`120`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,8 @@`
`2`	`2`
`3`	`3`	`#include "private.h"`
`4`	`4`
	`5`	`+#include <yt/yt/client/table_client/public.h>`
	`6`	`+`
`5`	`7`	`#include <yt/yt/client/api/client.h>`
`6`	`8`	`#include <yt/yt/client/api/transaction.h>`
`7`	`9`
`@@ -175,7 +177,7 @@ class TProducerSession`
`175`	`177`	`}));`
`176`	`178`	`}`
`177`	`179`
`178`		`- std::optional<TMD5Hash> GetDigest() const override`
	`180`	`+ std::optional<TRowsDigest> GetDigest() const override`
`179`	`181`	`{`
`180`	`182`	`return std::nullopt;`
`181`	`183`	`}`
Original file line number	Diff line number	Diff line change
`@@ -128,7 +128,7 @@ class TSchemalessApiFromWriterAdapter`
`128`	`128`	`return Schema_;`
`129`	`129`	`}`
`130`	`130`
`131`		`- std::optional<TMD5Hash> GetDigest() const override`
	`131`	`+ std::optional<TRowsDigest> GetDigest() const override`
`132`	`132`	`{`
`133`	`133`	`return std::nullopt;`
`134`	`134`	`}`
Original file line number	Diff line number	Diff line change
`@@ -234,7 +234,7 @@ class TSchemafulPipe::TWriter`
`234`	`234`	`return Data_->WriterReadyEvent;`
`235`	`235`	`}`
`236`	`236`
`237`		`- std::optional<TMD5Hash> GetDigest() const override`
	`237`	`+ std::optional<TRowsDigest> GetDigest() const override`
`238`	`238`	`{`
`239`	`239`	`return std::nullopt;`
`240`	`240`	`}`
Original file line number	Diff line number	Diff line change
`@@ -65,7 +65,7 @@ class TSchemalessDynamicTableWriter`
`65`	`65`	`return NameTable_;`
`66`	`66`	`}`
`67`	`67`
`68`		`- std::optional<TMD5Hash> GetDigest() const override`
	`68`	`+ std::optional<TRowsDigest> GetDigest() const override`
`69`	`69`	`{`
`70`	`70`	`return std::nullopt;`
`71`	`71`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1282,7 +1282,7 @@ class TWireProtocolRowsetWriter`
`1282`	`1282`	`return CompressedBlocks_;`
`1283`	`1283`	`}`
`1284`	`1284`
`1285`		`- std::optional<TMD5Hash> GetDigest() const override`
	`1285`	`+ std::optional<TRowsDigest> GetDigest() const override`
`1286`	`1286`	`{`
`1287`	`1287`	`return std::nullopt;`
`1288`	`1288`	`}`