perf: kll_sketch grows its items buffer by 8x up to k

benchmarks/CMakeLists.txt

@@ -50,3 +50,20 @@ target_sources(count_min_sketch_benchmark
     benchmark_count_min_sketch.cpp
     benchmark_count_min_sketch_serialization.cpp
 )
+
+add_executable(kll_sketch_benchmark)
+
+target_link_libraries(kll_sketch_benchmark
+  PRIVATE
+    kll
+    benchmark::benchmark_main
+)
+
+set_target_properties(kll_sketch_benchmark PROPERTIES
+  CXX_STANDARD_REQUIRED YES
+)
+
+target_sources(kll_sketch_benchmark
+  PRIVATE
+    benchmark_kll_sketch.cpp
+)

benchmarks/benchmark_kll_sketch.cpp

@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <benchmark/benchmark.h>
+#include <kll_sketch.hpp>
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace
+{
+
+using Sketch = datasketches::kll_sketch<double>;
+
+std::vector<double> makeItems(size_t size)
+{
+    std::vector<double> items;
+    items.reserve(size);
+    uint64_t state = 1;
+    for (size_t i = 0; i < size; ++i)
+    {
+        state = state * 6364136223846793005ULL + 1442695040888963407ULL;
+        items.push_back(static_cast<double>(state >> 11));
+    }
+    return items;
+}
+
+// A new sketch with k = range(0) gets range(1) items.
+void BM_KllBuild(benchmark::State & state)
+{
+    const auto k = static_cast<uint16_t>(state.range(0));
+    const auto items = makeItems(static_cast<size_t>(state.range(1)));
+    for (auto _ : state)
+    {
+        Sketch sketch(k);
+        for (const double item : items)
+            sketch.update(item);
+
+        benchmark::DoNotOptimize(sketch.get_n());
+        benchmark::ClobberMemory();
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(items.size()));
+}
+
+// Many sketches with k = range(0) are alive at once, each with range(1) items, like the groups of
+// a GROUP BY.
+void BM_KllBuildManySketches(benchmark::State & state)
+{
+    constexpr size_t NUM_SKETCHES = 1000;
+    const auto k = static_cast<uint16_t>(state.range(0));
+    const auto items = makeItems(static_cast<size_t>(state.range(1)));
+    for (auto _ : state)
+    {
+        std::vector<Sketch> sketches(NUM_SKETCHES, Sketch(k));
+        for (auto & sketch : sketches)
+            for (const double item : items)
+                sketch.update(item);
+
+        benchmark::DoNotOptimize(sketches.data());
+        benchmark::ClobberMemory();
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(NUM_SKETCHES * items.size()));
+}
+
+// A new sketch with k = range(0) merges 100 sketches of range(1) items each.
+void BM_KllMergeInto(benchmark::State & state)
+{
+    constexpr size_t NUM_INPUTS = 100;
+    const auto k = static_cast<uint16_t>(state.range(0));
+    const auto items = makeItems(static_cast<size_t>(state.range(1)));
+    std::vector<Sketch> inputs(NUM_INPUTS, Sketch(k));
+    for (auto & input : inputs)
+        for (const double item : items)
+            input.update(item);
+
+    for (auto _ : state)
+    {
+        Sketch sketch(k);
+        for (const auto & input : inputs)
+            sketch.merge(input);
+
+        benchmark::DoNotOptimize(sketch.get_n());
+        benchmark::ClobberMemory();
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(NUM_INPUTS * items.size()));
+}
+
+// Serializes a sketch with k = range(0) and range(1) items.
+void BM_KllSerialize(benchmark::State & state)
+{
+    Sketch sketch(static_cast<uint16_t>(state.range(0)));
+    for (const double item : makeItems(static_cast<size_t>(state.range(1))))
+        sketch.update(item);
+
+    for (auto _ : state)
+    {
+        auto bytes = sketch.serialize();
+        benchmark::DoNotOptimize(bytes.data());
+        benchmark::ClobberMemory();
+    }
+}
+
+void buildArgs(benchmark::internal::Benchmark * b)
+{
+    for (const int64_t k : {200, 1000, 5000, 20000, 65535})
+    {
+        // n = k fills the buffer up to k with no compaction, so it includes every growth step
+        for (const int64_t n : {int64_t{1}, int64_t{10}, int64_t{100}, int64_t{1000}, k, int64_t{10000},
+                                int64_t{100000}, int64_t{1000000}})
+        {
+            if (n <= 100 * k)
+                b->Args({k, n});
+        }
+    }
+}
+
+void manySketchesArgs(benchmark::internal::Benchmark * b)
+{
+    for (const int64_t k : {200, 1000, 5000, 20000, 65535})
+        for (const int64_t n : {1, 10, 100, 1000})
+            b->Args({k, n});
+}
+
+void mergeArgs(benchmark::internal::Benchmark * b)
+{
+    for (const int64_t k : {200, 5000, 65535})
+        for (const int64_t n : {10, 100, 1000})
+            b->Args({k, n});
+}
+
+void serializeArgs(benchmark::internal::Benchmark * b)
+{
+    for (const int64_t k : {200, 5000, 65535})
+        for (const int64_t n : {10, 1000, 100000})
+            b->Args({k, n});
+}
+
+}
+
+BENCHMARK(BM_KllBuild)->Apply(buildArgs);
+BENCHMARK(BM_KllBuildManySketches)->Apply(manySketchesArgs)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_KllMergeInto)->Apply(mergeArgs)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_KllSerialize)->Apply(serializeArgs);

kll/include/kll_sketch.hpp

@@ -39,6 +39,10 @@ namespace kll_constants {
   const uint16_t MIN_K = DEFAULT_M;
   /// max value of parameter K
   const uint16_t MAX_K = (1 << 16) - 1;
+  /// initial size of the items buffer if K is larger
+  const uint16_t INITIAL_ITEMS_SIZE = DEFAULT_K;
+  /// growth factor of the items buffer until it reaches K
+  const resize_factor RESIZE_FACTOR = resize_factor::X8;
 }
 
 /**
@@ -183,7 +187,9 @@ class kll_sketch {
     using quantile_return_type = typename quantiles_sorted_view<T, C, A>::quantile_return_type;
 
     /**
-     * Constructor
+     * Constructor.
+     * The first item allocates space for min(k, INITIAL_ITEMS_SIZE) items,
+     * which grows by RESIZE_FACTOR up to k items before the first compaction.
      * @param k affects the size of the sketch and its estimation error
      * @param comparator strict weak ordering function (see C++ named requirements: Compare)
      * @param allocator used by this sketch to allocate memory
@@ -521,6 +527,7 @@ class kll_sketch {
 
     /**
      * Gets the sorted view of this sketch
+     * An empty sketch returns an empty view without allocating item storage.
      * @return the sorted view of this sketch
      */
     quantiles_sorted_view<T, C, A> get_sorted_view() const;
@@ -558,7 +565,7 @@ class kll_sketch {
     bool is_level_zero_sorted_;
     uint64_t n_;
     vector_u32 levels_;
-    T* items_;
+    T* items_; // nullptr until the sketch first receives items, see allocate_items()
     uint32_t items_size_;
     optional<T> min_item_;
     optional<T> max_item_;
@@ -572,6 +579,13 @@ class kll_sketch {
 
     // common update code
     inline void update_min_max(const T& item);
+    // items_ is allocated lazily, and these are the only places that deal with it being nullptr:
+    // allocates items_ (items_size_ items) unless it is allocated already
+    inline void allocate_items();
+    // copy-constructs the items of other, if other has any, into this newly constructed sketch
+    template<typename O> void copy_items(const O& other);
+    // &items_[index], or nullptr without item storage, which makes every item range empty
+    inline T* item_ptr(uint32_t index) const;
     inline uint32_t internal_update();
 
     // The following code is only valid in the special case of exactly reaching capacity while updating.
@@ -580,6 +594,8 @@ class kll_sketch {
 
     uint8_t find_level_to_compact() const;
     void add_empty_top_level_to_completely_full_sketch();
+    void grow_items();
+    uint32_t get_serialized_levels_offset() const;
     void sort_level_zero();
 
     template<typename O> void merge_higher_levels(O&& other, uint64_t final_n);