ClickHouse OOM when computing segment membership counters for a lot of projects at once #7942

ServerException: DB::Exception: (total) memory limit exceeded: would use 7.20 GiB (attempt to allocate chunk of 0.00 B), current RSS: 7.18 GiB, maximum: 7.20 GiB. OvercommitTracker decision: Memory overcommit has not freed enough memory: While executing MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder). Stack trace:

0. ./ci/tmp/build/./src/Common/Exception.cpp:141:1: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x0000000016932b2a
1. ./src/Common/Exception.h:171:100: DB::Exception::Exception(String&&, int, String, bool) @ 0x000000000e7ab5ce
2. ./src/Common/Exception.h:57:54: DB::Exception::Exception(PreformattedMessage&&, int) @ 0x000000000e7aaf09
3. ./src/Common/Exception.h:189:77: DB::Exception::Exception<char const*, char const*, String, String, String, String, String, char const*, std::basic_string_view<char, std::char_traits<char>>>(int, FormatStringHelperImpl<std::type_identity<char const*>::type, std::type_identity<char const*>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<char const*>::type, std::type_identity<std::basic_string_view<char, std::char_traits<char>>>::type>, char const*&&, char const*&&, String&&, String&&, String&&, String&&, String&&, char const*&&, std::basic_string_view<char, std::char_traits<char>>&&) @ 0x00000000169c09f2
4. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:417:23: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bce9f
5. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
6. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
7. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
8. ./src/Common/CurrentMemoryTracker.cpp:98:39: DB::IMergeTreeReader::IMergeTreeReader(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader>, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*, DB::MarkCache*, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&) @ 0x000000001d0b7371
9. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReaderCompact.cpp:32:7: DB::MergeTreeReaderCompact::MergeTreeReaderCompact(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader>, DB::NamesAndTypesList, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*, DB::MarkCache*, DB::DeserializationPrefixesCache*, DB::MarkRanges, DB::MergeTreeReaderSettings, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>>, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int) @ 0x000000001d434542
10. ./src/Storages/MergeTree/MergeTreeReaderCompactMultiBuffer.h:17:11: DB::MergeTreeReaderCompactMultiBuffer::MergeTreeReaderCompactMultiBuffer<std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*&, DB::MarkCache*&, DB::DeserializationPrefixesCache*&, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int>(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*&, DB::MarkCache*&, DB::DeserializationPrefixesCache*&, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int&&) @ 0x000000001d2d375b
11. ./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:773:30: DB::createMergeTreeReaderCompact(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::MarkRanges const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, DB::UncompressedCache*, DB::MarkCache*, DB::DeserializationPrefixesCache*, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&) @ 0x000000001d2ce48f
12. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadTask.cpp:156:16: DB::MergeTreeReadTask::createReaders(std::shared_ptr<DB::MergeTreeReadTaskInfo const> const&, DB::MergeTreeReadTask::Extras const&, DB::MarkRanges const&, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>> const&)::$_0::operator()(DB::NamesAndTypesList const&, bool) const @ 0x000000001d430592
13. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadTask.cpp:171:24: DB::MergeTreeReadTask::createReaders(std::shared_ptr<DB::MergeTreeReadTaskInfo const> const&, DB::MergeTreeReadTask::Extras const&, DB::MarkRanges const&, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>> const&) @ 0x000000001d42f437
14. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolBase.cpp:393:24: DB::MergeTreeReadPoolBase::createTask(std::shared_ptr<DB::MergeTreeReadTaskInfo const>, DB::MarkRanges, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>>, DB::MergeTreeReadTask*, std::shared_ptr<DB::RuntimeDataflowStatisticsCacheUpdater>) const @ 0x000000001d450d3b
15. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolBase.cpp:416:12: DB::MergeTreeReadPoolBase::createTask(std::shared_ptr<DB::MergeTreeReadTaskInfo const>, DB::MarkRanges, DB::MergeTreeReadTask*, std::shared_ptr<DB::RuntimeDataflowStatisticsCacheUpdater>) const @ 0x000000001d45277c
16. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp:80:12: DB::MergeTreeReadPoolInOrder::getTask(unsigned long, DB::MergeTreeReadTask*) @ 0x000000001e4a4df2
17. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeSelectProcessor.cpp:322:35: DB::MergeTreeSelectProcessor::read() @ 0x000000001d480e58
18. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeSource.cpp:230:41: DB::MergeTreeSource::tryGenerate() @ 0x000000001e4a5fda
19. ./ci/tmp/build/./src/Processors/ISource.cpp:110:26: DB::ISource::work() @ 0x000000001de0ec81
20. ./ci/tmp/build/./src/Processors/Executors/ExecutionThreadContext.cpp:53:26: DB::ExecutionThreadContext::executeTask() @ 0x000000001de2cebd
21. ./ci/tmp/build/./src/Processors/Executors/PipelineExecutor.cpp:351:26: DB::PipelineExecutor::executeStepImpl(unsigned long, DB::IAcquiredSlot*, std::atomic<bool>*) @ 0x000000001de1e484
22. ./ci/tmp/build/./src/Processors/Executors/PipelineExecutor.cpp:279:5: void std::__function::__policy_func<void ()>::__call_func[abi:fe210105]<DB::PipelineExecutor::spawnThreads(std::shared_ptr<DB::IAcquiredSlot>)::$_0>(std::__function::__policy_storage const*) @ 0x000000001de228c3
23. ./contrib/llvm-project/libcxx/include/__functional/function.h:508:12: ? @ 0x0000000016ac42d6
24. ....
(2 additional frame(s) were not displayed)
...
  File "clickhouse_driver/client.py", line 580, in process_ordinary_query
    return self.receive_result(with_column_types=with_column_types,
  File "clickhouse_driver/client.py", line 212, in receive_result
    return result.get_result()
  File "clickhouse_driver/result.py", line 50, in get_result
    for packet in self.packet_generator:
  File "clickhouse_driver/client.py", line 228, in packet_generator
    packet = self.receive_packet()
  File "clickhouse_driver/client.py", line 245, in receive_packet
    raise packet.exception

OperationalError: Code: 241.
DB::Exception: (total) memory limit exceeded: would use 7.20 GiB (attempt to allocate chunk of 0.00 B), current RSS: 7.18 GiB, maximum: 7.20 GiB. OvercommitTracker decision: Memory overcommit has not freed enough memory: While executing MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder). Stack trace:

0. ./ci/tmp/build/./src/Common/Exception.cpp:141:1: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x0000000016932b2a
1. ./src/Common/Exception.h:171:100: DB::Exception::Exception(String&&, int, String, bool) @ 0x000000000e7ab5ce
2. ./src/Common/Exception.h:57:54: DB::Exception::Exception(PreformattedMessage&&, int) @ 0x000000000e7aaf09
3. ./src/Common/Exception.h:189:77: DB::Exception::Exception<char const*, char const*, String, String, String, String, String, char const*, std::basic_string_view<char, std::char_traits<char>>>(int, FormatStringHelperImpl<std::type_identity<char const*>::type, std::type_identity<char const*>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<char const*>::type, std::type_identity<std::basic_string_view<char, std::char_traits<char>>>::type>, char const*&&, char const*&&, String&&, String&&, String&&, String&&, String&&, char const*&&, std::basic_string_view<char, std::char_traits<char>>&&) @ 0x00000000169c09f2
4. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:417:23: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bce9f
5. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
6. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
7. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
8. ./src/Common/CurrentMemoryTracker.cpp:98:39: DB::IMergeTreeReader::IMergeTreeReader(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader>, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*, DB::MarkCache*, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&) @ 0x000000001d0b7371
9. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReaderCompact.cpp:32:7: DB::MergeTreeReaderCompact::MergeTreeReaderCompact(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader>, DB::NamesAndTypesList, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*, DB::MarkCache*, DB::DeserializationPrefixesCache*, DB::MarkRanges, DB::MergeTreeReaderSettings, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>>, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int) @ 0x000000001d434542
10. ./src/Storages/MergeTree/MergeTreeReaderCompactMultiBuffer.h:17:11: DB::MergeTreeReaderCompactMultiBuffer::MergeTreeReaderCompactMultiBuffer<std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*&, DB::MarkCache*&, DB::DeserializationPrefixesCache*&, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int>(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*&, DB::MarkCache*&, DB::DeserializationPrefixesCache*&, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int&&) @ 0x000000001d2d375b
11. ./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:773:30: DB::createMergeTreeReaderCompact(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::MarkRanges const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, DB::UncompressedCache*, DB::MarkCache*, DB::DeserializationPrefixesCache*, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&) @ 0x000000001d2ce48f
12. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadTask.cpp:156:16: DB::MergeTreeReadTask::createReaders(std::shared_ptr<DB::MergeTreeReadTaskInfo const> const&, DB::MergeTreeReadTask::Extras const&, DB::MarkRanges const&, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>> const&)::$_0::operator()(DB::NamesAndTypesList const&, bool) const @ 0x000000001d430592
13. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadTask.cpp:171:24: DB::MergeTreeReadTask::createReaders(std::shared_ptr<DB::MergeTreeReadTaskInfo const> const&, DB::MergeTreeReadTask::Extras const&, DB::MarkRanges const&, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>> const&) @ 0x000000001d42f437
14. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolBase.cpp:393:24: DB::MergeTreeReadPoolBase::createTask(std::shared_ptr<DB::MergeTreeReadTaskInfo const>, DB::MarkRanges, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>>, DB::MergeTreeReadTask*, std::shared_ptr<DB::RuntimeDataflowStatisticsCacheUpdater>) const @ 0x000000001d450d3b
15. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolBase.cpp:416:12: DB::MergeTreeReadPoolBase::createTask(std::shared_ptr<DB::MergeTreeReadTaskInfo const>, DB::MarkRanges, DB::MergeTreeReadTask*, std::shared_ptr<DB::RuntimeDataflowStatisticsCacheUpdater>) const @ 0x000000001d45277c
16. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp:80:12: DB::MergeTreeReadPoolInOrder::getTask(unsigned long, DB::MergeTreeReadTask*) @ 0x000000001e4a4df2
17. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeSelectProcessor.cpp:322:35: DB::MergeTreeSelectProcessor::read() @ 0x000000001d480e58
18. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeSource.cpp:230:41: DB::MergeTreeSource::tryGenerate() @ 0x000000001e4a5fda
19. ./ci/tmp/build/./src/Processors/ISource.cpp:110:26: DB::ISource::work() @ 0x000000001de0ec81
20. ./ci/tmp/build/./src/Processors/Executors/ExecutionThreadContext.cpp:53:26: DB::ExecutionThreadContext::executeTask() @ 0x000000001de2cebd
21. ./ci/tmp/build/./src/Processors/Executors/PipelineExecutor.cpp:351:26: DB::PipelineExecutor::executeStepImpl(unsigned long, DB::IAcquiredSlot*, std::atomic<bool>*) @ 0x000000001de1e484
22. ./ci/tmp/build/./src/Processors/Executors/PipelineExecutor.cpp:279:5: void std::__function::__policy_func<void ()>::__call_func[abi:fe210105]<DB::PipelineExecutor::spawnThreads(std::shared_ptr<DB::IAcquiredSlot>)::$_0>(std::__function::__policy_storage const*) @ 0x000000001de228c3
23. ./contrib/llvm-project/libcxx/include/__functional/function.h:508:12: ? @ 0x0000000016a...
  File "django/db/backends/utils.py", line 105, in _execute
    return self.cursor.execute(sql, params)
  File "clickhouse_backend/driver/connection.py", line 171, in execute
    super().execute(operation, parameters)
  File "clickhouse_driver/dbapi/cursor.py", line 117, in execute
    raise OperationalError(orig)

OperationalError: Code: 241.
DB::Exception: (total) memory limit exceeded: would use 7.20 GiB (attempt to allocate chunk of 0.00 B), current RSS: 7.18 GiB, maximum: 7.20 GiB. OvercommitTracker decision: Memory overcommit has not freed enough memory: While executing MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder). Stack trace:

0. ./ci/tmp/build/./src/Common/Exception.cpp:141:1: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x0000000016932b2a
1. ./src/Common/Exception.h:171:100: DB::Exception::Exception(String&&, int, String, bool) @ 0x000000000e7ab5ce
2. ./src/Common/Exception.h:57:54: DB::Exception::Exception(PreformattedMessage&&, int) @ 0x000000000e7aaf09
3. ./src/Common/Exception.h:189:77: DB::Exception::Exception<char const*, char const*, String, String, String, String, String, char const*, std::basic_string_view<char, std::char_traits<char>>>(int, FormatStringHelperImpl<std::type_identity<char const*>::type, std::type_identity<char const*>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<char const*>::type, std::type_identity<std::basic_string_view<char, std::char_traits<char>>>::type>, char const*&&, char const*&&, String&&, String&&, String&&, String&&, String&&, char const*&&, std::basic_string_view<char, std::char_traits<char>>&&) @ 0x00000000169c09f2
4. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:417:23: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bce9f
5. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
6. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
7. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600
8. ./src/Common/CurrentMemoryTracker.cpp:98:39: DB::IMergeTreeReader::IMergeTreeReader(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader>, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*, DB::MarkCache*, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&) @ 0x000000001d0b7371
9. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReaderCompact.cpp:32:7: DB::MergeTreeReaderCompact::MergeTreeReaderCompact(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader>, DB::NamesAndTypesList, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*, DB::MarkCache*, DB::DeserializationPrefixesCache*, DB::MarkRanges, DB::MergeTreeReaderSettings, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>>, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int) @ 0x000000001d434542
10. ./src/Storages/MergeTree/MergeTreeReaderCompactMultiBuffer.h:17:11: DB::MergeTreeReaderCompactMultiBuffer::MergeTreeReaderCompactMultiBuffer<std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*&, DB::MarkCache*&, DB::DeserializationPrefixesCache*&, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int>(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*&, DB::MarkCache*&, DB::DeserializationPrefixesCache*&, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int&&) @ 0x000000001d2d375b
11. ./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:773:30: DB::createMergeTreeReaderCompact(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::MarkRanges const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, DB::UncompressedCache*, DB::MarkCache*, DB::DeserializationPrefixesCache*, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&) @ 0x000000001d2ce48f
12. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadTask.cpp:156:16: DB::MergeTreeReadTask::createReaders(std::shared_ptr<DB::MergeTreeReadTaskInfo const> const&, DB::MergeTreeReadTask::Extras const&, DB::MarkRanges const&, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>> const&)::$_0::operator()(DB::NamesAndTypesList const&, bool) const @ 0x000000001d430592
13. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadTask.cpp:171:24: DB::MergeTreeReadTask::createReaders(std::shared_ptr<DB::MergeTreeReadTaskInfo const> const&, DB::MergeTreeReadTask::Extras const&, DB::MarkRanges const&, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>> const&) @ 0x000000001d42f437
14. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolBase.cpp:393:24: DB::MergeTreeReadPoolBase::createTask(std::shared_ptr<DB::MergeTreeReadTaskInfo const>, DB::MarkRanges, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>>, DB::MergeTreeReadTask*, std::shared_ptr<DB::RuntimeDataflowStatisticsCacheUpdater>) const @ 0x000000001d450d3b
15. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolBase.cpp:416:12: DB::MergeTreeReadPoolBase::createTask(std::shared_ptr<DB::MergeTreeReadTaskInfo const>, DB::MarkRanges, DB::MergeTreeReadTask*, std::shared_ptr<DB::RuntimeDataflowStatisticsCacheUpdater>) const @ 0x000000001d45277c
16. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp:80:12: DB::MergeTreeReadPoolInOrder::getTask(unsigned long, DB::MergeTreeReadTask*) @ 0x000000001e4a4df2
17. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeSelectProcessor.cpp:322:35: DB::MergeTreeSelectProcessor::read() @ 0x000000001d480e58
18. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeSource.cpp:230:41: DB::MergeTreeSource::tryGenerate() @ 0x000000001e4a5fda
19. ./ci/tmp/build/./src/Processors/ISource.cpp:110:26: DB::ISource::work() @ 0x000000001de0ec81
20. ./ci/tmp/build/./src/Processors/Executors/ExecutionThreadContext.cpp:53:26: DB::ExecutionThreadContext::executeTask() @ 0x000000001de2cebd
21. ./ci/tmp/build/./src/Processors/Executors/PipelineExecutor.cpp:351:26: DB::PipelineExecutor::executeStepImpl(unsigned long, DB::IAcquiredSlot*, std::atomic<bool>*) @ 0x000000001de1e484
22. ./ci/tmp/build/./src/Processors/Executors/PipelineExecutor.cpp:279:5: void std::__function::__policy_func<void ()>::__call_func[abi:fe210105]<DB::PipelineExecutor::spawnThreads(std::shared_ptr<DB::IAcquiredSlot>)::$_0>(std::__function::__policy_storage const*) @ 0x000000001de228c3
23. ./contrib/llvm-project/libcxx/include/__functional/function.h:508:12: ? @ 0x0000000016a...
(4 additional frame(s) were not displayed)
...
  File "segment_membership/tasks.py", line 222, in refresh_project_segment_counts
    membership_counts = compute_segment_counts_for_project(project, cursor)
  File "segment_membership/services.py", line 141, in compute_segment_counts_for_project
    cursor.execute(sql, {"env_keys": tuple(env_id_by_key)})

{'project__id': 6851, 'event': 'refresh.project.failed', 'logger': 'segment_membership', 'level': 'error', 'exception': 'Traceback (most recent call last):\n  File "/build/.venv/lib/python3.13/site-packages/clickhouse_driver/dbapi/cursor.py", line 111, in execute\n    response = execute(\n        operation, params=parameters, with_column_types=True,\n        **execute_kwargs\n    )\n  File "/build/.venv/lib/python3.13/site-packages/clickhouse_backend/driver/client.py", line 53, in execute\n    rv = self.process_ordinary_query(\n        query,\n    ...<5 lines>...\n        columnar=columnar,\n    )\n  File "/build/.venv/lib/python3.13/site-packages/clickhouse_driver/client.py", line 580, in process_ordinary_query\n    return self.receive_result(with_column_types=with_column_types,\n           ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n                               columnar=columnar)\n                               ^^^^^^^^^^^^^^^^^^\n  File "/build/.venv/lib/python3.13/site-packages/sentry_sdk/integrations/clickhouse_driver.py", line 106, in _inner_end\n    res = f(*args, **kwargs)\n  File "/build/.venv/lib/python3.13/site-packages/clickhouse_driver/client.py", line 212, in receive_result\n    return result.get_result()\n           ~~~~~~~~~~~~~~~~~^^\n  File "/build/.venv/lib/python3.13/site-packages/clickhouse_driver/result.py", line 50, in get_result\n    for packet in self.packet_generator:\n                  ^^^^^^^^^^^^^^^^^^^^^\n  File "/build/.venv/lib/python3.13/site-packages/clickhouse_driver/client.py", line 228, in packet_generator\n    packet = self.receive_packet()\n  File "/build/.venv/lib/python3.13/site-packages/clickhouse_driver/client.py", line 245, in receive_packet\n    raise packet.exception\nclickhouse_driver.errors.ServerException: Code: 241.\nDB::Exception: (total) memory limit exceeded: would use 7.20 GiB (attempt to allocate chunk of 0.00 B), current RSS: 7.18 GiB, maximum: 7.20 GiB. OvercommitTracker decision: Memory overcommit has not freed enough memory: While executing MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder). Stack trace:\n\n0. ./ci/tmp/build/./src/Common/Exception.cpp:141:1: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x0000000016932b2a\n1. ./src/Common/Exception.h:171:100: DB::Exception::Exception(String&&, int, String, bool) @ 0x000000000e7ab5ce\n2. ./src/Common/Exception.h:57:54: DB::Exception::Exception(PreformattedMessage&&, int) @ 0x000000000e7aaf09\n3. ./src/Common/Exception.h:189:77: DB::Exception::Exception<char const*, char const*, String, String, String, String, String, char const*, std::basic_string_view<char, std::char_traits<char>>>(int, FormatStringHelperImpl<std::type_identity<char const*>::type, std::type_identity<char const*>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<String>::type, std::type_identity<char const*>::type, std::type_identity<std::basic_string_view<char, std::char_traits<char>>>::type>, char const*&&, char const*&&, String&&, String&&, String&&, String&&, String&&, char const*&&, std::basic_string_view<char, std::char_traits<char>>&&) @ 0x00000000169c09f2\n4. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:417:23: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bce9f\n5. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600\n6. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600\n7. ./ci/tmp/build/./src/Common/MemoryTracker.cpp:484:29: MemoryTracker::allocImpl(long, bool, MemoryTracker*, double) @ 0x00000000169bc600\n8. ./src/Common/CurrentMemoryTracker.cpp:98:39: DB::IMergeTreeReader::IMergeTreeReader(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader>, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*, DB::MarkCache*, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&) @ 0x000000001d0b7371\n9. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReaderCompact.cpp:32:7: DB::MergeTreeReaderCompact::MergeTreeReaderCompact(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader>, DB::NamesAndTypesList, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*, DB::MarkCache*, DB::DeserializationPrefixesCache*, DB::MarkRanges, DB::MergeTreeReaderSettings, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>>, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int) @ 0x000000001d434542\n10. ./src/Storages/MergeTree/MergeTreeReaderCompactMultiBuffer.h:17:11: DB::MergeTreeReaderCompactMultiBuffer::MergeTreeReaderCompactMultiBuffer<std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*&, DB::MarkCache*&, DB::DeserializationPrefixesCache*&, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int>(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::UncompressedCache*&, DB::MarkCache*&, DB::DeserializationPrefixesCache*&, DB::MarkRanges const&, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int&&) @ 0x000000001d2d375b\n11. ./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:773:30: DB::createMergeTreeReaderCompact(std::shared_ptr<DB::IMergeTreeDataPartInfoForReader> const&, DB::NamesAndTypesList const&, std::shared_ptr<DB::StorageSnapshot> const&, std::shared_ptr<DB::MergeTreeSettings const> const&, DB::MarkRanges const&, std::unordered_map<String, DB::Field, std::hash<String>, std::equal_to<String>, std::allocator<std::pair<String const, DB::Field>>> const&, DB::UncompressedCache*, DB::MarkCache*, DB::DeserializationPrefixesCache*, DB::MergeTreeReaderSettings const&, std::map<String, double, std::less<String>, std::allocator<std::pair<String const, double>>> const&, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&) @ 0x000000001d2ce48f\n12. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadTask.cpp:156:16: DB::MergeTreeReadTask::createReaders(std::shared_ptr<DB::MergeTreeReadTaskInfo const> const&, DB::MergeTreeReadTask::Extras const&, DB::MarkRanges const&, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>> const&)::$_0::operator()(DB::NamesAndTypesList const&, bool) const @ 0x000000001d430592\n13. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadTask.cpp:171:24: DB::MergeTreeReadTask::createReaders(std::shared_ptr<DB::MergeTreeReadTaskInfo const> const&, DB::MergeTreeReadTask::Extras const&, DB::MarkRanges const&, std::vector<DB::MarkRanges, std::allocator<DB::MarkRanges>> const&) @ 0x000000001d42f437\n14. ./ci/tmp/build/./src/Storages/MergeTree/MergeTreeReadPoolBase.cpp:393:24: DB...