diff --git a/src/duckdb/src/catalog/catalog_search_path.cpp b/src/duckdb/src/catalog/catalog_search_path.cpp index 6388b9134..251efd5f3 100644 --- a/src/duckdb/src/catalog/catalog_search_path.cpp +++ b/src/duckdb/src/catalog/catalog_search_path.cpp @@ -195,8 +195,15 @@ void CatalogSearchPath::Set(CatalogSearchEntry new_value, CatalogSetPathType set Set(std::move(new_paths), set_type); } -const vector &CatalogSearchPath::Get() const { - return paths; +vector CatalogSearchPath::Get() const { + vector res; + for (auto &path : paths) { + if (path.schema.empty()) { + continue; + } + res.emplace_back(path); + } + return res; } string CatalogSearchPath::GetDefaultSchema(const string &catalog) const { @@ -248,7 +255,7 @@ vector CatalogSearchPath::GetCatalogsForSchema(const string &schema) con catalogs.push_back(SYSTEM_CATALOG); } else { for (auto &path : paths) { - if (StringUtil::CIEquals(path.schema, schema)) { + if (StringUtil::CIEquals(path.schema, schema) || path.schema.empty()) { catalogs.push_back(path.catalog); } } @@ -259,7 +266,7 @@ vector CatalogSearchPath::GetCatalogsForSchema(const string &schema) con vector CatalogSearchPath::GetSchemasForCatalog(const string &catalog) const { vector schemas; for (auto &path : paths) { - if (StringUtil::CIEquals(path.catalog, catalog)) { + if (!path.schema.empty() && StringUtil::CIEquals(path.catalog, catalog)) { schemas.push_back(path.schema); } } @@ -267,8 +274,8 @@ vector CatalogSearchPath::GetSchemasForCatalog(const string &catalog) co } const CatalogSearchEntry &CatalogSearchPath::GetDefault() const { - const auto &paths = Get(); D_ASSERT(paths.size() >= 2); + D_ASSERT(!paths[1].schema.empty()); return paths[1]; } diff --git a/src/duckdb/src/common/adbc/adbc.cpp b/src/duckdb/src/common/adbc/adbc.cpp index fcbfe191f..f852a71e6 100644 --- a/src/duckdb/src/common/adbc/adbc.cpp +++ b/src/duckdb/src/common/adbc/adbc.cpp @@ -13,6 +13,8 @@ #include "duckdb/common/adbc/single_batch_array_stream.hpp" #include "duckdb/function/table/arrow.hpp" #include "duckdb/common/adbc/wrappers.hpp" +#include +#include #include #include @@ -862,6 +864,22 @@ AdbcStatusCode StatementExecuteQuery(struct AdbcStatement *statement, struct Arr if (has_stream && to_table) { return IngestToTableFromBoundStream(wrapper, error); } + + if (!wrapper->statement) { + if (out) { + out->private_data = nullptr; + out->get_schema = nullptr; + out->get_next = nullptr; + out->release = nullptr; + out->get_last_error = nullptr; + } + + if (rows_affected) { + *rows_affected = 0; + } + return ADBC_STATUS_OK; + } + auto stream_wrapper = static_cast(malloc(sizeof(DuckDBAdbcStreamWrapper))); if (has_stream) { // A stream was bound to the statement, use that to bind parameters @@ -987,6 +1005,12 @@ AdbcStatusCode StatementSetSqlQuery(struct AdbcStatement *statement, const char return ADBC_STATUS_INVALID_ARGUMENT; } + auto query_len = strlen(query); + if (std::all_of(query, query + query_len, duckdb::StringUtil::CharacterIsSpace)) { + SetError(error, "No statements found"); + return ADBC_STATUS_INVALID_ARGUMENT; + } + auto wrapper = static_cast(statement->private_data); if (wrapper->ingestion_stream.release) { // Release any resources currently held by the ingestion stream before we overwrite it @@ -1006,6 +1030,13 @@ AdbcStatusCode StatementSetSqlQuery(struct AdbcStatement *statement, const char duckdb_destroy_extracted(&extracted_statements); return ADBC_STATUS_INTERNAL; } + + if (extract_statements_size == 0) { + // Query is non-empty, but there are no actual statements. + duckdb_destroy_extracted(&extracted_statements); + return ADBC_STATUS_OK; + } + // Now lets loop over the statements, and execute every one for (idx_t i = 0; i < extract_statements_size - 1; i++) { duckdb_prepared_statement statement_internal = nullptr; diff --git a/src/duckdb/src/function/scalar/create_sort_key.cpp b/src/duckdb/src/function/scalar/create_sort_key.cpp index 1a8570451..a44fdbcd9 100644 --- a/src/duckdb/src/function/scalar/create_sort_key.cpp +++ b/src/duckdb/src/function/scalar/create_sort_key.cpp @@ -703,8 +703,9 @@ void FinalizeSortData(Vector &result, idx_t size, const SortKeyLengthInfo &key_l auto result_data = FlatVector::GetData(result); // call Finalize on the result for (idx_t r = 0; r < size; r++) { - result_data[r].SetSizeAndFinalize(offsets[r], - key_lengths.variable_lengths[r] + key_lengths.constant_length); + const auto offset = static_cast(offsets[r]); + const auto allocated_size = key_lengths.variable_lengths[r] + key_lengths.constant_length; + result_data[r].SetSizeAndFinalize(offset, allocated_size); } break; } diff --git a/src/duckdb/src/function/scalar/generic/constant_or_null.cpp b/src/duckdb/src/function/scalar/generic/constant_or_null.cpp index c5c4307bd..e8cbda904 100644 --- a/src/duckdb/src/function/scalar/generic/constant_or_null.cpp +++ b/src/duckdb/src/function/scalar/generic/constant_or_null.cpp @@ -37,6 +37,7 @@ static void ConstantOrNullFunction(DataChunk &args, ExpressionState &state, Vect // there are null values: need to merge them into the result result.Flatten(args.size()); auto &result_mask = FlatVector::Validity(result); + result_mask.EnsureWritable(); result_mask.Combine(input_mask, args.size()); } break; diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index 6063e2aef..74b66360a 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "4-dev67" +#define DUCKDB_PATCH_VERSION "4-dev100" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 4 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 1 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v1.4.4-dev67" +#define DUCKDB_VERSION "v1.4.4-dev100" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "5a334c23da" +#define DUCKDB_SOURCE_ID "30bc6df28d" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp b/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp index bbe396cd9..5b2ab62c5 100644 --- a/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +++ b/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp @@ -48,7 +48,7 @@ class CatalogSearchPath { DUCKDB_API void Set(vector new_paths, CatalogSetPathType set_type); DUCKDB_API void Reset(); - DUCKDB_API const vector &Get() const; + DUCKDB_API vector Get() const; const vector &GetSetPaths() const { return set_paths; } diff --git a/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp b/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp index 939b570ec..34f9d935b 100644 --- a/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +++ b/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp @@ -6,6 +6,10 @@ // //===----------------------------------------------------------------------===// +//! NOTE: This file should not be included directly. +//! NOTE: When included directly, this file produces 'unused static method' warnings/errors. +//! NOTE: The methods in this file should be used through the TryCast:: methods defined in 'cast_operators.hpp'. + #pragma once #include "duckdb/common/operator/cast_operators.hpp" @@ -19,20 +23,18 @@ namespace duckdb { -//! Note: this should not be included directly, when these methods are required -//! They should be used through the TryCast:: methods defined in 'cast_operators.hpp' -//! This file produces 'unused static method' warnings/errors when included - template static bool TryCastWithOverflowCheck(SRC value, DST &result) { if (!Value::IsFinite(value)) { return false; } + + // SRC and DST do not have the same sign. if (NumericLimits::IsSigned() != NumericLimits::IsSigned()) { + // SRC is signed. if (NumericLimits::IsSigned()) { - // signed to unsigned conversion if (NumericLimits::Digits() > NumericLimits::Digits()) { - if (value < 0 || value > (SRC)NumericLimits::Maximum()) { + if (value < 0 || value > static_cast(NumericLimits::Maximum())) { return false; } } else { @@ -40,34 +42,33 @@ static bool TryCastWithOverflowCheck(SRC value, DST &result) { return false; } } - result = (DST)value; + result = static_cast(value); return true; - } else { - // unsigned to signed conversion - if (NumericLimits::Digits() >= NumericLimits::Digits()) { - if (value <= (SRC)NumericLimits::Maximum()) { - result = (DST)value; - return true; - } - return false; - } else { - result = (DST)value; + } + + // SRC is unsigned. + if (NumericLimits::Digits() >= NumericLimits::Digits()) { + if (value <= static_cast(NumericLimits::Maximum())) { + result = static_cast(value); return true; } + return false; } - } else { - // same sign conversion - if (NumericLimits::Digits() >= NumericLimits::Digits()) { - result = (DST)value; - return true; - } else { - if (value < SRC(NumericLimits::Minimum()) || value > SRC(NumericLimits::Maximum())) { - return false; - } - result = (DST)value; - return true; - } + result = static_cast(value); + return true; + } + + // SRC and DST have the same sign. + if (NumericLimits::Digits() >= NumericLimits::Digits()) { + result = static_cast(value); + return true; } + + if (value < SRC(NumericLimits::Minimum()) || value > SRC(NumericLimits::Maximum())) { + return false; + } + result = static_cast(value); + return true; } template @@ -594,14 +595,14 @@ struct NumericTryCastToBit { struct NumericTryCast { template - static inline bool Operation(SRC input, DST &result, bool strict = false) { + static bool Operation(SRC input, DST &result, bool strict = false) { return TryCastWithOverflowCheck(input, result); } }; struct NumericCast { template - static inline DST Operation(SRC input) { + static DST Operation(SRC input) { DST result; if (!NumericTryCast::Operation(input, result)) { throw InvalidInputException(CastExceptionText(input)); diff --git a/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp b/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp index 1e0ccf305..2d24ff75d 100644 --- a/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp +++ b/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp @@ -119,11 +119,11 @@ class BlockIteratorState return GetValueAtIndex(block_idx, tuple_idx); } - void SetKeepPinned(const bool &) { + static void SetKeepPinned(const bool &) { // NOP } - void SetPinPayload(const bool &) { + static void SetPinPayload(const bool &) { // NOP } diff --git a/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp b/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp index 30dd141b3..8b18a0f53 100644 --- a/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +++ b/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp @@ -29,7 +29,8 @@ class TableDataWriter { public: void WriteTableData(Serializer &metadata_serializer); - virtual void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) = 0; + virtual void WriteUnchangedTable(MetaBlockPointer pointer, const vector &metadata_pointers, + idx_t total_rows) = 0; virtual void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, Serializer &serializer) = 0; virtual unique_ptr GetRowGroupWriter(RowGroup &row_group) = 0; @@ -54,7 +55,8 @@ class SingleFileTableDataWriter : public TableDataWriter { MetadataWriter &table_data_writer); public: - void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override; + void WriteUnchangedTable(MetaBlockPointer pointer, const vector &metadata_pointers, + idx_t total_rows) override; void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, Serializer &serializer) override; unique_ptr GetRowGroupWriter(RowGroup &row_group) override; diff --git a/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp b/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp index e36e6169d..fb28af596 100644 --- a/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/in_memory_checkpoint.hpp @@ -49,7 +49,8 @@ class InMemoryTableDataWriter : public TableDataWriter { InMemoryTableDataWriter(InMemoryCheckpointer &checkpoint_manager, TableCatalogEntry &table); public: - void WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) override; + void WriteUnchangedTable(MetaBlockPointer pointer, const vector &metadata_pointers, + idx_t total_rows) override; void FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, RowGroupCollection &collection, Serializer &serializer) override; unique_ptr GetRowGroupWriter(RowGroup &row_group) override; diff --git a/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp b/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp index fa806478c..8a5152cc4 100644 --- a/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp @@ -23,6 +23,7 @@ class PersistentTableData { ~PersistentTableData(); MetaBlockPointer base_table_pointer; + vector read_metadata_pointers; TableStatistics table_stats; idx_t total_rows; idx_t row_group_count; diff --git a/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp b/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp index f09fcdeb3..0b01317ce 100644 --- a/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp @@ -52,7 +52,7 @@ class RowGroupCollection { void Initialize(PersistentCollectionData &data); void Initialize(PersistentTableData &data); void InitializeEmpty(); - void FinalizeCheckpoint(MetaBlockPointer pointer); + void FinalizeCheckpoint(MetaBlockPointer pointer, const vector &existing_pointers); bool IsEmpty() const; @@ -175,6 +175,8 @@ class RowGroupCollection { atomic allocation_size; //! Root metadata pointer, if the collection is loaded from disk MetaBlockPointer metadata_pointer; + //! Other metadata pointers + vector metadata_pointers; //! Whether or not we need to append a new row group prior to appending bool requires_new_row_group; }; diff --git a/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp b/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp index d93610138..a04203c57 100644 --- a/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp @@ -21,7 +21,7 @@ class RowGroupSegmentTree : public SegmentTree { explicit RowGroupSegmentTree(RowGroupCollection &collection); ~RowGroupSegmentTree() override; - void Initialize(PersistentTableData &data); + void Initialize(PersistentTableData &data, optional_ptr> read_pointers = nullptr); MetaBlockPointer GetRootPointer() const { return root_pointer; diff --git a/src/duckdb/src/planner/binder/statement/bind_create.cpp b/src/duckdb/src/planner/binder/statement/bind_create.cpp index f1ffd7496..24abb81f4 100644 --- a/src/duckdb/src/planner/binder/statement/bind_create.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_create.cpp @@ -50,34 +50,40 @@ namespace duckdb { void Binder::BindSchemaOrCatalog(CatalogEntryRetriever &retriever, string &catalog, string &schema) { auto &context = retriever.GetContext(); - if (catalog.empty() && !schema.empty()) { - // schema is specified - but catalog is not - // try searching for the catalog instead - auto &db_manager = DatabaseManager::Get(context); - auto database = db_manager.GetDatabase(context, schema); - if (database) { - // we have a database with this name - // check if there is a schema - auto &search_path = retriever.GetSearchPath(); - auto catalog_names = search_path.GetCatalogsForSchema(schema); - if (catalog_names.empty()) { - catalog_names.push_back(DatabaseManager::GetDefaultDatabase(context)); - } - for (auto &catalog_name : catalog_names) { - auto catalog_ptr = Catalog::GetCatalogEntry(retriever, catalog_name); - if (!catalog_ptr) { - continue; - } - if (catalog_ptr->CheckAmbiguousCatalogOrSchema(context, schema)) { - throw BinderException( - "Ambiguous reference to catalog or schema \"%s\" - use a fully qualified path like \"%s.%s\"", - schema, catalog_name, schema); - } - } - catalog = schema; - schema = string(); + if (schema.empty()) { + return; + } + if (!catalog.empty()) { + return; + } + // schema is specified - but catalog is not + // try searching for the catalog instead + auto &db_manager = DatabaseManager::Get(context); + auto database = db_manager.GetDatabase(context, schema); + if (!database) { + //! No database by that name was found + return; + } + // we have a database with this name + // check if there is a schema + auto &search_path = retriever.GetSearchPath(); + auto catalog_names = search_path.GetCatalogsForSchema(schema); + if (catalog_names.empty()) { + catalog_names.push_back(DatabaseManager::GetDefaultDatabase(context)); + } + for (auto &catalog_name : catalog_names) { + auto catalog_ptr = Catalog::GetCatalogEntry(retriever, catalog_name); + if (!catalog_ptr) { + continue; + } + if (catalog_ptr->CheckAmbiguousCatalogOrSchema(context, schema)) { + throw BinderException( + "Ambiguous reference to catalog or schema \"%s\" - use a fully qualified path like \"%s.%s\"", schema, + catalog_name, schema); } } + catalog = schema; + schema = string(); } void Binder::BindSchemaOrCatalog(ClientContext &context, string &catalog, string &schema) { diff --git a/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp b/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp index f8bf0e27b..cb10042a0 100644 --- a/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +++ b/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp @@ -101,8 +101,10 @@ vector Binder::GetSearchPath(Catalog &catalog, const string } auto default_schema = catalog.GetDefaultSchema(); if (schema_name.empty() && schema_name != default_schema) { - view_search_path.emplace_back(catalog.GetName(), default_schema); + view_search_path.emplace_back(catalog_name, default_schema); } + //! Signal that this catalog should be checked, regardless of the schema in the reference + view_search_path.emplace_back(catalog_name, INVALID_SCHEMA); return view_search_path; } @@ -267,7 +269,6 @@ unique_ptr Binder::Bind(BaseTableRef &ref) { table_or_view = entry_retriever.GetEntry(ref.catalog_name, ref.schema_name, table_lookup, OnEntryNotFound::THROW_EXCEPTION); } - switch (table_or_view->type) { case CatalogType::TABLE_ENTRY: { // base table: create the BoundBaseTableRef node diff --git a/src/duckdb/src/storage/checkpoint/table_data_writer.cpp b/src/duckdb/src/storage/checkpoint/table_data_writer.cpp index 4bbdff138..4a26a6531 100644 --- a/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +++ b/src/duckdb/src/storage/checkpoint/table_data_writer.cpp @@ -2,10 +2,13 @@ #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/common/serializer/binary_deserializer.hpp" #include "duckdb/common/serializer/binary_serializer.hpp" #include "duckdb/main/database.hpp" #include "duckdb/main/settings.hpp" #include "duckdb/parallel/task_scheduler.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/storage/checkpoint/table_data_reader.hpp" #include "duckdb/storage/table/column_checkpoint_state.hpp" #include "duckdb/storage/table/table_statistics.hpp" #include "duckdb/storage/metadata/metadata_reader.hpp" @@ -59,8 +62,11 @@ MetadataManager &SingleFileTableDataWriter::GetMetadataManager() { return checkpoint_manager.GetMetadataManager(); } -void SingleFileTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) { +void SingleFileTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, + const vector &metadata_pointers, + idx_t total_rows) { existing_pointer = pointer; + existing_pointers = metadata_pointers; existing_rows = total_rows; } @@ -68,11 +74,14 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat RowGroupCollection &collection, Serializer &serializer) { MetaBlockPointer pointer; idx_t total_rows; + auto debug_verify_blocks = DBConfig::GetSetting(GetDatabase()); if (!existing_pointer.IsValid()) { // write the metadata // store the current position in the metadata writer // this is where the row groups for this table start pointer = table_data_writer.GetMetaBlockPointer(); + vector written_pointers; + table_data_writer.SetWrittenPointers(written_pointers); // Serialize statistics as a single unit BinarySerializer stats_serializer(table_data_writer, serializer.GetOptions()); @@ -95,7 +104,8 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat RowGroup::Serialize(row_group_pointer, row_group_serializer); row_group_serializer.End(); } - collection.FinalizeCheckpoint(pointer); + table_data_writer.SetWrittenPointers(nullptr); + collection.FinalizeCheckpoint(pointer, written_pointers); } else { // we have existing metadata and the table is unchanged - write a pointer to the existing metadata pointer = existing_pointer; @@ -103,9 +113,44 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat // label the blocks as used again to prevent them from being freed auto &metadata_manager = checkpoint_manager.GetMetadataManager(); - MetadataReader reader(metadata_manager, pointer); - auto blocks = reader.GetRemainingBlocks(); - metadata_manager.ClearModifiedBlocks(blocks); + metadata_manager.ClearModifiedBlocks(existing_pointers); + + // verify that existing_pointers indeed corresponds to the metadata blocks + if (debug_verify_blocks) { + vector read_pointers; + MetadataReader reader(metadata_manager, pointer, read_pointers); + auto bound_info = Binder::BindCreateTableCheckpoint(table.GetInfo(), table.schema); + TableDataReader data_reader(reader, *bound_info, pointer); + data_reader.ReadTableData(); + for (idx_t row_group = 0; row_group < bound_info->data->row_group_count; ++row_group) { + BinaryDeserializer deserializer(reader); + deserializer.Begin(); + auto row_group_pointer = RowGroup::Deserialize(deserializer); + deserializer.End(); + } + set existing_block_ids; + for (auto &ptr : existing_pointers) { + existing_block_ids.insert(ptr.block_pointer); + } + set all_read_block_ids; + for (auto &ptr : read_pointers) { + all_read_block_ids.insert(ptr.block_pointer); + } + if (existing_block_ids != all_read_block_ids) { + std::stringstream oss; + oss << "Existing: "; + for (auto &block : existing_pointers) { + oss << block << ", "; + } + oss << "\n"; + oss << "Read: "; + for (auto &block : read_pointers) { + oss << block << ", "; + } + oss << "\n"; + throw InternalException("Reading existing blocks does not yield same blocks: " + oss.str()); + } + } } // Now begin the metadata as a unit @@ -129,7 +174,6 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat auto index_storage_infos = info.GetIndexes().SerializeToDisk(context, options); - auto debug_verify_blocks = DBConfig::GetSetting(GetDatabase()); if (debug_verify_blocks) { for (auto &entry : index_storage_infos) { for (auto &allocator : entry.allocator_infos) { diff --git a/src/duckdb/src/storage/checkpoint_manager.cpp b/src/duckdb/src/storage/checkpoint_manager.cpp index 0996b6bf8..b6476d0b2 100644 --- a/src/duckdb/src/storage/checkpoint_manager.cpp +++ b/src/duckdb/src/storage/checkpoint_manager.cpp @@ -596,11 +596,13 @@ void CheckpointReader::ReadTableData(CatalogTransaction transaction, Deserialize auto &binary_deserializer = dynamic_cast(deserializer); auto &reader = dynamic_cast(binary_deserializer.GetStream()); - MetadataReader table_data_reader(reader.GetMetadataManager(), table_pointer); + vector read_pointers; + MetadataReader table_data_reader(reader.GetMetadataManager(), table_pointer, read_pointers); TableDataReader data_reader(table_data_reader, bound_info, table_pointer); data_reader.ReadTableData(); bound_info.data->total_rows = total_rows; + bound_info.data->read_metadata_pointers = read_pointers; } } // namespace duckdb diff --git a/src/duckdb/src/storage/table/in_memory_checkpoint.cpp b/src/duckdb/src/storage/table/in_memory_checkpoint.cpp index 31a0170f3..74367844f 100644 --- a/src/duckdb/src/storage/table/in_memory_checkpoint.cpp +++ b/src/duckdb/src/storage/table/in_memory_checkpoint.cpp @@ -86,7 +86,8 @@ InMemoryTableDataWriter::InMemoryTableDataWriter(InMemoryCheckpointer &checkpoin : TableDataWriter(table, checkpoint_manager.GetClientContext()), checkpoint_manager(checkpoint_manager) { } -void InMemoryTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) { +void InMemoryTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, + const vector &metadata_pointers, idx_t total_rows) { } void InMemoryTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info, diff --git a/src/duckdb/src/storage/table/row_group_collection.cpp b/src/duckdb/src/storage/table/row_group_collection.cpp index fce6605fc..69e15ffd4 100644 --- a/src/duckdb/src/storage/table/row_group_collection.cpp +++ b/src/duckdb/src/storage/table/row_group_collection.cpp @@ -30,12 +30,12 @@ RowGroupSegmentTree::RowGroupSegmentTree(RowGroupCollection &collection) RowGroupSegmentTree::~RowGroupSegmentTree() { } -void RowGroupSegmentTree::Initialize(PersistentTableData &data) { +void RowGroupSegmentTree::Initialize(PersistentTableData &data, optional_ptr> read_pointers) { D_ASSERT(data.row_group_count > 0); current_row_group = 0; max_row_group = data.row_group_count; finished_loading = false; - reader = make_uniq(collection.GetMetadataManager(), data.block_pointer); + reader = make_uniq(collection.GetMetadataManager(), data.block_pointer, read_pointers); root_pointer = data.block_pointer; } @@ -97,13 +97,16 @@ void RowGroupCollection::Initialize(PersistentTableData &data) { D_ASSERT(this->row_start == 0); auto l = row_groups->Lock(); this->total_rows = data.total_rows; - row_groups->Initialize(data); - stats.Initialize(types, data); metadata_pointer = data.base_table_pointer; + metadata_pointers = data.read_metadata_pointers; + row_groups->Initialize(data, metadata_pointers); + stats.Initialize(types, data); } -void RowGroupCollection::FinalizeCheckpoint(MetaBlockPointer pointer) { +void RowGroupCollection::FinalizeCheckpoint(MetaBlockPointer pointer, + const vector &existing_pointers) { metadata_pointer = pointer; + metadata_pointers = existing_pointers; } void RowGroupCollection::Initialize(PersistentCollectionData &data) { @@ -1176,7 +1179,7 @@ void RowGroupCollection::Checkpoint(TableDataWriter &writer, TableStatistics &gl row_group.CheckpointDeletes(metadata_manager); row_groups->AppendSegment(l, std::move(entry.node)); } - writer.WriteUnchangedTable(metadata_pointer, total_rows.load()); + writer.WriteUnchangedTable(metadata_pointer, metadata_pointers, total_rows.load()); return; } }