diff --git a/src/common/classes/array.h b/src/common/classes/array.h index 897474fbb27..c6ac9a268fe 100644 --- a/src/common/classes/array.h +++ b/src/common/classes/array.h @@ -169,7 +169,6 @@ class Array : public Storage count = 0; } -protected: const T& getElement(size_type index) const noexcept { fb_assert(index < count); @@ -182,6 +181,7 @@ class Array : public Storage return data[index]; } +protected: void freeData() noexcept { // CVC: Warning, after this call, "data" is an invalid pointer, be sure to reassign it diff --git a/src/dsql/DsqlRequests.cpp b/src/dsql/DsqlRequests.cpp index 54e13e8064a..26a7f059efe 100644 --- a/src/dsql/DsqlRequests.cpp +++ b/src/dsql/DsqlRequests.cpp @@ -553,6 +553,8 @@ void DsqlDmlRequest::doExecute(thread_db* tdbb, jrd_tra** traHandle, firstRowFetched = false; const dsql_msg* message = dsqlStatement->getSendMsg(); + req_transaction->processUpdates(tdbb); + if (!message) { JRD_start(tdbb, request, req_transaction); diff --git a/src/jrd/CacheVector.h b/src/jrd/CacheVector.h index 87b0ca72434..24624329ebd 100644 --- a/src/jrd/CacheVector.h +++ b/src/jrd/CacheVector.h @@ -40,6 +40,7 @@ #include "../jrd/constants.h" #include "../jrd/tra_proto.h" #include "../jrd/QualifiedName.h" +#include "../jrd/obj.h" namespace Jrd { @@ -87,6 +88,13 @@ class ElementBase virtual ~ElementBase(); virtual void cleanup(thread_db* tdbb) = 0; + virtual ObjectType getObjectType() = 0; + virtual void getObjectName(QualifiedName& name) = 0; + virtual void newVersion(thread_db* tdbb) = 0; + virtual bool ensureVersioned(thread_db* tdbb, ObjectBase::Flag fl) = 0; + virtual void makeRequests(thread_db* tdbb) = 0; + virtual void commit(thread_db* tdbb, TraNumber curNumber = 0) = 0; + virtual MdcVersion getVersion(thread_db* tdbb) = 0; public: [[noreturn]] void busyError(thread_db* tdbb, MetaId id, const char* family); @@ -99,6 +107,9 @@ class ElementBase return locked; } + // fill dependencies info + void fillDeps(thread_db* tdbb, bool forceRecompile); + private: Lock* lock = nullptr; std::atomic locked = false; @@ -163,11 +174,17 @@ class ListEntry : public HazardObject public: enum State { INITIAL, RELOAD, MISSING, SCANNING, READY }; - ListEntry(Versioned* object, TraNumber traNumber, ObjectBase::Flag fl, ListEntry* link = nullptr) + ListEntry(thread_db* tdbb, Versioned* object, TraNumber traNumber, ObjectBase::Flag fl, ListEntry* link = nullptr) : object(object), traNumber(traNumber), cacheFlags(fl), state(INITIAL) { if (fl & CacheFlag::ERASED) fb_assert(!object); + + // Handle front & back versions of MDC + VersionIncr incr(tdbb); + version = incr.getVersion(); + + // Add to linked list if (link) next.store(link); } @@ -550,19 +567,9 @@ class ListEntry : public HazardObject return state == READY ? false : (thd == Thread::getCurrentThreadId()) && (state == SCANNING); } - static bool upgradable(HazardPtr& listEntry, const Versioned* from) + MdcVersion getVersion() { - for (; listEntry; listEntry.set(listEntry->next)) - { - if (listEntry->object == from) - return false; // not found upgrade version - - if (listEntry->getFlags() & CacheFlag::COMMITTED) - return true; // already upgraded by someone else - } - - fb_assert(false); - return false; // miss from what to upgrade + return version; } private: @@ -582,7 +589,6 @@ class ListEntry : public HazardObject TraNumber traNumber; // when COMMITTED not set - stores transaction that created this list element // when COMMITTED is set - stores transaction after which older elements are not needed // traNumber to be changed BEFORE setting COMMITTED - MdcVersion version = 0; // version of metadata cache when object was added ThreadId thd = 0; // thread that performs object scan() std::atomic cacheFlags; @@ -677,6 +683,11 @@ class CacheElement : public ElementBase, public P return getVersioned(tdbb, TransactionNumber::current(tdbb), fl); } + bool ensureVersioned(thread_db* tdbb, ObjectBase::Flag fl) override + { + return getVersioned(tdbb, fl); + } + bool isReady(thread_db* tdbb) { auto entry = getEntry(tdbb, TransactionNumber::current(tdbb), CacheFlag::NOSCAN | CacheFlag::NOCOMMIT); @@ -748,7 +759,7 @@ class CacheElement : public ElementBase, public P ListEntry* newEntry = nullptr; try { - newEntry = FB_NEW ListEntry(obj, traNum, fl & ~CacheFlag::ERASED); + newEntry = FB_NEW ListEntry(tdbb, obj, traNum, fl & ~CacheFlag::ERASED); } catch (const Firebird::Exception&) { @@ -776,6 +787,7 @@ class CacheElement : public ElementBase, public P return listEntry; // nullptr } + toUpdatedList(tdbb, fl); return HazardPtr>(newEntry); } @@ -784,7 +796,9 @@ class CacheElement : public ElementBase, public P fb_assert(list.load()); listEntry = list; } + fl &= ~CacheFlag::AUTOCREATE; + toUpdatedList(tdbb, fl); return ListEntry::getEntry(tdbb, listEntry, traNum, fl, this); } @@ -812,7 +826,7 @@ class CacheElement : public ElementBase, public P if (!cur) cur = TransactionNumber::current(tdbb); - ListEntry* newEntry = FB_NEW ListEntry(obj, cur, fl); + ListEntry* newEntry = FB_NEW ListEntry(tdbb, obj, cur, fl); if (!ListEntry::add(tdbb, list, newEntry)) { newEntry->cleanup(tdbb, false); @@ -867,15 +881,15 @@ class CacheElement : public ElementBase, public P return nullptr; } - void commit(thread_db* tdbb, TraNumber cur = 0) + void commit(thread_db* tdbb, TraNumber curNumber = 0) override { HazardPtr> current(list); if (current) { - if (!cur) - cur = TransactionNumber::current(tdbb); + if (!curNumber) + curNumber = TransactionNumber::current(tdbb); - auto flags = current->commit(tdbb, cur, TransactionNumber::next(tdbb)); + auto flags = current->commit(tdbb, curNumber, TransactionNumber::next(tdbb)); if (flags & CacheFlag::NOCOMMIT) // Committed newly created version in cache pingLock(tdbb, flags, this->getId(), Versioned::objectFamily(this)); @@ -938,12 +952,17 @@ class CacheElement : public ElementBase, public P return listEntry->scanInProgress(); } - static int getObjectType() + ObjectType getObjectType() override { return Versioned::objectType(); } - void newVersion(thread_db* tdbb) + void getObjectName(QualifiedName& name) override + { + name = this->getName(); + } + + void newVersion(thread_db* tdbb) override { TraNumber traNum; @@ -967,41 +986,25 @@ class CacheElement : public ElementBase, public P } } - bool upgrade(thread_db* tdbb, const Versioned* from) + bool nameIs(const QualifiedName& name) { - HazardPtr> l(list); - - // list of versions should be present - fb_assert(l); - if (!l) - return false; - - // if there is another version at the top nothing to be added - if (l->getVersioned() != from) - return ListEntry::upgradable(l, from); - - // we have candidate for upgrade - make sure it's not half-done - fb_assert(l->getFlags() & CacheFlag::COMMITTED); - if (!(l->getFlags() & CacheFlag::COMMITTED)) - return false; - - // Try to upgrade - ListEntry* newEntry = FB_NEW ListEntry(nullptr, TransactionNumber::current(tdbb), - CacheFlag::COMMITTED | CacheFlag::MINISCAN | CacheFlag::DB_VERSION, l.getPointer()); - if (l.replace(list, newEntry)) - return true; + return this->getName() == name; + } - // undo changes - delete newEntry; + // This is needed to check correctness of statements present in current object's version + void makeRequests(thread_db* tdbb) override + { + Versioned* v = getVersioned(tdbb, CacheFlag::AUTOCREATE); + if (!v) + return; - // Someone already added entry - see is it OK for us - l.set(list); - return ListEntry::upgradable(l, from); + v->makeRequests(tdbb); } - bool nameIs(const QualifiedName& name) + MdcVersion getVersion(thread_db* tdbb) override { - return this->getName() == name; + auto entry = getEntry(tdbb, TransactionNumber::current(tdbb), CacheFlag::NOSCAN | CacheFlag::NOCOMMIT); + return entry ? entry->getVersion() : 0; } private: @@ -1011,6 +1014,13 @@ class CacheElement : public ElementBase, public P atomics::memory_order_release, atomics::memory_order_relaxed); } + // Check flags and may be fill dependencies info + void toUpdatedList(thread_db* tdbb, ObjectBase::Flag fl) + { + if (fl & CacheFlag::DEPENDS) + ElementBase::fillDeps(tdbb, true); + } + private: std::atomic*> list = nullptr; std::atomic resetAt = 0; @@ -1211,28 +1221,6 @@ class CacheVector : public Firebird::PermanentStorage return data; } - bool upgrade(thread_db* tdbb, MetaId id, const Versioned* from) - { - fb_assert(id < getCount()); - - if (id < getCount()) - { - auto ptr = getDataPointer(id); - fb_assert(ptr); - - if (ptr) - { - StoredElement* data = ptr->load(atomics::memory_order_acquire); - fb_assert(data); - - if (data) - return data->upgrade(tdbb, from); - } - } - - return false; - } - bool lookup(thread_db* tdbb, const QualifiedName& name, ObjectBase::Flag fl, StoredElement** element, Versioned** versioned) { diff --git a/src/jrd/CharSetContainer.h b/src/jrd/CharSetContainer.h index df397fdb249..d25440cd251 100644 --- a/src/jrd/CharSetContainer.h +++ b/src/jrd/CharSetContainer.h @@ -134,6 +134,8 @@ class CharSetVers final : public ObjectBase static ObjectType objectType() noexcept; + void makeRequests(thread_db* tdbb) { } + bool hash(thread_db*, Firebird::sha512&) { return true; diff --git a/src/jrd/Function.h b/src/jrd/Function.h index 1f14c5c8a8c..d842ba5dc16 100644 --- a/src/jrd/Function.h +++ b/src/jrd/Function.h @@ -71,7 +71,7 @@ namespace Jrd } public: - int getObjectType() const noexcept override + ObjectType getObjectType() const noexcept override { return objectType(); } diff --git a/src/jrd/Package.h b/src/jrd/Package.h index 6d54bda85c4..036f614f83b 100644 --- a/src/jrd/Package.h +++ b/src/jrd/Package.h @@ -214,7 +214,7 @@ class Package final : public Firebird::PermanentStorage, public ObjectBase return getPermanent()->id; } - int getObjectType() const noexcept + ObjectType getObjectType() const noexcept { return objectType(); } @@ -226,6 +226,8 @@ class Package final : public Firebird::PermanentStorage, public ObjectBase static ObjectType objectType() noexcept; + void makeRequests(thread_db* tdbb) {/*!!!!!!!!!!!!!!!*/} + bool hash(thread_db* tdbb, Firebird::sha512& digest); Cached::Package* getPermanent() const noexcept diff --git a/src/jrd/Relation.h b/src/jrd/Relation.h index 28f66c8a73b..aed4f7d8a08 100644 --- a/src/jrd/Relation.h +++ b/src/jrd/Relation.h @@ -254,6 +254,8 @@ class DbTriggers final : public Triggers, public ObjectBase static ObjectType objectType() noexcept; + void makeRequests(thread_db* tdbb) {/*!!!!!!!!!!!!!!!*/} + private: DbTriggersHeader* perm; @@ -565,6 +567,8 @@ class IndexVersion final : public ObjectBase static std::optional getIdByName(thread_db* tdbb, ExName name); static ObjectType objectType() noexcept; + void makeRequests(thread_db* tdbb) {/*!!!!!!!!!!!!!!!*/} + ScanResult reload(thread_db* tdbb, ObjectBase::Flag flags) { return scan(tdbb, flags); @@ -693,6 +697,8 @@ class jrd_rel final : public ObjectBase static const char* objectFamily(RelationPermanent* perm); static ObjectType objectType() noexcept; + void makeRequests(thread_db* tdbb) {/*!!!!!!!!!!!!!!!*/} + void releaseTriggers(thread_db* tdbb, bool destroy); const Trigger* findTrigger(const QualifiedName& trig_name) const; const Format* currentFormat(thread_db* tdbb); diff --git a/src/jrd/Routine.cpp b/src/jrd/Routine.cpp index e32096996e9..2b2e6c12403 100644 --- a/src/jrd/Routine.cpp +++ b/src/jrd/Routine.cpp @@ -233,6 +233,13 @@ void Routine::parseMessages(thread_db* tdbb, CompilerScratch* csb, BlrReader blr } } +void Routine::makeRequests(thread_db* tdbb) +{ + auto *req = statement->findRequest(tdbb); + if (req) + req->setUnused(); +} + bool Routine::hash(thread_db* tdbb, Firebird::sha512& digest) { if (inputFields.hasData()) diff --git a/src/jrd/Routine.h b/src/jrd/Routine.h index acbdf75466d..2f23608c47a 100644 --- a/src/jrd/Routine.h +++ b/src/jrd/Routine.h @@ -169,9 +169,11 @@ namespace Jrd void sharedCheckUnlock(thread_db* tdbb); + void makeRequests(thread_db* tdbb); + public: virtual RoutinePermanent* getPermanent() const noexcept = 0; // Permanent part of data - virtual int getObjectType() const noexcept = 0; + virtual ObjectType getObjectType() const noexcept = 0; virtual SLONG getSclType() const noexcept = 0; private: diff --git a/src/jrd/dfw.epp b/src/jrd/dfw.epp index 2c289fee6b2..c9aad867061 100644 --- a/src/jrd/dfw.epp +++ b/src/jrd/dfw.epp @@ -421,6 +421,7 @@ static bool clear_cache(thread_db*, SSHORT, DeferredWork*, jrd_tra*); static bool change_repl_state(thread_db*, SSHORT, DeferredWork*, jrd_tra*); static bool set_statistics(thread_db*, SSHORT, DeferredWork*, jrd_tra*); static bool deps_to_disk(thread_db*, SSHORT, DeferredWork*, jrd_tra*); +static bool update_dependencies(thread_db*, SSHORT, DeferredWork*, jrd_tra*); // ---------------------------------------------------------------- @@ -1228,6 +1229,7 @@ static inline constexpr deferred_task task_table[] = { dfw_modify_package_constant, createOrAlterConstant }, { dfw_delete_package_constant, deleteConstant }, { dfw_create_package, createPackage }, + { dfw_update_dependencies, update_dependencies }, // must be last { dfw_null, NULL } }; @@ -3473,6 +3475,46 @@ static bool delete_collation(thread_db* tdbb, SSHORT phase, DeferredWork* work, } +static bool update_dependencies(thread_db* tdbb, SSHORT phase, DeferredWork* work, jrd_tra* transaction) +{ +/******************************************* + * + * u p d a t e _ d e p e n d e n c i e s + * + ******************************************* + * + * Functional description + * Update dependent from changed in this transaction objects + * and commit them at specific DFW phase. + * + **************************************/ + SET_TDBB(tdbb); + + switch (phase) + { + case 0: + return false; + + case 1: + case 2: + case 3: + case 4: + case 5: + return true; + + case 6: + transaction->processUpdates(tdbb); + return true; + + case 7: + transaction->processCommits(tdbb); + break; + } + + return false; +} + + static bool delete_parameter(thread_db* tdbb, SSHORT phase, DeferredWork*, jrd_tra*) { /************************************** @@ -3530,12 +3572,7 @@ static bool create_index(thread_db* tdbb, SSHORT phase, DeferredWork* work, jrd_ * Create a new index or change the state of an index between active/inactive. * **************************************/ - jrd_rel* relation = nullptr; - SET_TDBB(tdbb); - Jrd::Attachment* attachment = tdbb->getAttachment(); - Database* dbb = tdbb->getDatabase(); - AutoRequest request; switch (phase) { diff --git a/src/jrd/met.epp b/src/jrd/met.epp index f0c1f1fcfe5..483b306303b 100644 --- a/src/jrd/met.epp +++ b/src/jrd/met.epp @@ -5656,3 +5656,52 @@ jrd_rel* MetadataCache::getLtt(thread_db* tdbb, const QualifiedName& name) return nullptr; } + + +void ElementBase::fillDeps(thread_db* tdbb, bool forceRecompile) +{ + auto* attachment = tdbb->getAttachment(); + auto* metaTransaction = attachment->getMetaTransaction(tdbb); + auto* transaction = tdbb->getTransaction(); + + ObjectType objType = getObjectType(); + QualifiedName name; + getObjectName(name); + + MetaName object = name.object; + if (name.package.hasData()) + { + objType = obj_package_header; + object = name.package; + } + + AUTO_HANDLE(hndl); + + FOR (REQUEST_HANDLE hndl TRANSACTION_HANDLE metaTransaction) + D IN RDB$DEPENDENCIES + WITH D.RDB$DEPENDED_ON_TYPE EQ objType + AND D.RDB$DEPENDED_ON_SCHEMA_NAME EQ name.schema.c_str() + AND D.RDB$DEPENDED_ON_NAME EQ object.c_str() + REDUCED TO D.RDB$DEPENDENT_TYPE, D.RDB$DEPENDENT_SCHEMA_NAME, D.RDB$PACKAGE_NAME, D.RDB$DEPENDENT_NAME + { + switch (D.RDB$DEPENDENT_TYPE) + { + case obj_procedure: + { + QualifiedName name; + name.schema = D.RDB$DEPENDENT_SCHEMA_NAME; + if (!D.RDB$PACKAGE_NAME.NULL) + name.package = D.RDB$PACKAGE_NAME; + name.object = D.RDB$DEPENDENT_NAME; + + transaction->storeUpdate(MetadataCache::getPerm(tdbb, name, CacheFlag::AUTOCREATE), forceRecompile); + } + break; + + default: + break; + } + } + END_FOR +} + diff --git a/src/jrd/met.h b/src/jrd/met.h index cbbb1b6bb90..2a15d322ade 100644 --- a/src/jrd/met.h +++ b/src/jrd/met.h @@ -141,7 +141,7 @@ class jrd_prc : public Routine } public: - int getObjectType() const noexcept override + ObjectType getObjectType() const noexcept override { return obj_procedure; } diff --git a/src/jrd/tra.cpp b/src/jrd/tra.cpp index a2576184ed2..8e7c02628f4 100644 --- a/src/jrd/tra.cpp +++ b/src/jrd/tra.cpp @@ -4453,3 +4453,117 @@ void jrd_tra::eraseSecDbContext() noexcept delete tra_sec_db_context; tra_sec_db_context = NULL; } + +void jrd_tra::storeUpdate(ElementBase* obj, bool forceRecompile) +{ + if (!accumulatedDeps) + { + accumulatedDeps = FB_NEW_POOL(getPool()) Deps(getPool()); + DFW_post_work(this, dfw_update_dependencies, nullptr, nullptr, 0u); + } + + bool* recompile = accumulatedDeps->get(obj); + if (recompile) + *recompile = forceRecompile || *recompile; + else + { + recompile = accumulatedDeps->put(obj); + *recompile = forceRecompile; + } +} + +void jrd_tra::storeCommit(ElementBase* obj) +{ + if (!updateCommits) + updateCommits = FB_NEW_POOL(getPool()) UpdateCommits(getPool()); + + updateCommits->push(obj); +} + +bool jrd_tra::processUpdates(thread_db* tdbb) +{ + if (!(accumulatedDeps || processingDeps)) + return false; + + MdcVersion startingVersion; + { + VersionIncr incr(tdbb); + startingVersion = incr.getVersion(); + } + + while (accumulatedDeps || processingDeps) + { + // switch deps + if (!processingDeps) + processingDeps = accumulatedDeps.release(); + + // process updates + for(auto iter : *processingDeps) + { + auto* elem = iter.first; + bool use = iter.second; + + // may be this element was already processed in this update? + if (elem->getVersion(tdbb) >= startingVersion) + continue; + + if (!use) + { + // now ask cache element to create all possible requests + try + { + elem->makeRequests(tdbb); + } + + // handle specific for outdated element error + catch (const status_exception& ex) + { + if (ex.value()[1] != isc_old_format) + throw; + tdbb->tdbb_status_vector->init(); + use = true; + } + } + + if (use) + { + // get all existing currently dependencies + elem->fillDeps(tdbb, false); + + // make new version and compile it + elem->newVersion(tdbb); + auto rc = elem->ensureVersioned(tdbb, 0); + fb_assert(rc); + + storeCommit(elem); + } + } + + // This portion of update is ready + delete processingDeps.release(); + } + + return true; +} + +void jrd_tra::processCommits(thread_db* tdbb) +{ + if (!updateCommits) + return; + + unsigned pos = 0; + try + { + for (pos = 0; pos < updateCommits->getCount(); pos++) + updateCommits->getElement(pos)->commit(tdbb); + } + catch(const Exception&) + { + // We do not expect exceptions in metacache commits - but + // already committed elements should better go away + if (pos > 0) + updateCommits->removeRange(0, pos); + + throw; + } +} diff --git a/src/jrd/tra.h b/src/jrd/tra.h index 0e06f7c3c07..02c3c981813 100644 --- a/src/jrd/tra.h +++ b/src/jrd/tra.h @@ -35,6 +35,8 @@ #include "../include/fb_blk.h" #include "../common/classes/tree.h" #include "../common/classes/GenericMap.h" +#include "../common/classes/auto.h" +#include "../common/classes/array.h" #include "../jrd/exe.h" #include "../jrd/rpb_chain.h" #include "../jrd/blb.h" // For bid structure @@ -430,6 +432,26 @@ class jrd_tra final : public pool_alloc // Finish and delete BulkInsert that belongs to the request void finiBulkInsert(thread_db* tdbb, Request* request); + + // Store an object to be updated + void storeUpdate(ElementBase* obj, bool forceRecompile); + + // Store updated object to be committed + void storeCommit(ElementBase* obj); + + // Process updates/commits accumulated by transaction + bool processUpdates(thread_db* tdbb); + void processCommits(thread_db* tdbb); + +private: + // Under processing and accumulated sets of dependencies + typedef Firebird::GenericMap> Deps; + Firebird::AutoPtr processingDeps; + Firebird::AutoPtr accumulatedDeps; + + // Set of updated objects to be committed + typedef Firebird::HalfStaticArray UpdateCommits; + Firebird::AutoPtr updateCommits; }; // System transaction is always transaction 0. @@ -574,7 +596,10 @@ enum dfw_t : int { dfw_delete_package_constant, // Package - dfw_create_package + dfw_create_package, + + // Update various objects dependent from modified in this transaction + dfw_update_dependencies }; } //namespace Jrd