CXX-382 Backport server r2.7.8..r2.7.9 fixes to legacy branch

Author: acmorrow

SConstruct

@@ -1297,7 +1297,8 @@ def doConfigure(myenv):
 
         sanitizer_list = get_option('sanitize').split(',')
 
-        using_asan = 'address' in sanitizer_list or 'leak' in sanitizer_list
+        using_lsan = 'leak' in sanitizer_list
+        using_asan = 'address' in sanitizer_list or using_lsan
 
         # If the user asked for leak sanitizer turn on the detect_leaks
         # ASAN_OPTION. If they asked for address sanitizer as well, drop
@@ -1307,8 +1308,10 @@ def doConfigure(myenv):
         # --sanitize=address,leak:   -fsanitize=address, detect_leaks=1
         # --sanitize=address:        -fsanitize=address
         #
-        if 'leak' in sanitizer_list:
-            myenv['ENV']['ASAN_OPTIONS'] = "detect_leaks=1"
+        if using_lsan:
+            if using_asan:
+                myenv['ENV']['ASAN_OPTIONS'] = "detect_leaks=1"
+            myenv['ENV']['LSAN_OPTIONS'] = "suppressions=%s" % myenv.File("#etc/lsan.suppressions").abspath
             if 'address' in sanitizer_list:
                 sanitizer_list.remove('leak')
 
@@ -1321,12 +1324,34 @@ def doConfigure(myenv):
             print( 'Failed to enable sanitizers with flag: ' + sanitizer_option )
             Exit(1)
 
+        blackfiles_map = {
+            "address" : myenv.File("#etc/asan.blacklist"),
+            "leak" : myenv.File("#etc/asan.blacklist"),
+            "thread" : myenv.File("#etc/tsan.blacklist"),
+            "undefined" : myenv.File("#etc/ubsan.blacklist"),
+        }
+
+        blackfiles = set([v for (k, v) in blackfiles_map.iteritems() if k in sanitizer_list])
+        blacklist_options=["-fsanitize-blacklist=%s" % blackfile for blackfile in blackfiles]
+
+        for blacklist_option in blacklist_options:
+            if AddToCCFLAGSIfSupported(myenv, blacklist_option):
+                myenv.Append(LINKFLAGS=[blacklist_option])
+
         llvm_symbolizer = get_option('llvm-symbolizer')
-        if not os.path.isabs(llvm_symbolizer):
+        if os.path.isabs(llvm_symbolizer):
+            if not myenv.File(llvm_symbolizer).exists():
+                print("WARNING: Specified symbolizer '%s' not found" % llvm_symbolizer)
+                llvm_symbolizer = None
+        else:
             llvm_symbolizer = myenv.WhereIs(llvm_symbolizer)
+
         if llvm_symbolizer:
-            if using_asan:
-                myenv['ENV']['ASAN_SYMBOLIZER_PATH'] = llvm_symbolizer
+            myenv['ENV']['ASAN_SYMBOLIZER_PATH'] = llvm_symbolizer
+            myenv['ENV']['LSAN_SYMBOLIZER_PATH'] = llvm_symbolizer
+        elif using_lsan:
+            print("Using the leak sanitizer requires a valid symbolizer")
+            Exit(1)
 
     # When using msvc, check for VS 2013 Update 2+ so we can use new compiler flags
     if using_msvc():

etc/asan.blacklist

@@ -312,6 +312,7 @@ clientHeaders = [
     'mongo/util/net/message_port.h',
     'mongo/util/net/operation.h',
     'mongo/util/net/sock.h',
+    'mongo/util/shared_buffer.h',
     'mongo/util/time_support.h',
     'mongo/version.h',
 ]

etc/lsan.suppressions

@@ -83,7 +83,7 @@ error_code("RoleDataInconsistent", 80)
 error_code("NoWhereParseContext", 81)
 error_code("NoProgressMade", 82)
 error_code("RemoteResultsUnavailable", 83)
-error_code("UniqueIndexViolation", 84)
+error_code("DuplicateKeyValue", 84)
 error_code("IndexOptionsConflict", 85 )
 error_code("IndexKeySpecsConflict", 86 )
 error_code("CannotSplit", 87)
@@ -111,7 +111,7 @@ error_code("InconsistentReplicaSetNames", 108)
 error_code("ConfigurationInProgress", 109)
 error_code("CannotInitializeNodeWithData", 110)
 error_code("NotExactValueField", 111)
-error_code("DeadLock", 112)
+error_code("WriteConflict", 112)
 
 # Non-sequential error codes (for compatibility only)
 error_code("NotMaster", 10107) #this comes from assert_util.h

etc/tsan.blacklist

@@ -31,6 +31,7 @@
 #include "mongo/bson/util/builder.h"
 #include "mongo/client/export_macros.h"
 #include "mongo/platform/atomic_word.h"
+#include "mongo/util/shared_buffer.h"
 
 namespace mongo {
 
@@ -94,11 +95,16 @@ namespace mongo {
             init(bsonData);
         }
 
+        explicit BSONObj(SharedBuffer ownedBuffer)
+                : _objdata(ownedBuffer.get() ? ownedBuffer.get() : BSONObj().objdata())
+                , _ownedBuffer(ownedBuffer.moveFrom()) {
+        }
+
 #if __cplusplus >= 201103L
         /** Move construct a BSONObj */
         BSONObj(BSONObj&& other)
             : _objdata(std::move(other._objdata))
-            , _holder(std::move(other._holder)) {
+            , _ownedBuffer(std::move(other._ownedBuffer)) {
             other._objdata = BSONObj()._objdata; // To return to an empty state.
         }
 
@@ -121,7 +127,7 @@ namespace mongo {
         void swap(BSONObj& other) {
             using std::swap;
             swap(_objdata, other._objdata);
-            swap(_holder, other._holder);
+            swap(_ownedBuffer, other._ownedBuffer);
         }
 
         /**
@@ -152,7 +158,7 @@ namespace mongo {
 
            @return true if this is in owned mode
         */
-        bool isOwned() const { return _holder.get() != 0; }
+        bool isOwned() const { return _ownedBuffer.get() != 0; }
 
         /** assure the data buffer is under the control of this BSONObj and not a remote buffer
             @see isOwned()
@@ -510,59 +516,20 @@ namespace mongo {
 
         template<typename T> bool coerceVector( std::vector<T>* out ) const;
 
-        class Holder {
-        public:
-            explicit Holder(AtomicUInt32::WordType initial = AtomicUInt32::WordType())
-                : _refCount(initial) {}
-
-            // these are called automatically by boost::intrusive_ptr
-            friend void intrusive_ptr_add_ref(Holder* h) {
-                h->_refCount.fetchAndAdd(1);
-            }
-
-            friend void intrusive_ptr_release(Holder* h) {
-                if (h->_refCount.subtractAndFetch(1) == 0) {
-                    // We placement new'ed a Holder in BSONObj::takeOwnership below,
-                    // so we must destroy the object here.
-                    h->~Holder();
-                    free(h);
-                }
-            }
-
-            char* data() {
-                return reinterpret_cast<char *>(this + 1);
-            }
-
-            const char* data() const {
-                return reinterpret_cast<const char *>(this + 1);
-            }
-
-        private:
-            AtomicUInt32 _refCount;
-        };
+        typedef SharedBuffer::Holder Holder;
 
         /** Given a pointer to a region of un-owned memory containing BSON data, prefixed by
          *  sufficient space for a BSONObj::Holder object, return a BSONObj that owns the
          *  memory.
+         *
+         * This class will call free(holderPrefixedData), so it must have been allocated in a way
+         * that makes that valid.
          */
         static BSONObj takeOwnership(char* holderPrefixedData) {
-            // Initialize the refcount to 1 so we don't need to increment it in the constructor
-            // (see private BSONObj(Holder*) constructor below).
-            //
-            // TODO: Should dassert alignment of holderPrefixedData
-            // here if possible.
-            return BSONObj(new(holderPrefixedData) BSONObj::Holder(1U));
+            return BSONObj(SharedBuffer::takeOwnership(holderPrefixedData));
         }
 
     private:
-        /** Construct a new BSONObj using the given Holder object */
-        explicit BSONObj(Holder* holder)
-            : _holder(holder, false) {
-            // NOTE: The 'false' is because we have already initialized the Holder with a
-            // refcount of '1' in takeOwnership above.
-            init(holder->data());
-        }
-
         void _assertInvalid() const;
 
         void init(const char *data) {
@@ -580,7 +547,7 @@ namespace mongo {
         Status _okForStorage(bool root, bool deep) const;
 
         const char* _objdata;
-        boost::intrusive_ptr< Holder > _holder;
+        SharedBuffer _ownedBuffer;
     };
 
     MONGO_CLIENT_API std::ostream& MONGO_CLIENT_FUNC operator<<( std::ostream &s, const BSONObj &o );

etc/ubsan.blacklist

@@ -105,6 +105,20 @@ namespace mongo {
         return Status::OK();
     }
 
+    Status bsonExtractOIDFieldWithDefault(const BSONObj& object,
+                                          const StringData& fieldName,
+                                          const OID& defaultValue,
+                                          OID* out) {
+        Status status = bsonExtractOIDField(object, fieldName, out);
+        if (status == ErrorCodes::NoSuchKey) {
+            *out = defaultValue;
+        }
+        else if (!status.isOK()) {
+            return status;
+        }
+        return Status::OK();
+    }
+
     Status bsonExtractStringFieldWithDefault(const BSONObj& object,
                                              const StringData& fieldName,
                                              const StringData& defaultValue,

src/SConscript.client

@@ -159,4 +159,17 @@ namespace mongo {
                                              const StringData& defaultValue,
                                              std::string* out);
 
+    /**
+     * Finds an OID-typed element named "fieldName" in "object" and stores its value in *out.
+     *
+     * Returns Status::OK() and sets *out to the found element's OID value on success.  If no field
+     * named "fieldName" is present, *out is set to "defaultValue" and Status::OK() is returned.
+     * Returns ErrorCodes::TypeMismatch if the type of the matching element is not OID.  For return
+     * values other than Status::OK(), the resulting value of *out is undefined.
+     */
+    Status bsonExtractOIDFieldWithDefault(const BSONObj& object,
+                                          const StringData& fieldName,
+                                          const OID& defaultValue,
+                                          OID* out);
+
 }  // namespace mongo