Addition of GridFileBuilder class for on-the-fly writting of data to GridFS.

- Added GridFileBuilder
- Added unity test for GridFileBuilder
- Updated code style
- Updated with coding style changes and addition of _insertChunk private method in GridFS

Author: pakozm

src/mongo/client/gridfs.cpp

@@ -17,6 +17,7 @@
 
 #include "mongo/client/gridfs.h"
 
+#include <algorithm>
 #include <boost/smart_ptr.hpp>
 #include <fcntl.h>
 #include <fstream>
@@ -35,6 +36,7 @@
 
 namespace mongo {
 
+    using std::min;
     using std::auto_ptr;
     using std::cout;
     using std::ios;
@@ -209,6 +211,10 @@ namespace mongo {
         return _client.query( _filesNS.c_str() , o );
     }
 
+    void GridFS::_insertChunk( const GridFSChunk& chunk ) {
+        _client.insert( _chunksNS.c_str() , chunk._data );
+    }
+
     BSONObj GridFile::getMetadata() const {
         BSONElement meta_element = _obj["metadata"];
         if( meta_element.eoo() ) {
@@ -259,5 +265,86 @@ namespace mongo {
     void GridFile::_exists() const {
         uassert( 10015 ,  "doesn't exists" , exists() );
     }
-
+    
+    GridFileBuilder::GridFileBuilder( GridFS* const grid ) :
+        _grid( grid ),
+        _chunkSize( grid->getChunkSize() ),
+        _currentChunk( 0 ),
+        _pendingData( new char[_chunkSize] ),
+        _pendingDataSize( 0 ),
+        _fileLength( 0 ) {
+        _fileId.init();
+        _fileIdObj = BSON( "_id" << _fileId );
+    }
+    
+    const char* GridFileBuilder::_appendChunk( const char* data,
+                                               size_t length,
+                                               bool forcePendingInsert ) {
+        const char* const end = data + length;
+        while (data < end) {
+            size_t chunkLen = min( _chunkSize, static_cast<size_t>(end-data) );
+            // the last chunk needs to be stored as pendingData, break while if
+            // necessary
+            if ((chunkLen < _chunkSize) && (!forcePendingInsert))
+                break;
+            GridFSChunk chunk( _fileIdObj, _currentChunk, data, chunkLen );
+            _grid->_insertChunk( chunk );
+            ++_currentChunk;
+            data += chunkLen;
+            _fileLength += chunkLen;
+        }
+        return data;
+    }
+    
+    void GridFileBuilder::_appendPendingData() {
+        if (_pendingDataSize > 0) {
+            _appendChunk( _pendingData.get(), _pendingDataSize, true );
+            _pendingDataSize = 0;
+        }
+    }
+    
+    void GridFileBuilder::appendChunk( const char* data, size_t length ) {
+        if (length == 0)
+            return;
+
+        // check if there is pending data
+        if (_pendingDataSize > 0) {
+            size_t totalSize = _pendingDataSize + length;
+            size_t size = min( _chunkSize, totalSize ) - _pendingDataSize;
+            memcpy( _pendingData.get() + _pendingDataSize, data, size );
+            _pendingDataSize += size;
+            invariant( _pendingDataSize <= _chunkSize );
+            if (_pendingDataSize == _chunkSize) {
+                _appendPendingData();
+                _appendChunk( data + size, length - size, false );
+            }
+        }
+        else {
+            const char* const end = data + length;
+            // split data in _chunkSize blocks, and store as pending the last block if
+            // necessary
+            data = _appendChunk( data, length, false );
+            // process pending data if necessary
+            if (data != end) {
+                size_t size = static_cast<size_t>(end-data);
+                memcpy( _pendingData.get() + _pendingDataSize, data, size );
+                _pendingDataSize += size;
+            }
+        }
+    }
+    
+    BSONObj GridFileBuilder::buildFile( const string& remoteName,
+                                        const string& contentType ) {
+        _appendPendingData();
+        BSONObj ret = _grid->insertFile( remoteName, _fileId, _fileLength,
+                                         contentType );
+        // resets the object to allow more data append for a GridFile
+        _currentChunk = 0;
+        _pendingDataSize = 0;
+        _fileLength = 0;
+        _fileId.init();
+        _fileIdObj = BSON( "_id" << _fileId );
+        return ret;
+    }
+    
 }

src/mongo/client/gridfs.h

@@ -28,6 +28,7 @@ namespace mongo {
 
     class GridFS;
     class GridFile;
+    class GridFileBuilder;
 
     class MONGO_CLIENT_API GridFSChunk {
     public:
@@ -131,7 +132,12 @@ namespace mongo {
         // insert fileobject. All chunks must be in DB.
         BSONObj insertFile(const std::string& name, const OID& id, gridfs_offset length, const std::string& contentType);
 
+        // Insert a chunk into DB, this method is intended to be used by
+        // GridFileBuilder to incrementally insert chunks
+        void _insertChunk(const GridFSChunk& chunk);
+
         friend class GridFile;
+        friend class GridFileBuilder;
     };
 
     /**
@@ -203,4 +209,54 @@ namespace mongo {
 
         friend class GridFS;
     };
+    
+    /**
+     * class which allow to build GridFiles in a stream fashion way
+     */
+    class GridFileBuilder {
+    public:
+        /**
+         * @param grid - gridfs instance
+         */
+        GridFileBuilder( GridFS* const grid );
+        
+        /**
+         * Appends a chunk of data. Data will be split as many times as
+         * necessary in chunkSize blocks. Sizes not multiple of chunkSize will
+         * copy the reamining bytes to a pendingData pointer. In this way,
+         * it is possible to add data in a stream fashion way.
+         * @param data - C string with data
+         * @param length - size of the string
+         */
+        void appendChunk( const char* data, size_t length );
+
+        /**
+         * Inserts the description of the file in GridFS collection. Note that
+         * the stream will be reinitialized after the build call, so it will be
+         * possible to continue appending data to build another file.
+         * @param remoteName filename to use for file stored in GridFS
+         * @param contentType optional MIME type for this object.
+         *                    (default is to omit)
+         * @return the file object
+         */
+        mongo::BSONObj buildFile( const std::string& remoteName,
+                                  const std::string& contentType="" );
+        
+    private:
+        GridFS* const _grid;
+        const size_t _chunkSize; // taken from GridFS in the constructor
+        unsigned int _currentChunk;
+        OID _fileId;
+        BSONObj _fileIdObj;
+        boost::scoped_array<char> _pendingData; // pointer with _chunkSize space
+        size_t _pendingDataSize;
+        gridfs_offset _fileLength;
+
+        const char* _appendChunk( const char* data, size_t length,
+                                  bool forcePendingInsert );
+
+        void _appendPendingData();
+    };
+
+
 }

src/mongo/unittest/gridfs_test.cpp

@@ -13,6 +13,7 @@
  *    limitations under the License.
  */
 
+#include <algorithm>
 #include <fstream>
 #include <sstream>
 #include <string>
@@ -24,6 +25,7 @@ using boost::scoped_ptr;
 using std::auto_ptr;
 using std::ios;
 using std::ifstream;
+using std::min;
 using std::string;
 using std::stringstream;
 
@@ -223,4 +225,24 @@ namespace {
         );
         ASSERT_EQUALS(files_bad->itcount(), 0);
     }
+
+    TEST_F(GridFSTest, GridFileBuilder) {
+        GridFileBuilder gfb(_gfs.get());
+        for (int i=0; i<DATA_LEN; i+=2)
+            gfb.appendChunk(DATA + i, min(2, DATA_LEN - i));
+        gfb.buildFile(DATA_NAME);
+        GridFile gf = _gfs->findFile(DATA_NAME);
+        ASSERT_EQUALS(gf.getNumChunks(), 1);
+    }
+
+    TEST_F(GridFSTest, GridFileBuilderMultipleChunks) {
+        _gfs->setChunkSize(1);
+        GridFileBuilder gfb(_gfs.get());
+        for (int i=0; i<DATA_LEN; i+=2)
+            gfb.appendChunk(DATA + i, min(2, DATA_LEN - i));
+        gfb.buildFile(DATA_NAME);
+        GridFile gf = _gfs->findFile(DATA_NAME);
+        ASSERT_EQUALS(gf.getNumChunks(), DATA_LEN);
+    }
+
 } // namespace