Enhanced lob.getData() method to accept offset and amount arguments (Issue #1643)

Author: sharadraju

doc/src/release_notes.rst

@@ -13,6 +13,9 @@ node-oracledb `v6.4.0 <https://github.com/oracle/node-oracledb/compare/v6.3.0...
 Common Changes
 ++++++++++++++
 
+#)  Enhanced :meth:`~lob.getData` method to accept offset and amount arguments.
+    See `Issue #1643 <https://github.com/oracle/node-oracledb/issues/1643>`__.
+
 #)  Add support for fetching BLOB columns which have "IS JSON FORMAT OSON"
     constraint enabled in the same way as columns of type JSON.
     In node-oracledb :ref:`Thick mode <enablingthick>` this requires

lib/lob.js

@@ -185,12 +185,28 @@ class Lob extends Duplex {
   //---------------------------------------------------------------------------
   // getData()
   //
-  // Returns all of the data in the LOB as a single string or buffer.
+  // Return a portion (or all) of the data in the LOB. Note that the amount
+  // and offset are in bytes for BLOB and BFILE type LOBs and in UCS - 2 code
+  // points for CLOB and NCLOB type LOBs.UCS-2 code points are equivalent
+  // to characters for all but supplemental characters.If supplemental
+  // characters are in the LOB, the offset and amount will have to be chosen
+  // carefully to avoid splitting a character.
+  // Returns data in the LOB as a single string or buffer.
   //---------------------------------------------------------------------------
-  async getData() {
-    errors.assertArgCount(arguments, 0, 0);
+  async getData(offset, amount) {
+    errors.assertArgCount(arguments, 0, 2);
+    if (offset === undefined) {
+      offset = 1;
+    } else {
+      errors.assertParamValue(Number.isInteger(offset) && offset > 0, 1);
+    }
+    if (amount === undefined) {
+      amount = 0;
+    } else {
+      errors.assertParamValue(Number.isInteger(amount) && amount > 0, 2);
+    }
     errors.assert(this._impl, errors.ERR_INVALID_LOB);
-    return await this._impl.getData();
+    return await this._impl.getData(offset, amount);
   }
 
   //---------------------------------------------------------------------------

lib/thin/lob.js

@@ -89,11 +89,11 @@ class ThinLobImpl extends LobImpl {
     return this.dbType;
   }
 
-  async getData() {
-    if (this._length < 0) {
-      errors.throwErr(errors.ERR_INVALID_LOB);
+  async getData(offset = 1, len = this._length) {
+    if (!len) {
+      len = this._length;
     }
-    return await this.read(1, this._length);
+    return await this.read(offset, len);
   }
 
   async read(offset, length) {

src/njsLob.c

@@ -188,13 +188,20 @@ NJS_NAPI_METHOD_IMPL_SYNC(njsLob_getType, 0, NULL)
 
 //-----------------------------------------------------------------------------
 // njsLob_getData()
-//   Read all of the data from the LOB and return it as a single string or
+//   Read data from the LOB and return it as a single string or
 // buffer.
 //
-// PARAMETERS - NONE
+// PARAMETERS
+//   - lobOffset
+//   - lobAmount
+//
 //-----------------------------------------------------------------------------
-NJS_NAPI_METHOD_IMPL_ASYNC(njsLob_getData, 0, NULL)
+NJS_NAPI_METHOD_IMPL_ASYNC(njsLob_getData, 2, NULL)
 {
+    NJS_CHECK_NAPI(env, napi_get_value_uint32(env, args[0],
+            &baton->lobOffset))
+    NJS_CHECK_NAPI(env, napi_get_value_uint32(env, args[1],
+            &baton->lobAmount))
     return njsBaton_queueWork(baton, env, "GetData", njsLob_getDataAsync,
             njsLob_getDataPostAsync, returnValue);
 }
@@ -208,18 +215,31 @@ static bool njsLob_getDataAsync(njsBaton *baton)
 {
     njsLob *lob = (njsLob*) baton->callingInstance;
     bool ok = true;
+    uint32_t len;
 
     // if the length is marked dirty, acquire it at this time
     if (lob->dirtyLength) {
         if (dpiLob_getSize(lob->handle, &lob->length) < 0)
             return njsBaton_setErrorDPI(baton);
         lob->dirtyLength = false;
     }
+    len = lob->length;
+    if ((baton->lobAmount == 0) || (baton->lobAmount >= lob->length)) {
+        // If user has not given lobAmount or user gave greater than
+        // lob length, adjust the len value.
+        if (lob->length >= baton->lobOffset) {
+            len = lob->length - (baton->lobOffset - 1);
+        } else {
+            len = 1;
+        }
+    } else {
+        len = baton->lobAmount;
+    }
 
     // determine size of buffer that is required
     if (lob->dataType == NJS_DATATYPE_BLOB) {
-        baton->bufferSize = lob->length;
-    } else if (dpiLob_getBufferSize(lob->handle, lob->length,
+        baton->bufferSize = len;
+    } else if (dpiLob_getBufferSize(lob->handle, len,
             &baton->bufferSize) < 0) {
         ok = njsBaton_setErrorDPI(baton);
     }
@@ -232,8 +252,8 @@ static bool njsLob_getDataAsync(njsBaton *baton)
     }
 
     // read from the LOB into the provided buffer
-    if (ok && baton->bufferSize > 0 && dpiLob_readBytes(lob->handle, 1,
-            lob->length, baton->bufferPtr, &baton->bufferSize) < 0)
+    if (ok && baton->bufferSize > 0 && dpiLob_readBytes(lob->handle,
+            baton->lobOffset, len, baton->bufferPtr, &baton->bufferSize) < 0)
         ok = njsBaton_setErrorDPI(baton);
 
     return ok;

test/dataTypeBlob.js

@@ -42,6 +42,7 @@ const assert   = require('assert');
 const dbConfig = require('./dbconfig.js');
 const assist   = require('./dataTypeAssist.js');
 const testsUtil = require(`./testsUtil.js`);
+const random   = require('./random.js');
 
 const inFileName = 'test/fuzzydinosaur.jpg';  // contains the image to be inserted
 const outFileName = 'test/blobstreamout.jpg';
@@ -146,6 +147,60 @@ describe('41. dataTypeBlob.js', function() {
       const blob = await lob.getData();
       assert.deepStrictEqual(data, blob);
     }); // 41.1.2
+
+    it('41.1.3 BLOB getData(offset, len)', async function() {
+      const size = 32768;
+      const specialStr = "41.1.3";
+      const bigStr = random.getRandomString(size, specialStr);
+      const bufferStr = Buffer.from(bigStr, "utf-8");
+
+      let result = await connection.execute(
+        `INSERT INTO nodb_myblobs (num, content) VALUES (:1, :2) `,
+        [4, bufferStr]);
+
+      assert.strictEqual(result.rowsAffected, 1);
+      result = await connection.execute(
+        "SELECT content FROM nodb_myblobs WHERE num = :n",
+        { n: 4 });
+
+      const lob = result.rows[0][0];
+
+      // both offset and end.
+      let offset = 5;
+      let len = 10;
+
+      // Returns data from offset -1 (lob[offset - 1])
+      let blob = await lob.getData(offset, len);
+      assert.deepStrictEqual(bufferStr.subarray(offset - 1,
+        offset + len - 1), blob);
+
+      // end not specified gives entire data starting from offset
+      offset = 5;
+      blob = await lob.getData(offset);
+      assert.deepStrictEqual(bufferStr.subarray(offset - 1), blob);
+
+      // large number of bytes starting from offset 5.
+      offset = 5;
+      len = 9999;
+      blob = await lob.getData(offset, len);
+      assert.deepStrictEqual(bufferStr.subarray(offset - 1, offset + len - 1),
+        blob);
+
+      // large length which is ignored and entire lob data starting
+      // from offset is returned.
+      offset = 5;
+      len = 99999;
+      blob = await lob.getData(offset, len);
+      assert.deepStrictEqual(bufferStr.subarray(offset - 1),
+        blob);
+
+      // Invalid offset, we return null.
+      offset = 99999;
+      len = 10;
+      blob = await lob.getData(offset, len);
+      assert.equal(blob, null);
+
+    }); // 41.1.3
   }); //41.1
 
   describe('41.2 stores null value correctly', function() {

test/dataTypeClob.js

@@ -43,6 +43,7 @@ const assert   = require('assert');
 const dbConfig = require('./dbconfig.js');
 const assist   = require('./dataTypeAssist.js');
 const testsUtil = require('./testsUtil.js');
+const random   = require('./random.js');
 
 const inFileName = 'test/clobexample.txt';  // the file with text to be inserted into the database
 const outFileName = 'test/clobstreamout.txt'; // output file with the stream out data
@@ -164,6 +165,89 @@ describe('40. dataTypeClob.js', function() {
 
     }); // 40.1.2
 
+    it('40.1.3 CLOB getData(offset, len)', async function() {
+      const lenStr = 32768;
+      const specialStr = "40.1.3";
+      const largeStr = random.getRandomString(lenStr, specialStr);
+      const multiByteStr = 'aüÅæÖÆåøübcd';
+
+      const binds = [
+        [3, largeStr],
+        [4, multiByteStr]
+      ];
+      let result = await connection.executeMany(
+        `INSERT INTO nodb_myclobs (num, content) ` +
+        `VALUES (:1, :2)`,
+        binds,
+        [
+          { type: oracledb.NUMBER },
+          { type: oracledb.CLOB }
+        ]);
+      assert.strictEqual(result.rowsAffected, 2);
+
+      result = await connection.execute(
+        "SELECT content FROM nodb_myclobs WHERE num = :n",
+        { n: 3 });
+
+      let lob = result.rows[0][0];
+      let clob = await lob.getData();
+      assert.strictEqual(largeStr, clob);
+
+      let offset = 5;
+      let len = 10;
+      // starting from position 4 (largeStr[4]) to 10 characters.
+      clob = await lob.getData(offset, len);
+      assert.strictEqual(largeStr.slice(offset - 1, offset + len - 1), clob);
+
+      // len not specified gives entire data starting from offset
+      offset = 5;
+      clob = await lob.getData(offset);
+      assert.strictEqual(largeStr.slice(offset - 1), clob);
+
+      // len specified as 0 gives error.
+      offset = 5;
+      len = 0;
+      await assert.rejects(
+        async () => await lob.getData(offset, len),
+        /NJS-005:/
+      );
+
+      // large number of characters starting from offset 5.
+      offset = 5;
+      len = 9999;
+      clob = await lob.getData(offset, len);
+      assert.strictEqual(largeStr.slice(offset - 1,  offset + len - 1), clob);
+
+      // len exceeding lob length is simply ignored and
+      // characters till end starting from offset is returned.
+      offset = 5;
+      len = 99999;
+      clob = await lob.getData(offset, len);
+      assert.strictEqual(largeStr.slice(offset - 1), clob);
+
+      // Invalid ofset, we get null.
+      offset = 99999;
+      len = 10;
+      clob = await lob.getData(offset, len);
+      assert.equal(clob, null);
+
+      result = await connection.execute(
+        "SELECT content FROM nodb_myclobs WHERE num = :n",
+        { n: 4 });
+
+      lob = result.rows[0][0];
+
+      clob = await lob.getData();
+      assert.strictEqual(multiByteStr, clob);
+
+      offset = 2;
+      len = 10;
+      // starting from position 1 (multiByteStr[1]) to 10 characters. "üÅæÖÆåøübc"
+      clob = await lob.getData(offset, len);
+      assert.strictEqual(multiByteStr.slice(offset - 1, offset + len - 1), clob);
+
+    }); // 40.1.3
+
   }); // 40.1
 
   describe('40.2 stores null value correctly', function() {

test/dataTypeNclob.js

@@ -83,6 +83,17 @@ describe('123. dataTypeNclob.js', function() {
 
     });
 
+    it('123.1.3 works with lob getData(offset, len)', async function() {
+      const insertLength = 100;
+
+      const insertStr = random.getRandomLengthString(insertLength);
+
+      await insertData(tableName, insertStr);
+
+      await getLobData(tableName, insertStr);
+
+    });
+
   }); // 123.1
 
   describe('123.2 insert and fetch as string with fetchInfo', function() {
@@ -236,6 +247,28 @@ describe('123. dataTypeNclob.js', function() {
     });
   };
 
+  const getLobData = async function(tableName, originalStr) {
+    let result = null;
+    result = await connection.execute("SELECT TO_CLOB(content) FROM " + tableName + " where num = " + insertID);
+    const lob = result.rows[0][0];
+
+    assert(lob);
+    lob.setEncoding('utf8'); // set the encoding so we get a 'string' not a 'buffer'
+    let data = await lob.getData();
+    assert.strictEqual(originalStr, data);
+
+    let offset = 5;
+    data = await lob.getData(offset);
+    assert.strictEqual(originalStr.slice(offset - 1), data);
+
+    // len exceeding lob length is simply ignored and
+    // characters till end starting from offset is returned.
+    offset = 5;
+    const len = 99999;
+    data = await lob.getData(offset, len);
+    assert.strictEqual(originalStr.slice(offset - 1), data);
+  };
+
   const fetchLob_fetchInfo = async function(tableName, originalStr) {
     let result = null;
     result = await connection.execute(

test/list.txt

@@ -645,13 +645,15 @@ Overview of node-oracledb functional tests
     40.1 testing CLOB data type
         40.1.1 stores CLOB value correctly
         40.1.2 CLOB getData()
+        40.1.3 CLOB getData(offset, len)
     40.2 stores null value correctly
         40.2.1 testing Null, Empty string and Undefined
 
 41. dataTypeBlob.js
     41.1 testing BLOB data type
         41.1.1 stores BLOB value correctly
         41.1.2 BLOB getData()
+        41.1.3 BLOB getData(offset, len)
     41.2 stores null value correctly
         41.2.1 testing Null, Empty string and Undefined
     41.3 OSON column metadata
@@ -3409,6 +3411,7 @@ oracledb.OUT_FORMAT_OBJECT and resultSet =  true
     123.1 insert and stream out
         123.1.1 works with data size 100
         123.1.2 works with data size 3000
+        123.1.3 works with lob getData (offset, len)
     123.2 insert and fetch as string with fetchInfo
         123.2.1 works with data size 100
         123.2.2 works with data size 3000