Merge pull request #142 from JingyuanZhang/master

JingyuanZhang · web-flow · commit bde871f1064d · 2021-04-24T23:05:49.000+08:00
feat(core): del useless filecount param instead get chunknum from model
diff --git a/e2e/dist/models/custom_model/model.json b/e2e/dist/models/custom_model/model.json
@@ -1,4 +1,5 @@
 {
+    "chunkNum": 0,
     "ops": [
         {
             "attrs": {
diff --git a/packages/paddlejs-backend-cpu/test/model/modelTest.js b/packages/paddlejs-backend-cpu/test/model/modelTest.js
@@ -4,7 +4,6 @@ import '../../src/index';
 async function run() {
     const runner = new Runner({
         modelPath: 'https://paddlejs.cdn.bcebos.com/models/mobileNetV2/model.json',
-        fileCount: 4,
         feedShape: {
             fw: 224,
             fh: 224
diff --git a/packages/paddlejs-backend-nodegl/test/model/modelTest.ts b/packages/paddlejs-backend-nodegl/test/model/modelTest.ts
@@ -12,8 +12,7 @@ async function run() {
         feedShape: {
             fw: 5,
             fh: 3
-        },
-        fileCount: 0
+        }
     });
     const preheatRes = await runner.init();
     console.log(preheatRes); // [ 30, 25, 40, 30, 25, 40 ]
diff --git a/packages/paddlejs-backend-nodegl/test/op/opTest.ts b/packages/paddlejs-backend-nodegl/test/op/opTest.ts
@@ -10,7 +10,6 @@ const modelPath = `${modelDir}/${opName}.json`;
 async function run() {
     const runner = new Runner({
         modelPath,
-        fileCount: 0,
         feedShape: {
             fw: 3,
             fh: 3
diff --git a/packages/paddlejs-backend-webgl/test/model/modelTest.js b/packages/paddlejs-backend-webgl/test/model/modelTest.js
@@ -10,8 +10,7 @@ async function run() {
         feedShape: {
             fw: 5,
             fh: 3
-        },
-        fileCount: 0
+        }
     });
 
     env.set('debug', true);
diff --git a/packages/paddlejs-backend-webgl/test/op/opTest.js b/packages/paddlejs-backend-webgl/test/op/opTest.js
@@ -9,7 +9,6 @@ const modelPath = `${modelDir}${opName}.json`;
 async function run() {
     const runner = new Runner({
         modelPath,
-        fileCount: 0,
         feedShape: {
             fw: 3,
             fh: 3
diff --git a/packages/paddlejs-backend-webgpu/test/model/modelTest.ts b/packages/paddlejs-backend-webgpu/test/model/modelTest.ts
@@ -10,8 +10,7 @@ async function run() {
         feedShape: {
             fw: 5,
             fh: 3
-        },
-        fileCount: 0
+        }
     });
     const preheatRes = await runner.init();
     console.log(preheatRes);
diff --git a/packages/paddlejs-backend-webgpu/test/op/opTest.ts b/packages/paddlejs-backend-webgpu/test/op/opTest.ts
@@ -47,7 +47,6 @@ async function run() {
             fw: 3,
             fh: 3
         },
-        fileCount: 0,
         needPreheat: false
     });
     await runner.init();
diff --git a/packages/paddlejs-benchmark/src/config.vue b/packages/paddlejs-benchmark/src/config.vue
@@ -58,7 +58,6 @@ export default Vue.extend({
             modelList:  [
                 {
                     name: 'mobileNetV2',
-                    fileCount: 4,
                     feedShape: {
                         fw: 224,
                         fh: 224
@@ -70,7 +69,6 @@ export default Vue.extend({
                 },
                 {
                     name: 'mobileNetV2Opt',
-                    fileCount: 4,
                     feedShape: {
                         fw: 224,
                         fh: 224
@@ -82,7 +80,6 @@ export default Vue.extend({
                 },
                 {
                     name: 'wine',
-                    fileCount: 3,
                     feedShape: {
                         fw: 224,
                         fh: 224
@@ -94,7 +91,6 @@ export default Vue.extend({
                 },
                 {
                     name: 'gesture_detect',
-                    fileCount: 2,
                     feedShape: {
                         fw: 256,
                         fh: 256
@@ -106,7 +102,6 @@ export default Vue.extend({
                 },
                 {
                     name: 'gesture_rec',
-                    fileCount: 1,
                     feedShape: {
                         fw: 224,
                         fh: 224
@@ -118,7 +113,6 @@ export default Vue.extend({
                 },
                 {
                     name: 'humanseg',
-                    fileCount: 1,
                     feedShape: {
                         fw: 192,
                         fh: 192
diff --git a/packages/paddlejs-core/README.md b/packages/paddlejs-core/README.md
@@ -16,7 +16,6 @@ import '@paddlejs/paddlejs-backend-webgl';
 
 const runner = new Runner({
     modelPath: '/model/mobilenetv2', // model path, e.g. http://xx.cc/path, http://xx.cc/path/model.json, /localModelDir/model.json, /localModelDir
-    fileCount?: 4, // model data file count, default value is 1
     feedShape: { // input shape
         fw: 256,
         fh: 256
diff --git a/packages/paddlejs-core/README_cn.md b/packages/paddlejs-core/README_cn.md
@@ -15,7 +15,6 @@ import '@paddlejs/paddlejs-backend-webgl';
 
 const runner = new Runner({
     modelPath: '/model/mobilenetv2', // model path, e.g. http://xx.cc/path, http://xx.cc/path/model.json, /localModelDir/model.json, /localModelDir
-    fileCount?: 4, // model data file count, default value is 1
     feedShape: { // input shape
         fw: 256,
         fh: 256
diff --git a/packages/paddlejs-core/__tests__/spec/loader.test.ts b/packages/paddlejs-core/__tests__/spec/loader.test.ts
@@ -11,7 +11,7 @@ describe('test loader', () => {
         fetch.resetMocks();
         fetch.mockResponseOnce(JSON.stringify(modelInfo));
         const modelPath = 'https://mobileNetV2Opt/model.json';
-        const loader = new Loader(modelPath, 0);
+        const loader = new Loader(modelPath);
         const {
             ops
         } = await loader.load();
@@ -22,7 +22,7 @@ describe('test loader', () => {
         fetch.resetMocks();
         fetch.mockResponseOnce(JSON.stringify(modelInfo));
         const modelPath = '/path';
-        const loader = new Loader(modelPath, 0);
+        const loader = new Loader(modelPath);
         const {
             vars
         } = await loader.load();
diff --git a/packages/paddlejs-core/__tests__/spec/runner.test.ts b/packages/paddlejs-core/__tests__/spec/runner.test.ts
@@ -14,8 +14,7 @@ const runner = new Runner({
     feedShape: {
         fw: 5,
         fh: 3
-    },
-    fileCount: 0
+    }
 });
 
 it('test case: unregister backend', async () => {
diff --git a/packages/paddlejs-core/src/commons/interface.ts b/packages/paddlejs-core/src/commons/interface.ts
@@ -17,6 +17,7 @@ export interface ModelVar {
 }
 
 export interface Model {
+    chunkNum?: number;
     ops: ModelOp[];
     vars: ModelVar[];
     multiOutputs?: ModelVar[]
diff --git a/packages/paddlejs-core/src/loader.ts b/packages/paddlejs-core/src/loader.ts
@@ -3,6 +3,7 @@
  */
 
 import env from './env';
+import { Model, ModelVar } from './commons/interface';
 
 interface UrlConf {
     dir: string;
@@ -15,33 +16,26 @@ interface FetchParams {
     mode?: string;
 }
 
-interface ModelVars {
-    name: string;
-    shape: number[];
-    data?: number[] | Float32Array;
-    persistable?: boolean;
-}
-
 export default class ModelLoader {
     urlConf: UrlConf = {
         dir: '',
         main: ''
     };
 
-    multipart = false;
-    chunkNum = 0;
-    dataType = 'binary';
+    separateChunk: boolean = true;
+    chunkNum: number = 1;
+    dataType: string = 'binary';
     params: FetchParams = {
         type: 'fetch'
     };
 
-    inNode = false;
+    inNode: boolean = false;
     isLocalFile: boolean = false;
     realFetch: Function = function () {
         throw new Error('ERROR: empty fetch funciton');
     };
 
-    constructor(modelPath: string, fileCount: number = 1) {
+    constructor(modelPath: string) {
         let modelDir = '';
         let filename = 'model.json';
         if (modelPath.endsWith('.json')) {
@@ -64,15 +58,15 @@ export default class ModelLoader {
             main: filename // 主文件
         };
 
-        this.multipart = fileCount > 0;
-        this.dataType = 'binary';
-        this.chunkNum = fileCount;
         this.inNode = env.get('platform') === 'node';
     }
 
     async load() {
-        const modelInfo: any = await this.fetchModel();
-        if (this.multipart === true) {
+        const modelInfo: Model = await this.fetchModel();
+        this.separateChunk = !!modelInfo.chunkNum && modelInfo.chunkNum > 0;
+        this.chunkNum = this.separateChunk ? modelInfo.chunkNum : 0;
+
+        if (this.separateChunk) {
             if (this.dataType === 'binary') {
                 await this.fetchChunks().then(allChunksData =>
                     this.traverse(modelInfo.vars, allChunksData)
@@ -130,7 +124,7 @@ export default class ModelLoader {
         });
     }
 
-    traverse(arr: ModelVars[], allChunksData: Float32Array) {
+    traverse(arr: ModelVar[], allChunksData: Float32Array) {
         let marker = 0; // 读到哪个位置了
         let len; // 当前op长度
         arr.filter(item => {
diff --git a/packages/paddlejs-core/src/runner.ts b/packages/paddlejs-core/src/runner.ts
@@ -22,7 +22,6 @@ interface ModelConfig {
         height: number;
         width: number;
     };
-    fileCount?: number; // 参数分片chunk_*.dat 个数
     fill?: string; // 缩放后用什么颜色填充不足方形部分
     mean?: number[];
     std?: number[];
@@ -84,8 +83,8 @@ export default class Runner {
     }
 
     async load() {
-        const { modelPath, fileCount } = this.modelConfig;
-        const loader = new Loader(modelPath, fileCount);
+        const { modelPath } = this.modelConfig;
+        const loader = new Loader(modelPath);
         this.model = await loader.load();
     }
 
@@ -153,7 +152,7 @@ export default class Runner {
         if (Array.isArray(data)) {
             if ((data[0] as InputFeed)?.data) {
                 // 已经构建好的inputFeed
-                let inputData = (data[0] as InputFeed).data;
+                const inputData = (data[0] as InputFeed).data;
 
                 // 确保输入是Float32Array
                 if (!(inputData instanceof Float32Array)) {

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`{`
	`2`	`+ "chunkNum": 0,`
`2`	`3`	`"ops": [`
`3`	`4`	`{`
`4`	`5`	`"attrs": {`