Merge pull request #151 from JingyuanZhang/master

fix mediaProcessor param scale & add op bilinear_interp_v2

Author: JingyuanZhang

packages/paddlejs-backend-webgl/src/ops/index.ts

@@ -23,6 +23,7 @@ import split from './shader/split';
 import batchnorm from './shader/batchnorm';
 import reshape2 from './shader/reshape2';
 import bilinear_interp from './shader/bilinear_interp';
+import bilinear_interp_v2 from './shader/bilinear_interp_v2';
 import transpose2 from './shader/transpose2';
 import softmax from './shader/softmax';
 import dynamic from './shader/dynamic';
@@ -88,7 +89,8 @@ const ops = {
     pow: dynamic('pow'),
     sqrt: dynamic('sqrt'),
     squeeze2,
-    pad3d
+    pad3d,
+    bilinear_interp_v2
 };
 export {
     ops

packages/paddlejs-backend-webgl/src/ops/shader/bilinear_interp.ts

@@ -25,12 +25,6 @@ function mainFunc(
     void main(void) {
         // 输出数据
         ivec4 oPos = getOutputTensorPos();
-        // 输出坐标转换为输入坐标
-        //int sumVal = oPos.g
-            + oPos.a * ${out.channel}
-            + oPos.b * ${out.channel} * ${out.width_shape}
-            + oPos.r * ${out.channel} * ${out.width_shape} * ${out.height_shape};
-
 
         bool align_flag = ${align_mode} == 0 && !${align_corners};
 

packages/paddlejs-backend-webgl/src/ops/shader/bilinear_interp_v2.ts

@@ -0,0 +1,8 @@
+/**
+ * @file bilinear_interp_v2
+ * @implements 差值实现方式（BilinearInterpolation）与 bilinear_interp 一致，唯一的区别是 outshape 计算
+ */
+
+import bilinear_interp from './bilinear_interp';
+
+export default bilinear_interp;

packages/paddlejs-backend-webgl/src/webgl/buildShader.ts

@@ -13,10 +13,9 @@ import * as textureFunc from '../ops/atom/common_func_with_texture';
 
 export default function buildShader(textureConf, type, inputTensors, fShaderParams, runtime: number, isPacked = false) {
     let code = '';
+    const opName = getExactOpName(type, isPacked);
     try {
 
-        const opName = getExactOpName(type, isPacked);
-
         const { params = {}, mainFunc, textureFuncConf = {}, commonFuncConf } = ops[opName];
 
         // textureList: [filter, origin, bias]
@@ -50,7 +49,7 @@ export default function buildShader(textureConf, type, inputTensors, fShaderPara
         code = populateData(code);
     }
     catch (e) {
-        console.error(e);
+        console.error(`[${opName}]: ` + e);
     }
 
     return code;

packages/paddlejs-core/src/mediaProcessor.ts

@@ -8,7 +8,7 @@ type Color = string;
 
 export default class MediaProcessor {
     targetContext: CanvasRenderingContext2D = {} as CanvasRenderingContext2D;
-    gapFillWith: Color = '#000';
+    gapFillWith: Color = '#fff';
     mean: number[] = [0, 0, 0];
     std: number[] = [1, 1, 1];
     bgr: boolean = false;
@@ -53,7 +53,9 @@ export default class MediaProcessor {
         let data: ImageData | number[] = [];
         let scaleSize;
 
-        if (!(pixels instanceof HTMLImageElement || pixels instanceof HTMLVideoElement)) {
+        if (!(pixels instanceof HTMLImageElement
+            || pixels instanceof HTMLVideoElement
+            || pixels instanceof HTMLCanvasElement)) {
             return [{
                 data: data,
                 shape: opt.shape || opt.targetShape,
@@ -63,19 +65,17 @@ export default class MediaProcessor {
 
         this.pixelWidth = pixels.width;
         this.pixelHeight = pixels.height;
-
-        if (opt.scale && opt.targetSize) { // Moblienet的情况
+        if (opt.scale && opt.targetSize) {
             data = this.resizeAndFitTargetSize(pixels, opt);
         }
-        else if (opt.targetSize) { // 如果有targetSize，就是装在目标宽高里的模式 TinyYolo的情况
+        else if (opt.targetSize) { // 如果有 targetSize，就是装在目标宽高里的模式
             scaleSize = this.fitToTargetSize(pixels, opt);
             data = this.getImageData(0, 0, scaleSize);
         }
         else {
             scaleSize = this.reSize(pixels, opt);
             data = this.getImageData(0, 0, scaleSize);
         }
-
         if (opt.gray) {
             data = this.grayscale(data);
         }
@@ -142,18 +142,17 @@ export default class MediaProcessor {
      * @param opt.mean 均值
      * @param opt.std 方差
      * @param opt.targetShape 输出shape
-     * @param opt.normalizeType 0：将数据映射为0~1， 1：映射为-1~1之间
      */
     allReshapeToRGB(imageData, opt) {
 
         // mean和std是介于0-1之间的
-        const { mean, std, normalizeType = 0, targetShape } = opt;
+        const { mean, std, targetShape } = opt;
         const [, c, h, w] = targetShape;
         const data = imageData.data || imageData;
 
         const result = this.result;
         let offset = 0;
-
+        const normalizeType = 0; // 将数据映射为0~1， 1：映射为-1~1之间
         // h w c
         for (let i = 0; i < h; ++i) {
             const iw = i * w;
@@ -168,7 +167,6 @@ export default class MediaProcessor {
                 }
             }
         }
-
         return result;
     }
 
@@ -261,7 +259,7 @@ export default class MediaProcessor {
     /**
      * 缩放成目标尺寸并居中
      */
-    fitToTargetSize(image, params, center?) {
+    fitToTargetSize(image, params) {
         // 目标尺寸
         const targetWidth = params.targetSize.width;
         const targetHeight = params.targetSize.height;
@@ -284,12 +282,9 @@ export default class MediaProcessor {
             sh = Math.round(sw * this.pixelHeight / this.pixelWidth);
             y = Math.floor((targetHeight - sh) / 2);
         }
-        if (center) {
-            this.targetContext.drawImage(image, x, y, sw, sh);
-        }
-        else {
-            this.targetContext.drawImage(image, 0, 0, sw, sh);
-        }
+
+        this.targetContext.drawImage(image, x, y, sw, sh);
+
         return { sw: targetWidth, sh: targetHeight };
     }
 

packages/paddlejs-core/src/runner.ts

@@ -50,11 +50,7 @@ export default class Runner {
     needPreheat: boolean = true;
 
     constructor(options: ModelConfig | null) {
-        const opts = {
-            fill: '#fff',
-            scale: 256
-        };
-        this.modelConfig = Object.assign(opts, options);
+        this.modelConfig = Object.assign({}, options);
         this.needPreheat = options.needPreheat === undefined ? true : options.needPreheat;
         this.modelName = options.modelName || Date.now().toString();
         this.weightMap = [];
@@ -209,7 +205,6 @@ export default class Runner {
                 });
             }
         }
-
         else {
             preheatFeedData = vars.find(item => item.name === 'image');
             if (preheatFeedData) {
@@ -264,6 +259,7 @@ export default class Runner {
         if (op.type === 'fetch') {
             return;
         }
+
         op.execute(this.isExecuted);
         if (env.get('debug')
             && op.opData?.outputTensors

packages/paddlejs-core/src/transform/index.ts

@@ -1,5 +1,3 @@
-
-import { GLOBALS } from '../globals';
 import TexturePacking from './texturePacking';
 import FormatInputsX from './formatInputsX';
 import type Transformer from './transformer';
@@ -16,14 +14,10 @@ const actions: TransformerAction = {
         new SplitOp()
     ],
     transforms: [
-        new FormatInputsX()
+        new FormatInputsX(),
+        new TexturePacking()
     ],
     postTransforms: []
 };
 
-// wegbl backend单独处理, 在第一个transform后面添加texturePacking
-if (GLOBALS.backend === 'webgl') {
-    actions.transforms.splice(1, 0, new TexturePacking());
-}
-
 export default actions;

packages/paddlejs-core/src/transform/texturePacking.ts

@@ -1,6 +1,8 @@
 /**
  * @file texture_packing
  */
+
+import { GLOBALS } from '../globals';
 import OpExecutor from '../opFactory/opExecutor';
 import Transformer from './transformer';
 
@@ -107,6 +109,10 @@ export default class TexturePacking extends Transformer {
     }
 
     transform(...args: any) {
+        // wegbl backend 单独处理
+        if (GLOBALS.backend !== 'webgl') {
+            return;
+        }
         const [originOp, vars, opsMap] = args;
 
         if (!(packedOpConditions[originOp.type] && packedOpConditions[originOp.type](originOp, vars))) {