feat(paddlejs-converter & paddlejs-webgl): update paddlelite and add hard_swish, dropout op fuse

Author: changy1105

packages/paddlejs-backend-webgl/src/ops/atom/fuse_ops.ts

@@ -28,6 +28,15 @@ interface OpParams {
         prelu?: {};
         sqrt?: {};
         tanh?: {};
+        hard_swish?: {
+            offset?: number;
+            scale?: number;
+            threshold?: number;
+        };
+        dropout?: {
+            dropout_implementation: string;
+            dropout_prob: number;
+        }
     }
 }
 
@@ -83,8 +92,32 @@ export default function genFuseOpCode(params: OpParams) {
                 default:
                     break;
             }
-            activation_func += func[act_name];
-            calculation_str += `res = ${act_name}(x, float(${multi_value}), float(${bias_value}));`;
+
+            if (fuse === 'hard_swish') {
+                const offset = params.fuse_opt.hard_swish.offset !== undefined
+                    ? params.fuse_opt.hard_swish.offset
+                    : 3;
+                const scale = params.fuse_opt.hard_swish.scale !== undefined
+                    ? params.fuse_opt.hard_swish.scale
+                    : 6;
+                const threshold = params.fuse_opt.hard_swish.threshold !== undefined
+                    ? params.fuse_opt.hard_swish.threshold
+                    : 6;
+                // eslint-disable-next-line max-len
+                calculation_str += `res = res * min(max(0.0, res + float(${offset})), float(${threshold})) / float(${scale});`;
+            }
+            else if (fuse === 'dropout') {
+                const dropout_implementation = params.fuse_opt.dropout.dropout_implementation;
+                const dropout_prob = params.fuse_opt.dropout.dropout_prob;
+                calculation_str += `
+                if (${dropout_implementation === 'downgrade_in_infer'}) {
+                    res = res * (1.0 - float(${dropout_prob}));
+                }`;
+            }
+            else {
+                activation_func += func[act_name];
+                calculation_str += `res = ${act_name}(res, float(${multi_value}), float(${bias_value}));`;
+            }
         }
     }
 

packages/paddlejs-backend-webgl/src/ops/shader/conv2d_transpose.ts

@@ -5,12 +5,15 @@
 function mainFunc({
     origin,
     filter,
-    out
+    out,
+    bias
 }, {
     groups = 1,
     strides = [],
     paddings = [],
-    dilations = []
+    dilations = [],
+    fuse_relu,
+    act_type
 }) {
     const [stride_v = 1, stride_h = 1] = strides;
     let [padLeft = 0, padTop = 0] = paddings;
@@ -71,6 +74,16 @@ function mainFunc({
             }
             oy += ${dilation_v};
         }
+        
+        ${bias ? 'res += getValueFromTensorPos_bias(0, 0, 0, c);' : ''}
+        
+        if (${fuse_relu}) {
+            res = max(0.0, res);
+        }
+        else if (${act_type === 'relu6'}) {
+            res = min(max(0.0, res), 6.0);
+        }
+        
         setOutput(float(res));
     }
 `;
@@ -79,7 +92,8 @@ export default {
     mainFunc,
     textureFuncConf: {
         filter: ['getValueFromTensorPos'],
-        origin: ['getValueFromTensorPos']
+        origin: ['getValueFromTensorPos'],
+        bias: ['getValueFromTensorPos']
     },
     behaviors: [
         'adaptPaddings',

packages/paddlejs-backend-webgl/src/ops/shader/dropout.ts

@@ -16,7 +16,7 @@ function mainFunc(
         float o = 0.0;
         o = getValueFromTensorPos_origin(oPos.r, oPos.g, oPos.b, oPos.a);
         if (${dropout_implementation === 'downgrade_in_infer'}) {
-            o = o * (1.0 - ${dropout_prob});
+            o = o * (1.0 - float(${dropout_prob}));
         }
         setOutput(float(o));
     }

packages/paddlejs-converter/README.md

@@ -5,7 +5,7 @@
 System Requirements:
 
 * paddlepaddle >= 2.0.0
-* paddlelite == 2.7.1
+* paddlejslite >= 0.0.2
 * Python3： 3.5.1+ / 3.6 / 3.7
 * Python2： 2.7.15+
 
@@ -26,4 +26,4 @@ pip3 install paddlejsconverter
 ```shell
 paddlejsconverter --modelPath=user_model_path --paramPath=user_model_params_path --outputDir=model_saved_path --useGPUOpt=True
 ```
-注意：useGPUOpt 选项默认不开启，如果模型用在 gpu backend（webgl/webgpu），则开启 useGPUOpt，如果模型运行在（wasm/plain js）则不要开启。
+注意：useGPUOpt 选项默认不开启，如果模型用在 gpu backend（webgl/webgpu），则开启 useGPUOpt，如果模型运行在（wasm/plain js）则不要开启。

packages/paddlejs-converter/fuseOps.py

@@ -12,7 +12,9 @@ def opListFuse(ops):
         'hard_sigmoid',
         'pow',
         'sqrt',
-        'tanh'
+        'tanh',
+        'hard_swish',
+        'dropout'
     ]
 
     # 判断op是否为单节点
@@ -38,8 +40,23 @@ def opExistSingleNode(opName):
 
     for index in reversed(range(len(ops))):
         if index > 0:
+            op = ops[index]
+
+            # 兼容paddlelite 算子融合字段
+            if 'act_type' in op['attrs']:
+                name = op['attrs']['act_type']
+                op['attrs']['fuse_opt'] = {}
+                op['attrs']['fuse_opt'][name] = {}
+
+                if name == 'hard_swish':
+                    op['attrs']['fuse_opt'][name]['offset'] = op['attrs']['hard_swish_offset']
+                    op['attrs']['fuse_opt'][name]['scale'] = op['attrs']['hard_swish_scale']
+                    op['attrs']['fuse_opt'][name]['threshold'] = op['attrs']['hard_swish_threshold']
+
+                if name == 'relu6':
+                    op['attrs']['fuse_opt'][name]['threshold'] = op['attrs']['fuse_brelu_threshold']
+
             for fuse in fuseOpList:
-                op = ops[index]
                 if op['type'] == fuse:
                     prevOp = ops[index - 1]
 
@@ -53,3 +70,6 @@ def opExistSingleNode(opName):
                         prevOp['outputs']['Out'] = op['outputs']['Out']
 
                         del ops[index]
+
+
+

packages/paddlejs-converter/optimizeModel.py

@@ -4,7 +4,7 @@
 import collections
 import argparse
 import traceback
-from paddlelite import lite
+from paddlejslite import lite
 import pkg_resources
 from packaging import version
 
@@ -24,46 +24,6 @@ def optimizeModel(inputDir, modelPath, paramPath, outputDir):
     opt.set_valid_places("arm")
     opt.set_model_type("protobuf")
     opt.set_optimize_out(outputDir)
-
-    if version.parse(lite_version) <= version.parse('2.7.1'):
-        print("python paddlelite version: " + lite_version)
-
-        optimize_passes = [
-            "lite_conv_elementwise_fuse_pass",
-            "lite_conv_bn_fuse_pass",
-            "lite_conv_elementwise_fuse_pass",
-            "lite_conv_activation_fuse_pass",
-            "lite_var_conv_2d_activation_fuse_pass",
-            "lite_fc_fuse_pass",
-            "lite_shuffle_channel_fuse_pass",
-            "lite_transpose_softmax_transpose_fuse_pass",
-            "lite_interpolate_fuse_pass",
-            "identity_scale_eliminate_pass",
-            "elementwise_mul_constant_eliminate_pass",
-            "lite_sequence_pool_concat_fuse_pass",
-            "lite_elementwise_add_activation_fuse_pass",
-            "static_kernel_pick_pass",
-            "variable_place_inference_pass",
-            "argument_type_display_pass",
-            "type_target_cast_pass",
-            "variable_place_inference_pass",
-            "argument_type_display_pass",
-            "io_copy_kernel_pick_pass",
-            "argument_type_display_pass",
-            "variable_place_inference_pass",
-            "argument_type_display_pass",
-            "type_precision_cast_pass",
-            "variable_place_inference_pass",
-            "argument_type_display_pass",
-            "type_layout_cast_pass",
-            "argument_type_display_pass",
-            "variable_place_inference_pass",
-            "argument_type_display_pass",
-            "runtime_context_assign_pass",
-            "argument_type_display_pass"
-        ]
-        opt.set_passes_internal(optimize_passes)
-
     opt.run()
 
 
@@ -74,20 +34,20 @@ def main():
         p.add_argument('--modelPath', help='fluid模型文件所在路径，使用合并参数文件时使用该参数', required=False)
         p.add_argument('--paramPath', help='fluid参数文件所在路径，使用合并参数文件时使用该参数', required=False)
         p.add_argument("--outputDir", help='优化后fluid模型目录，必要参数', required=True)
-       
+
         args = p.parse_args()
         inputDir = args.inputDir
         modelPath = args.modelPath
         paramPath = args.paramPath
         outputDir = args.outputDir
 
         optimizeModel(inputDir, modelPath, paramPath, outputDir)
-            
+
     except Exception as identifier:
         print("\033[31mA fetal error occured. Failed to optimize model.\033[0m")
         print(traceback.format_exc())
         pass
 
 
 if __name__ == "__main__":
-    main()
+    main()

packages/paddlejs-converter/setup.py

@@ -24,7 +24,7 @@
     python_requires='>=3.6',
     install_requires=[
         "paddlepaddle >= 2.0.0",
-        "paddlelite==2.7.1",
+        "paddlejslite >= 0.0.2",
         "numpy"
     ],
     entry_points={
@@ -34,4 +34,4 @@
             "pdjsOptimizeModel = optimizeModel:main"
         ]
     }
-)
+)