Notebook for deploying a fine-tuned ResNet18 CNN model using AzureML for Image Similarity (#560)

* Added notebook for deploying a fine-tuned resnet18 CNN model using AzureML that builds on the html_demo notebooks with a custom scoring script that saves the features in the intermediate layers and returns them in JSON format. Also added fix to script.js to support image similarity scenarios.

* Updated readme

* Fixed typo in readme

* Addressed PR comment - removed link in notebook.

* Addressing PR comments: Added descriptions for the notebook and updated its name.

* Addressed PR comment: Made DATA_PATH point to the returned value from unzip_url using our fridge_objects dataset so that the user can just run the notebook.

Author: youngpark

contrib/html_demo/JupyterCode/4_train_and_deploy_custom_image_similarity_webapp.ipynb

@@ -10,6 +10,7 @@ This directory contains a few helper notebooks that upload files and deploy mode
 | [1_image_similarity_export.ipynb](1_image_similarity_export.ipynb)| Exports computed reference image features for use in visualizng results (see details in "Image Similarity" section below) |
 | [2_upload_ui.ipynb](2_upload_ui.ipynb)| Uploads web page files to Azure Blob storage |
 | [3_deployment_to_azure_app_service.ipynb](3_deployment_to_azure_app_service.ipynb)| Deploys image classification model as an Azure app service |
+| [4_train_and_deploy_custom_image_similarity_webapp.ipynb](4_train_and_deploy_custom_image_similarity_webapp.ipynb) | Fine-tunes a ResNet18 CNN model and deploys a custom image similarity webapp using AzureML
 
 ### Requirements
 
@@ -28,14 +29,15 @@ To run the code in the [2_upload_ui.ipynb](2_upload_ui.ipynb) notebook, you must
 * If you want to use an image similarity model, you can run [1_image_similarity_export.ipynb](1_image_similarity_export.ipynb) to export your image features for the web page to use.
 * To upload the web page for sharing, notebook [2_upload_ui.ipynb](2_upload_ui.ipynb) outlines the process of uploading to your Azure Blob storage.
 * As the web page needs the API to allow CORS, we recommend uploading models as an Azure app service. Notebook [3_deployment_to_azure_app_service.ipynb](3_deployment_to_azure_app_service.ipynb) gives a tutorial on how to do so with an image classification model.
+* [4_train_and_deploy_custom_image_similarity_webapp.ipynb](4_train_and_deploy_custom_image_similarity_webapp.ipynb) guides through the process of deploying a custom image similarity web application - from finetuning a RESNET50 model using a sample dataset in ImageNet directory structure format to updating required files for the web application and deploying them along with the model.
 
 ### Image Similarity
 
 Image similarity relies on comparing DNN features of a query image, to the respective DNN features of potentially tens of thousands of references images. The notebooks in this directory compute these reference image DNN features and package them for use in the HTML UI. The DNN features are exported into a text file and compressed to be uploaded with the HTML UI files. To compare a query image to these exported reference image features, you will need a DNN model deployed to Azure App services that is able to compute the features of the query image and return them via API call.      
 
 Steps:
-1. Execute "1_image_similarity_export.ipynb" notebook to generate your reference image features and export them to compressed ZIP files
-2. Execute "2_upload_ui.ipynb" notebook to upload the HTML UI and all supporting files to your Azure Blob storage account
-3. Execute "3_deployment_to_azure_app_service.ipynb" notebook to upload your model for generating DNN features for your query image and create an API endpoint in Azure App service
+1. Execute [1_image_similarity_export.ipynb](1_image_similarity_export.ipynb) notebook to generate your reference image features and export them to compressed ZIP files
+2. Execute [2_upload_ui.ipynb](2_upload_ui.ipynb) notebook to upload the HTML UI and all supporting files to your Azure Blob storage account
+3. Execute [3_deployment_to_azure_app_service.ipynb](3_deployment_to_azure_app_service.ipynb) notebook to upload your model for generating DNN features for your query image and create an API endpoint in Azure App service
 4. Open the index.html file from your Blob storage account in a browser, enter your API endpoint URL, upload a query image and see what you get back
-
+5. *(Optional)* Execute [4_train_and_deploy_custom_image_similarity_webapp.ipynb](4_train_and_deploy_custom_image_similarity_webapp.ipynb) notebook to finetune a RESNET50 model and deploy a custom image similarity web application 

contrib/html_demo/JupyterCode/readme.md

@@ -48,7 +48,6 @@ function populateTable(i, tableData) {
     item.classList.add("item");
     var img = document.createElement('img');
     img.src = 'https://cvbp.blob.core.windows.net/public/html_demo/small-150/' + rowData[0];
-    //img.src = 'small-150/' + rowData[0];
     var txt = document.createElement('p');
     txt.innerHTML = rowData[0] + "<br/><i>Dist.: " + rowData[1] + "</i>";
     item.appendChild(img);
@@ -78,7 +77,6 @@ function calcSimilar(top, queryFeatures, simType) {
     var queryRow = Math.floor(Math.random() * (rows - 0 + 1) + 0);
     var queryimg = ref_array[queryRow];
     retImg = 'https://cvbp.blob.core.windows.net/public/html_demo/small-150/' + fn_array[queryRow];
-    //retImg = 'small-150/' + fn_array[queryRow];
   } else {
     var queryimg = queryFeatures;
   }
@@ -104,7 +102,6 @@ async function parseSimFileNames(fileType) {
       zipFile_fn = 'data/ref_filenames.zip';
       if (fileType == "example") 
         zipFile_fn = 'https://cvbp.blob.core.windows.net/public/html_demo/data/ref_filenames.zip';
-      //JSZipUtils.getBinaryContent('data/ref_filenames.zip', function(err, data) {
       JSZipUtils.getBinaryContent(zipFile_fn, function(err, data) {
           if (err) {
             reject(err);
@@ -138,7 +135,6 @@ async function parseSimFileFeatures(fileType) {
       zipFile_ref = 'data/ref_features.zip';
       if (fileType == "example") 
         zipFile_ref = 'https://cvbp.blob.core.windows.net/public/html_demo/data/ref_features.zip';
-      // JSZipUtils.getBinaryContent('data/ref_features.zip', function(err, data) {
       JSZipUtils.getBinaryContent(zipFile_ref, function(err, data) {
           if (err) {
               reject(err);
@@ -319,7 +315,7 @@ $('#multiCollapseWebcam').on('shown.bs.collapse', function () {
 
 $('#multiCollapseSample').on('hidden.bs.collapse', function () {
   document.getElementById("btnSample").classList.remove("active");
-  document.getElementById("btnSample").innerText = "Samples";
+  document.getElementById("btnSample").innerText = "Choose From Samples";
 })
 
 $('#multiCollapseSample').on('shown.bs.collapse', function () {
@@ -556,7 +552,7 @@ async function jsonParser(jString, ovr) {
   if (Array.isArray(resp[0])) {
     if (resp[0][0].hasOwnProperty("top")) {
       // "[[top: #, ]]"
-      //will need to target a different feature if another scenario ends up doing rectangle boxes
+      // will need to target a different feature if another scenario ends up doing rectangle boxes
       for (let i in resp) {
         let j = i
         if (ovr) {
@@ -600,7 +596,11 @@ async function jsonParser(jString, ovr) {
       }
       await renderImage(j);
       let features = resp[i].features;
-      imgsimilarity(j, 5, features);
+
+      // parse the json into an array
+      let featuresArray = JSON.parse(features)
+
+      imgsimilarity(j, 5, featuresArray);
     }
     return "similarity"
   }
@@ -618,8 +618,7 @@ function APIRequest() {
   uplBtn.disabled = "true";
   uplStatus.classList.remove("hide");
   uplStatus.innerHTML = 'Loading... <div class="spinner-border ml-auto spinner-border-sm" role="status" aria-hidden="true"></div>';
-  //console.log(url);
-  //console.log(JSON.stringify({data: b64o}));
+
   let xhr = new XMLHttpRequest();
 
   xhr.onload = function() {