Merge pull request #5 from algorithmiaio/advanced-example-update

moved all global functionality into load function dictionary

Author: zeryx

README.md

@@ -104,6 +104,7 @@ def load():
     # The return object from this function can be passed directly as input to your apply function.
     # A great example would be any model files that need to be available to this algorithm
     # during runtime.
+
     # Any variables returned here, will be passed as the secondary argument to your 'algorithm' function
     globals = {}
     globals['payload'] = "Loading has been completed."
@@ -121,53 +122,41 @@ algorithm.init("Algorithmia")
 ## [pytorch based image classification](examples/pytorch_image_classification)
 <!-- embedme examples/pytorch_image_classification/src/Algorithm.py -->
 ```python
-from Algorithmia import client, ADK
+from Algorithmia import ADK
+import Algorithmia
 import torch
 from PIL import Image
 import json
 from torchvision import models, transforms
 
-CLIENT = client()
-SMID_ALGO = "algo://util/SmartImageDownloader/0.2.x"
-LABEL_PATH = "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json"
-MODEL_PATHS = {
-    "squeezenet": 'data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth',
-    'alexnet': 'data://AlgorithmiaSE/image_cassification_demo/alexnet-owt-4df8aa71.pth',
-}
-
-
-def load_labels():
-    local_path = CLIENT.file(LABEL_PATH).getFile().name
+def load_labels(label_path, client):
+    local_path = client.file(label_path).getFile().name
     with open(local_path) as f:
         labels = json.load(f)
     labels = [labels[str(k)][1] for k in range(len(labels))]
     return labels
 
 
-def load_model(name):
-    if name == "squeezenet":
-        model = models.squeezenet1_1()
-        models.densenet121()
-        weights = torch.load(CLIENT.file(MODEL_PATHS['squeezenet']).getFile().name)
-    else:
-        model = models.alexnet()
-        weights = torch.load(CLIENT.file(MODEL_PATHS['alexnet']).getFile().name)
+def load_model(model_paths, client):
+    model = models.squeezenet1_1()
+    local_file = client.file(model_paths["filepath"]).getFile().name
+    weights = torch.load(local_file)
     model.load_state_dict(weights)
     return model.float().eval()
 
 
-def get_image(image_url):
-    input = {"image": image_url, "resize": {'width': 224, 'height': 224}}
-    result = CLIENT.algo(SMID_ALGO).pipe(input).result["savePath"][0]
-    local_path = CLIENT.file(result).getFile().name
+def get_image(image_url, smid_algo, client):
+    input = {"image": image_url, "resize": {"width": 224, "height": 224}}
+    result = client.algo(smid_algo).pipe(input).result["savePath"][0]
+    local_path = client.file(result).getFile().name
     img_data = Image.open(local_path)
     return img_data
 
 
 def infer_image(image_url, n, globals):
-    model = globals['model']
-    labels = globals['labels']
-    image_data = get_image(image_url)
+    model = globals["model"]
+    labels = globals["labels"]
+    image_data = get_image(image_url, globals["SMID_ALGO"], globals["CLIENT"])
     transformed = transforms.Compose([
         transforms.ToTensor(),
         transforms.Normalize(mean=[0.485, 0.456, 0.406],
@@ -185,31 +174,36 @@ def infer_image(image_url, n, globals):
     return result
 
 
-def load():
-    globals = {'model': load_model("squeezenet"), 'labels': load_labels()}
+def load(manifest):
+
+    globals = {}
+    client = Algorithmia.client()
+    globals["SMID_ALGO"] = "algo://util/SmartImageDownloader/0.2.x"
+    globals["model"] = load_model(manifest["squeezenet"], client)
+    globals["labels"] = load_labels(manifest["label_file"], client)
     return globals
 
 
 def apply(input, globals):
     if isinstance(input, dict):
         if "n" in input:
-            n = input['n']
+            n = input["n"]
         else:
             n = 3
         if "data" in input:
-            if isinstance(input['data'], str):
-                output = infer_image(input['data'], n, globals)
-            elif isinstance(input['data'], list):
-                for row in input['data']:
-                    row['predictions'] = infer_image(row['image_url'], n, globals)
-                output = input['data']
+            if isinstance(input["data"], str):
+                output = infer_image(input["data"], n, globals)
+            elif isinstance(input["data"], list):
+                for row in input["data"]:
+                    row["predictions"] = infer_image(row["image_url"], n, globals)
+                output = input["data"]
             else:
-                raise Exception("'data' must be a image url or a list of image urls (with labels)")
+                raise Exception("\"data\" must be a image url or a list of image urls (with labels)")
             return output
         else:
-            raise Exception("'data' must be defined")
+            raise Exception("\"data\" must be defined")
     else:
-        raise Exception('input must be a json object')
+        raise Exception("input must be a json object")
 
 
 algorithm = ADK(apply_func=apply, load_func=load)

examples/pytorch_image_classification/model_manifest.json

@@ -0,0 +1,14 @@
+{
+  "label_file": {
+    "filepath": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
+    "md5_hash": "c2c37ea517e94d9795004a39431a14cb",
+    "origin_ref": "this file came from imagenet.org",
+    "uploaded_utc": "2021-05-03-11:05"
+  },
+    "squeezenet": {
+      "filepath": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
+      "md5_hash": "46a44d32d2c5c07f7f66324bef4c7266",
+      "origin_ref": "From https://download.pytorch.org/models/squeezenet1_1-f364aa15.pth",
+      "uploaded_utc": "2021-05-03-11:05"
+    }
+}

examples/pytorch_image_classification/src/Algorithm.py

@@ -1,50 +1,38 @@
-from Algorithmia import client, ADK
+from Algorithmia import ADK
+import Algorithmia
 import torch
 from PIL import Image
 import json
 from torchvision import models, transforms
 
-CLIENT = client()
-SMID_ALGO = "algo://util/SmartImageDownloader/0.2.x"
-LABEL_PATH = "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json"
-MODEL_PATHS = {
-    "squeezenet": 'data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth',
-    'alexnet': 'data://AlgorithmiaSE/image_cassification_demo/alexnet-owt-4df8aa71.pth',
-}
-
-
-def load_labels():
-    local_path = CLIENT.file(LABEL_PATH).getFile().name
+def load_labels(label_path, client):
+    local_path = client.file(label_path).getFile().name
     with open(local_path) as f:
         labels = json.load(f)
     labels = [labels[str(k)][1] for k in range(len(labels))]
     return labels
 
 
-def load_model(name):
-    if name == "squeezenet":
-        model = models.squeezenet1_1()
-        models.densenet121()
-        weights = torch.load(CLIENT.file(MODEL_PATHS['squeezenet']).getFile().name)
-    else:
-        model = models.alexnet()
-        weights = torch.load(CLIENT.file(MODEL_PATHS['alexnet']).getFile().name)
+def load_model(model_paths, client):
+    model = models.squeezenet1_1()
+    local_file = client.file(model_paths["filepath"]).getFile().name
+    weights = torch.load(local_file)
     model.load_state_dict(weights)
     return model.float().eval()
 
 
-def get_image(image_url):
-    input = {"image": image_url, "resize": {'width': 224, 'height': 224}}
-    result = CLIENT.algo(SMID_ALGO).pipe(input).result["savePath"][0]
-    local_path = CLIENT.file(result).getFile().name
+def get_image(image_url, smid_algo, client):
+    input = {"image": image_url, "resize": {"width": 224, "height": 224}}
+    result = client.algo(smid_algo).pipe(input).result["savePath"][0]
+    local_path = client.file(result).getFile().name
     img_data = Image.open(local_path)
     return img_data
 
 
 def infer_image(image_url, n, globals):
-    model = globals['model']
-    labels = globals['labels']
-    image_data = get_image(image_url)
+    model = globals["model"]
+    labels = globals["labels"]
+    image_data = get_image(image_url, globals["SMID_ALGO"], globals["CLIENT"])
     transformed = transforms.Compose([
         transforms.ToTensor(),
         transforms.Normalize(mean=[0.485, 0.456, 0.406],
@@ -62,31 +50,36 @@ def infer_image(image_url, n, globals):
     return result
 
 
-def load():
-    globals = {'model': load_model("squeezenet"), 'labels': load_labels()}
+def load(manifest):
+
+    globals = {}
+    client = Algorithmia.client()
+    globals["SMID_ALGO"] = "algo://util/SmartImageDownloader/0.2.x"
+    globals["model"] = load_model(manifest["squeezenet"], client)
+    globals["labels"] = load_labels(manifest["label_file"], client)
     return globals
 
 
 def apply(input, globals):
     if isinstance(input, dict):
         if "n" in input:
-            n = input['n']
+            n = input["n"]
         else:
             n = 3
         if "data" in input:
-            if isinstance(input['data'], str):
-                output = infer_image(input['data'], n, globals)
-            elif isinstance(input['data'], list):
-                for row in input['data']:
-                    row['predictions'] = infer_image(row['image_url'], n, globals)
-                output = input['data']
+            if isinstance(input["data"], str):
+                output = infer_image(input["data"], n, globals)
+            elif isinstance(input["data"], list):
+                for row in input["data"]:
+                    row["predictions"] = infer_image(row["image_url"], n, globals)
+                output = input["data"]
             else:
-                raise Exception("'data' must be a image url or a list of image urls (with labels)")
+                raise Exception("\"data\" must be a image url or a list of image urls (with labels)")
             return output
         else:
-            raise Exception("'data' must be defined")
+            raise Exception("\"data\" must be defined")
     else:
-        raise Exception('input must be a json object')
+        raise Exception("input must be a json object")
 
 
 algorithm = ADK(apply_func=apply, load_func=load)