fix(VisualReplayStrategy): increase default NMS thresh/conf (#758)

* repair VisualReplayStrategy

* black/flake8

Author: abrichr

experiments/fastsamsom.py

@@ -25,7 +25,13 @@ def main() -> None:
         image_contrasted.show()
 
     segmentation_adapter = adapters.get_default_segmentation_adapter()
-    segmented_image = segmentation_adapter.fetch_segmented_image(image)
+    segmented_image = segmentation_adapter.fetch_segmented_image(
+        image,
+        # threshold below which boxes will be filtered out
+        conf=0,
+        # discards all overlapping boxes with IoU > iou_threshold
+        iou=0.05,
+    )
     if DEBUG:
         segmented_image.show()
 

openadapt/adapters/openai.py

@@ -195,6 +195,7 @@ def prompt(
     """
     logger.info(f"system_prompt=\n{system_prompt}")
     logger.info(f"prompt=\n{prompt}")
+    images = images or []
     logger.info(f"{len(images)=}")
     payload = create_payload(
         prompt,

openadapt/adapters/ultralytics.py

@@ -49,21 +49,23 @@
 def fetch_segmented_image(
     image: Image.Image,
     model_name: str = DEFAULT_MODEL_NAME,
+    **kwargs,
 ) -> Image.Image:
     """Segment a PIL.Image using ultralytics.
 
     Args:
         image: The input image to be segmented.
         model_name: The name of the model to use.
+        kwargs: Arguments to pass to segmentation function.
 
     Returns:
         The segmented image as a PIL Image.
     """
     assert model_name in MODEL_NAMES, "{model_name=} must be in {MODEL_NAMES=}"
     if model_name in FASTSAM_MODEL_NAMES:
-        return do_fastsam(image, model_name)
+        return do_fastsam(image, model_name, **kwargs)
     else:
-        return do_sam(image, model_name)
+        return do_sam(image, model_name, **kwargs)
 
 
 @cache.cache()
@@ -75,9 +77,9 @@ def do_fastsam(
     retina_masks: bool = True,
     imgsz: int | tuple[int, int] | None = 1024,
     # threshold below which boxes will be filtered out
-    conf: float = 0,
+    conf: float = 0.4,
     # discards all overlapping boxes with IoU > iou_threshold
-    iou: float = 0.05,
+    iou: float = 0.9,
 ) -> Image:
     model = FastSAM(model_name)
 

openadapt/strategies/visual.py

@@ -45,6 +45,7 @@
 """
 
 from dataclasses import dataclass
+from pprint import pformat
 import time
 
 from loguru import logger
@@ -485,6 +486,7 @@ def prompt_for_descriptions(
         exceptions=exceptions,
     )
     logger.info(f"prompt=\n{prompt}")
+    logger.info(f"{len(images)=}")
     descriptions_json = prompt_adapter.prompt(
         prompt,
         system_prompt,
@@ -498,8 +500,16 @@ def prompt_for_descriptions(
             len(masked_images),
         )
     except Exception as exc:
-        # TODO XXX
-        raise exc
+        exceptions = exceptions or []
+        exceptions.append(exc)
+        logger.info(f"exceptions=\n{pformat(exceptions)}")
+        return prompt_for_descriptions(
+            original_image,
+            masked_images,
+            active_segment_description,
+            exceptions,
+        )
+
     # remove indexes
     descriptions = [desc for idx, desc in descriptions]
     return descriptions

openadapt/window/__init__.py

@@ -19,14 +19,16 @@
     raise Exception(f"Unsupported platform: {sys.platform}")
 
 
-def get_active_window_data() -> dict[str, Any] | None:
+def get_active_window_data(
+    include_window_data: bool = config.RECORD_WINDOW_DATA,
+) -> dict[str, Any] | None:
     """Get data of the active window.
 
     Returns:
         dict or None: A dictionary containing information about the active window,
         or None if the state is not available.
     """
-    state = get_active_window_state(config.RECORD_WINDOW_DATA)
+    state = get_active_window_state(include_window_data)
     if not state:
         return None
     title = state["title"]

scripts/downloads.py

@@ -2,7 +2,7 @@
 
 import requests
 import matplotlib.pyplot as plt
-from datetime import datetime
+from datetime import datetime, timedelta
 from pprint import pformat
 
 import numpy as np
@@ -58,7 +58,8 @@ def fetch_download_data(api_url: str) -> dict:
 def plot_downloads(data: dict) -> None:
     """Plots number of downloads and cumulative downloads over time using matplotlib.
 
-    Includes total cumulative in the title.
+    Includes total cumulative in the title and annotates a specific event date with
+    styled text.
 
     Args:
         data (dict): A dictionary with dates as keys and download counts as values.
@@ -82,9 +83,28 @@ def plot_downloads(data: dict) -> None:
         color="r",
         label="Cumulative Downloads",
     )
+
+    # Annotation for the release download button addition
+    event_date = datetime(2024, 5, 9, 2, 46)  # Year, Month, Day, Hour, Minute
+    plt.axvline(x=event_date, color="g", linestyle=":", label="Download Buttons Added")
+    plt.annotate(
+        "Download Buttons Added at\nwww.openadapt.ai",
+        xy=(event_date, plt.ylim()[0] + 100),
+        xytext=(
+            event_date - timedelta(days=10),
+            plt.ylim()[1] * 0.85,
+        ),  # Shift left by 10 days
+        horizontalalignment="center",
+        fontsize=10,
+        bbox=dict(
+            boxstyle="round,pad=0.5", edgecolor="green", facecolor="#ffffcc", alpha=0.9
+        ),
+    )
+
     current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     plt.title(
-        "Downloads Over Time"
+        "github.com/OpenAdaptAI/OpenAdapt"
+        "\nRelease Downloads Over Time"
         f"\n(Total Cumulative: {total_cumulative_downloads}) "
         f"\n{current_time}"
     )