Merge pull request #200 from microsoft/vyokky/dev

release of UFO2

Author: vyokky

README.md

@@ -1,6 +1,6 @@
 version: 0.1
 
-CONTROL_BACKEND: "uia"  # The backend for control action
+CONTROL_BACKEND: "uia"  # The list of backend for control action, currently we support uia and win32, 
 CONTROL_LIST: ["Button", "Edit", "TabItem", "Document", "ListItem", "MenuItem", "ScrollBar", "TreeItem", "Hyperlink", "ComboBox", "RadioButton", "DataItem", "Spinner"] 
 PRINT_LOG: False  # Whether to print the log  
 LOG_LEVEL: "INFO"  # The log level

README_v1.md

@@ -16,7 +16,7 @@
 _ufo_configs = UFOConfig.get_instance().config_data
 
 if _ufo_configs is not None:
-    _BACKEND = _ufo_configs["CONTROL_BACKEND"]
+    _BACKEND = "uia"
 if _configs is not None:
     _MATCH_STRATEGY = _configs.get("MATCH_STRATEGY", "contains")
 
@@ -64,14 +64,16 @@ def close(self) -> None:
             if self.app_window and self.app_window.process_id():
                 self.app_window.close()
             sleep(1)
-            # Forcefully close the application window  
-            if self.app_window.element_info.name.lower() != '':            
+            # Forcefully close the application window
+            if self.app_window.element_info.name.lower() != "":
                 self._check_and_kill_process()
         except Exception as e:
-            logging.warning(f"Graceful close failed: {e}. Attempting to forcefully terminate the process.")
+            logging.warning(
+                f"Graceful close failed: {e}. Attempting to forcefully terminate the process."
+            )
             self._check_and_kill_process()
             raise e
-    
+
     def _check_and_kill_process(self) -> None:
         """
         Checks if the process is still running and kills it if it is.
@@ -151,9 +153,11 @@ def _calculate_match_score(self, control, control_text) -> int:
             return 100 if re.search(pattern, control_content) else 0
         else:
             raise ValueError(f"Unknown match strategy: {_MATCH_STRATEGY}")
-    
-    def find_matching_controller(self, filtered_annotation_dict: Dict[int, UIAWrapper], control_text: str) -> Tuple[str, UIAWrapper]:
-        """"
+
+    def find_matching_controller(
+        self, filtered_annotation_dict: Dict[int, UIAWrapper], control_text: str
+    ) -> Tuple[str, UIAWrapper]:
+        """ "
         Select the best matched controller.
         :param filtered_annotation_dict: The filtered annotation dictionary.
         :param control_text: The text content of the control for additional context.

assets/comparison.png

@@ -15,8 +15,6 @@
 
 _configs = InstantiationConfig.get_instance().config_data
 _ufo_configs = UFOConfig.get_instance().config_data
-if _ufo_configs is not None:
-    BACKEND = _ufo_configs["CONTROL_BACKEND"]
 
 
 class ExecuteFlow(AppAgentProcessor):
@@ -316,9 +314,11 @@ def execute_action(self) -> None:
         if self.control_text == "":
             control_selected = self.application_window
         else:
-            self._control_label, control_selected = self._app_env.find_matching_controller(
-                self.filtered_annotation_dict, self.control_text
+            self._control_label, control_selected = (
+                self._app_env.find_matching_controller(
+                    self.filtered_annotation_dict, self.control_text
                 )
+            )
             if control_selected:
                 self._matched_control = control_selected.window_text()
 

assets/framework2.png

@@ -59,15 +59,13 @@ def message_constructor(
         dynamic_examples: str,
         given_task: str,
         reference_steps: List[str],
-        doc_control_state: Dict[str, str],
         log_path: str,
     ) -> List[str]:
         """
         Construct the prompt message for the PrefillAgent.
         :param dynamic_examples: The dynamic examples retrieved from the self-demonstration and human demonstration.
         :param given_task: The given task.
         :param reference_steps: The reference steps.
-        :param doc_control_state: The document control state.
         :param log_path: The path of the log.
         :return: The prompt message.
         """
@@ -76,7 +74,7 @@ def message_constructor(
             dynamic_examples
         )
         prefill_agent_prompt_user_message = self.prompter.user_content_construction(
-            given_task, reference_steps, doc_control_state, log_path
+            given_task, reference_steps, log_path
         )
         appagent_prompt_message = self.prompter.prompt_construction(
             prefill_agent_prompt_system_message,

dataflow/config/config_dev.yaml

@@ -15,8 +15,7 @@
 
 _configs = Config.get_instance().config_data
 _ufo_configs = UFOConfig.get_instance().config_data
-if _ufo_configs is not None:
-    _BACKEND = _ufo_configs["CONTROL_BACKEND"]
+_BACKEND = "uia"
 
 
 class PrefillFlow(AppAgentProcessor):
@@ -98,10 +97,10 @@ def execute(
         finally:
             self.execution_time = round(time.time() - start_time, 3)
 
-        return  {
+        return {
             "instantiated_request": instantiated_request,
             "instantiated_plan": instantiated_plan,
-        }   
+        }
 
     def _instantiate_task(
         self, template_copied_path: str, original_task: str, refined_steps: List[str]
@@ -131,7 +130,7 @@ def _instantiate_task(
             raise e
 
         return instantiated_request, instantiated_plan
-    
+
     def _update_state(self, file_path: str) -> None:
         """
         Update the current state of the app by inspecting UI elements.
@@ -194,7 +193,6 @@ def _get_prefill_actions(
             "",
             given_task,
             reference_steps,
-            self._filtered_control_info,
             self._log_path_configs,
         )
 
@@ -260,7 +258,7 @@ def _save_screenshot(self, doc_name: str, save_path: str) -> None:
         :param doc_name: The name or description of the document to match the window.
         :param save_path: The path where the screenshot will be saved.
         """
-        
+
         try:
             # Find the window matching the document name
             matched_window = self._app_env.find_matching_window(doc_name)

dataflow/env/env_manager.py

@@ -75,20 +75,18 @@ def system_prompt_construction(self, additional_examples: List = []) -> str:
         return self.prompt_template["system"].format(apis=apis, examples=examples)
 
     def user_prompt_construction(
-        self, given_task: str, reference_steps: List, doc_control_state: Dict
+        self, given_task: str, reference_steps: List
     ) -> str:
         """
         Construct the prompt for the user.
         :param given_task: The given task.
         :param reference_steps: The reference steps.
-        :param doc_control_state: The document control state.
         :return: The prompt for the user.
         """
 
         prompt = self.prompt_template["user"].format(
             given_task=given_task,
-            reference_steps=json.dumps(reference_steps),
-            doc_control_state=json.dumps(doc_control_state),
+            reference_steps=json.dumps(reference_steps)
         )
 
         return prompt
@@ -110,14 +108,12 @@ def user_content_construction(
         self,
         given_task: str,
         reference_steps: List,
-        doc_control_state: Dict,
         log_path: str,
     ) -> List[Dict]:
         """
         Construct the prompt for LLMs.
         :param given_task: The given task.
         :param reference_steps: The reference steps.
-        :param doc_control_state: The document control state.
         :param log_path: The path of the log.
         :return: The prompt for LLMs.
         """
@@ -135,7 +131,7 @@ def user_content_construction(
             {
                 "type": "text",
                 "text": self.user_prompt_construction(
-                    given_task, reference_steps, doc_control_state
+                    given_task, reference_steps
                 ),
             }
         )