From d1bbec1258fa0056433f4fa62725e9b6cf38c094 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 22:04:04 +0530 Subject: [PATCH 01/10] UN-4435 fix: enable highlight automatically for HITL confidence rules When a workflow's HITL rules (DB or API) contain a confidence condition, the orchestrator marks the execution with auto_enable_highlight in manual_review_config and writes one line to the execution logs. The file worker then passes the structure tool a copy of the tool instance metadata with enable_highlight on. The saved setting and word confidence are not changed. The check comes from the manual_review plugin; the OSS null util returns False, so OSS behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 18 ++++++++ workers/general/tasks.py | 14 +++++- workers/shared/utils/manual_review_factory.py | 10 ++++ workers/shared/workflow/execution/service.py | 30 ++++++++++-- workers/tests/test_structure_tool_pipeline.py | 46 ++++++++++++++++++- 5 files changed, 111 insertions(+), 7 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index fa7a218d1a..4a271c16b8 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -22,6 +22,11 @@ from shared.infrastructure.logging.workflow_logger import WorkerWorkflowLogger from shared.patterns.retry.utils import retry from shared.processing.files import FileProcessingUtils +from shared.utils.manual_review_factory import ( + AUTO_ENABLE_HIGHLIGHT, + HIGHLIGHT_AUTO_ENABLED_MESSAGE, + get_manual_review_workflow_util, +) from shared.workflow.execution import WorkerExecutionContext, WorkflowOrchestrationUtils from shared.workflow.execution.tool_validation import validate_workflow_tool_instances from worker import app @@ -665,6 +670,17 @@ def _run_workflow_api( total_files=total_files, ) + auto_enable_highlight = get_manual_review_workflow_util( + api_client, schema_name + ).has_confidence_rules(workflow_id) + if auto_enable_highlight: + _log_api_statistics_to_ui( + execution_id=execution_id, + organization_id=schema_name, + pipeline_id=pipeline_id, + message=HIGHLIGHT_AUTO_ENABLED_MESSAGE, + ) + # Create batch tasks following the exact Django pattern batch_tasks = [] execution_mode_str = ( @@ -687,6 +703,8 @@ def _run_workflow_api( total_files=total_files, **kwargs, ) + if auto_enable_highlight: + file_data.manual_review_config[AUTO_ENABLE_HIGHLIGHT] = True # Calculate manual review decisions for this specific batch if file_data.manual_review_config.get("review_required", False): diff --git a/workers/general/tasks.py b/workers/general/tasks.py index 861ddc7ddc..73d9e141b6 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -42,7 +42,11 @@ from shared.patterns.retry.utils import circuit_breaker from shared.processing.files import FileProcessingUtils from shared.processing.types import FileDataValidator, TypeConverter -from shared.utils.manual_review_factory import get_manual_review_service +from shared.utils.manual_review_factory import ( + AUTO_ENABLE_HIGHLIGHT, + HIGHLIGHT_AUTO_ENABLED_MESSAGE, + get_manual_review_service, +) from shared.workflow.connectors.source import WorkerSourceConnector from shared.workflow.execution import ( WorkerExecutionContext, @@ -978,6 +982,14 @@ def _orchestrate_file_processing_general( total_files=len(source_files), ) logger.debug("Global file data configured for manual review") + if workflow_util.has_confidence_rules(workflow_id): + global_file_data.manual_review_config[AUTO_ENABLE_HIGHLIGHT] = True + _log_batch_statistics_to_ui( + execution_id=execution_id, + organization_id=organization_id, + pipeline_id=pipeline_id, + message=HIGHLIGHT_AUTO_ENABLED_MESSAGE, + ) # Pre-calculate file decisions for ALL files based on total count - not per batch! q_file_no_list = global_file_data.manual_review_config.get("q_file_no_list", []) logger.info( diff --git a/workers/shared/utils/manual_review_factory.py b/workers/shared/utils/manual_review_factory.py index 69c1011553..e3f8c51dd8 100644 --- a/workers/shared/utils/manual_review_factory.py +++ b/workers/shared/utils/manual_review_factory.py @@ -17,6 +17,13 @@ MANUAL_REVIEW_PLUGIN_NAME = "manual_review" +# manual_review_config key: the workflow has HITL confidence rules, so the +# structure tool runs with highlight on to produce confidence scores. +AUTO_ENABLE_HIGHLIGHT = "auto_enable_highlight" +HIGHLIGHT_AUTO_ENABLED_MESSAGE = ( + "Highlight enabled automatically because HITL confidence rules are configured" +) + class ManualReviewServiceProtocol(Protocol): """Protocol defining the interface for manual review services.""" @@ -459,6 +466,9 @@ def get_hitl_ttl_seconds(self, *args, **kwargs): def get_manual_review_config(self, workflow_id, total_files): return {} + def has_confidence_rules(self, workflow_id): + return False + return WorkflowUtilNull() diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py index ebf8bce17c..c29f7422b1 100644 --- a/workers/shared/workflow/execution/service.py +++ b/workers/shared/workflow/execution/service.py @@ -53,6 +53,7 @@ from ...api.internal_client import InternalAPIClient from ...infrastructure.logging import WorkerLogger from ...utils.error_utils import get_user_friendly_error_message +from ...utils.manual_review_factory import AUTO_ENABLE_HIGHLIGHT from ..destination_connector import ( DestinationConfig, WorkerDestinationConnector, @@ -847,7 +848,12 @@ def _execute_workflow_with_service( # Continue with normal execution if check fails # Step 3: Build and execute workflow - self._build_and_execute_workflow(execution_service, file_name) + review_config = file_processing_context.file_data.manual_review_config + self._build_and_execute_workflow( + execution_service, + file_name, + auto_enable_highlight=review_config.get(AUTO_ENABLE_HIGHLIGHT, False), + ) return True @@ -969,7 +975,10 @@ def _prepare_workflow_input_file( raise file_prep_error def _build_and_execute_workflow( - self, execution_service: WorkflowExecutionService, file_name: str + self, + execution_service: WorkflowExecutionService, + file_name: str, + auto_enable_highlight: bool = False, ) -> None: """Build and execute the workflow. @@ -977,7 +986,9 @@ def _build_and_execute_workflow( execute_structure_tool task instead of the Docker container flow. """ if self._is_structure_tool_workflow(execution_service): - self._execute_structure_tool_workflow(execution_service, file_name) + self._execute_structure_tool_workflow( + execution_service, file_name, auto_enable_highlight + ) else: # Original Docker-based flow (unchanged) execution_service.build_workflow() @@ -1015,7 +1026,10 @@ def _is_structure_tool_workflow( return False def _execute_structure_tool_workflow( - self, execution_service: WorkflowExecutionService, file_name: str + self, + execution_service: WorkflowExecutionService, + file_name: str, + auto_enable_highlight: bool = False, ) -> None: """Execute structure tool as a task instead of a Docker container. @@ -1030,6 +1044,12 @@ def _execute_structure_tool_workflow( tool_instance = execution_service.tool_instances[0] file_handler = execution_service.file_handler + tool_instance_metadata = tool_instance.metadata + if auto_enable_highlight: + # HITL confidence rules need highlight data. Override a copy so the + # user's saved setting is unchanged. + tool_instance_metadata = {**tool_instance_metadata, "enable_highlight": True} + # Read metadata from METADATA.json for file_hash and exec_metadata metadata = {} try: @@ -1047,7 +1067,7 @@ def _execute_structure_tool_workflow( "workflow_id": execution_service.workflow_id, "execution_id": execution_service.execution_id, "file_execution_id": execution_service.file_execution_id, - "tool_instance_metadata": tool_instance.metadata, + "tool_instance_metadata": tool_instance_metadata, "platform_service_api_key": platform_api_key, "input_file_path": str(file_handler.infile), "output_dir_path": str(file_handler.execution_dir), diff --git a/workers/tests/test_structure_tool_pipeline.py b/workers/tests/test_structure_tool_pipeline.py index 4c2812bf66..601632f89c 100644 --- a/workers/tests/test_structure_tool_pipeline.py +++ b/workers/tests/test_structure_tool_pipeline.py @@ -907,7 +907,7 @@ def test_structure_tool_params_passthrough(self, mock_is_struct, mock_exec_struc service._build_and_execute_workflow(mock_exec_service, "test.pdf") # Verify _execute_structure_tool_workflow was called - mock_exec_struct.assert_called_once_with(mock_exec_service, "test.pdf") + mock_exec_struct.assert_called_once_with(mock_exec_service, "test.pdf", False) @patch( "shared.workflow.execution.service.WorkerWorkflowExecutionService." @@ -963,6 +963,50 @@ def test_source_file_name_uses_real_filename_not_sentinel( assert params["source_file_name"] == "invoice.pdf" assert params["source_file_name"] != "SOURCE" + @pytest.mark.parametrize( + "user_highlight, auto_enable, expected", + [ + (False, True, True), # HITL confidence rule present + (True, False, True), # user setting respected + (False, False, False), + ], + ) + @patch( + "shared.workflow.execution.service.WorkerWorkflowExecutionService." + "_get_platform_service_api_key", + return_value="sk-test", + ) + @patch("file_processing.structure_tool_task.execute_structure_tool") + def test_auto_enable_highlight_overrides_a_copy( + self, mock_execute_struct, mock_get_key, user_highlight, auto_enable, expected + ): + """HITL confidence rules turn highlight on for the run only. + + Word confidence and the saved tool instance metadata stay as the + user set them. + """ + from shared.workflow.execution.service import ( + WorkerWorkflowExecutionService, + ) + + saved_metadata = { + "enable_highlight": user_highlight, + "enable_word_confidence": False, + } + mock_exec_service = MagicMock() + mock_exec_service.tool_instances = [MagicMock(metadata=saved_metadata)] + mock_exec_service.file_handler.get_workflow_metadata.return_value = {} + mock_execute_struct.return_value = {"success": True} + + WorkerWorkflowExecutionService()._execute_structure_tool_workflow( + mock_exec_service, "invoice.pdf", auto_enable_highlight=auto_enable + ) + + sent = mock_execute_struct.call_args[0][0]["tool_instance_metadata"] + assert sent["enable_highlight"] is expected + assert sent["enable_word_confidence"] is False + assert saved_metadata["enable_highlight"] is user_highlight + class TestHelperFunctions: """Test standalone helper functions.""" From 5cfbe0661face00ea002543ceea902582ce1431f Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 22:17:44 +0530 Subject: [PATCH 02/10] UN-4435 fix: guard has_confidence_rules for older plugins and test the flag path Self-review round 1: - Call has_confidence_rules through a getattr-guarded helper so an older manual_review plugin without the method does not fail every execution. - Document the accepted DB+API rule scan and the agentic no-confidence case. - Test the flag from manual_review_config to the structure tool call, the helper, and both orchestrators with rules present and absent. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 7 +- workers/general/tasks.py | 3 +- workers/shared/utils/manual_review_factory.py | 14 ++- workers/shared/workflow/execution/service.py | 3 +- workers/tests/test_barrier.py | 119 ++++++++++++++++++ workers/tests/test_structure_tool_pipeline.py | 65 +++++++++- 6 files changed, 200 insertions(+), 11 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index 4a271c16b8..d625620afc 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -26,6 +26,7 @@ AUTO_ENABLE_HIGHLIGHT, HIGHLIGHT_AUTO_ENABLED_MESSAGE, get_manual_review_workflow_util, + has_confidence_rules, ) from shared.workflow.execution import WorkerExecutionContext, WorkflowOrchestrationUtils from shared.workflow.execution.tool_validation import validate_workflow_tool_instances @@ -670,9 +671,9 @@ def _run_workflow_api( total_files=total_files, ) - auto_enable_highlight = get_manual_review_workflow_util( - api_client, schema_name - ).has_confidence_rules(workflow_id) + auto_enable_highlight = has_confidence_rules( + get_manual_review_workflow_util(api_client, schema_name), workflow_id + ) if auto_enable_highlight: _log_api_statistics_to_ui( execution_id=execution_id, diff --git a/workers/general/tasks.py b/workers/general/tasks.py index 73d9e141b6..5bb433bdd9 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -46,6 +46,7 @@ AUTO_ENABLE_HIGHLIGHT, HIGHLIGHT_AUTO_ENABLED_MESSAGE, get_manual_review_service, + has_confidence_rules, ) from shared.workflow.connectors.source import WorkerSourceConnector from shared.workflow.execution import ( @@ -982,7 +983,7 @@ def _orchestrate_file_processing_general( total_files=len(source_files), ) logger.debug("Global file data configured for manual review") - if workflow_util.has_confidence_rules(workflow_id): + if has_confidence_rules(workflow_util, workflow_id): global_file_data.manual_review_config[AUTO_ENABLE_HIGHLIGHT] = True _log_batch_statistics_to_ui( execution_id=execution_id, diff --git a/workers/shared/utils/manual_review_factory.py b/workers/shared/utils/manual_review_factory.py index e3f8c51dd8..7bd5371bac 100644 --- a/workers/shared/utils/manual_review_factory.py +++ b/workers/shared/utils/manual_review_factory.py @@ -390,6 +390,17 @@ def get_manual_review_workflow_util(api_client: Any, organization_id: str) -> An return _create_null_workflow_util() +def has_confidence_rules(workflow_util: Any, workflow_id: str) -> bool: + """Whether the workflow's HITL rules contain a confidence condition. + + False in OSS, and with an older plugin that lacks the method. The plugin + checks both DB and API rules whichever flow runs; a leftover rule of the + other type can turn highlight on needlessly. Accepted to keep one check. + """ + check = getattr(workflow_util, "has_confidence_rules", None) + return bool(check and check(workflow_id)) + + def has_manual_review_plugin(): return has_client_plugin(MANUAL_REVIEW_PLUGIN_NAME) @@ -466,9 +477,6 @@ def get_hitl_ttl_seconds(self, *args, **kwargs): def get_manual_review_config(self, workflow_id, total_files): return {} - def has_confidence_rules(self, workflow_id): - return False - return WorkflowUtilNull() diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py index c29f7422b1..d7adb43cc1 100644 --- a/workers/shared/workflow/execution/service.py +++ b/workers/shared/workflow/execution/service.py @@ -1047,7 +1047,8 @@ def _execute_structure_tool_workflow( tool_instance_metadata = tool_instance.metadata if auto_enable_highlight: # HITL confidence rules need highlight data. Override a copy so the - # user's saved setting is unchanged. + # user's saved setting is unchanged. Agentic projects produce no + # confidence, so for them this only adds source refs and cost. tool_instance_metadata = {**tool_instance_metadata, "enable_highlight": True} # Read metadata from METADATA.json for file_hash and exec_metadata diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py index 7bf65d1825..8516c0c39e 100644 --- a/workers/tests/test_barrier.py +++ b/workers/tests/test_barrier.py @@ -773,5 +773,124 @@ def test_manual_review_required_invokes_decision_helper_per_batch( ) +class TestAutoEnableHighlightOrchestration: + """HITL confidence rules set ``auto_enable_highlight`` (UN-4435). + + The orchestrators check the rules once and put the flag in + ``manual_review_config``, which the file workers read. + """ + + @pytest.mark.parametrize("has_rules", [True, False]) + def test_api_deployment_sets_flag_on_every_batch(self, monkeypatch, has_rules): + from shared.utils.manual_review_factory import ( + AUTO_ENABLE_HIGHLIGHT, + HIGHLIGHT_AUTO_ENABLED_MESSAGE, + ) + + mocks = _setup_workflow_api_mocks(monkeypatch, num_batches=2) + util = MagicMock(name="workflow_util") + util.has_confidence_rules.return_value = has_rules + monkeypatch.setattr( + mocks.api_tasks, "get_manual_review_workflow_util", lambda *a: util + ) + configs = [] + monkeypatch.setattr( + mocks.api_tasks, + "_create_file_data", + lambda **kwargs: MagicMock(manual_review_config={}), + ) + monkeypatch.setattr( + mocks.api_tasks, + "_create_batch_data", + lambda **kwargs: configs.append( + dict(kwargs["file_data"].manual_review_config) + ), + ) + ui_log = MagicMock(name="_log_api_statistics_to_ui") + monkeypatch.setattr(mocks.api_tasks, "_log_api_statistics_to_ui", ui_log) + + mocks.api_tasks._run_workflow_api( + api_client=mocks.api_client, + schema_name="org_test", + workflow_id="wf-1", + execution_id="exec-1", + hash_values_of_files={"f1": MagicMock(name="file_1")}, + scheduled=False, + execution_mode=None, + pipeline_id="pipe-1", + use_file_history=False, + task_id="task-1", + ) + + util.has_confidence_rules.assert_called_once_with("wf-1") + assert len(configs) == 2 + assert all(c.get(AUTO_ENABLE_HIGHLIGHT, False) is has_rules for c in configs) + messages = [c.kwargs.get("message") for c in ui_log.call_args_list] + assert (HIGHLIGHT_AUTO_ENABLED_MESSAGE in messages) is has_rules + + @pytest.mark.parametrize("has_rules", [True, False]) + def test_general_sets_flag_on_global_file_data(self, monkeypatch, has_rules): + from general import tasks as general_tasks + from shared.utils.manual_review_factory import ( + AUTO_ENABLE_HIGHLIGHT, + HIGHLIGHT_AUTO_ENABLED_MESSAGE, + ) + + api_client = MagicMock(name="api_client") + util = MagicMock(name="workflow_util") + util.has_confidence_rules.return_value = has_rules + util.create_workflow_file_data_with_manual_review.return_value = MagicMock( + manual_review_config={} + ) + service = MagicMock(name="manual_review_service") + service.get_workflow_util.return_value = util + monkeypatch.setattr( + general_tasks, "get_manual_review_service", lambda **kwargs: service + ) + monkeypatch.setattr( + general_tasks, + "_get_file_batches_general", + lambda **kwargs: [MagicMock(name="batch_1")], + ) + configs = [] + monkeypatch.setattr( + general_tasks, + "_create_batch_data_general", + lambda **kwargs: configs.append( + dict(kwargs["file_data"].manual_review_config) + ), + ) + ui_log = MagicMock(name="_log_batch_statistics_to_ui") + monkeypatch.setattr(general_tasks, "_log_batch_statistics_to_ui", ui_log) + monkeypatch.setattr( + general_tasks.WorkflowOrchestrationUtils, + "create_chord_execution", + MagicMock(return_value=MagicMock(id="chord-id")), + ) + + try: + general_tasks._orchestrate_file_processing_general( + api_client=api_client, + workflow_id="wf-1", + execution_id="exec-1", + source_files={"f1": MagicMock(name="file_1")}, + pipeline_id="pipe-1", + scheduled=False, + execution_mode=None, + use_file_history=False, + organization_id="org_test", + ) + except Exception: + # Post-chord status updates hit the mocked client; the flag is + # already captured by then. + if not configs: + raise + + util.has_confidence_rules.assert_called_once_with("wf-1") + assert configs[0].get(AUTO_ENABLE_HIGHLIGHT, False) is has_rules + messages = [c.kwargs.get("message") for c in ui_log.call_args_list] + assert (HIGHLIGHT_AUTO_ENABLED_MESSAGE in messages) is has_rules + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/workers/tests/test_structure_tool_pipeline.py b/workers/tests/test_structure_tool_pipeline.py index 601632f89c..00e021d4df 100644 --- a/workers/tests/test_structure_tool_pipeline.py +++ b/workers/tests/test_structure_tool_pipeline.py @@ -894,7 +894,10 @@ class TestStructureToolParamsPassthrough: "_is_structure_tool_workflow", return_value=True, ) - def test_structure_tool_params_passthrough(self, mock_is_struct, mock_exec_struct): + @pytest.mark.parametrize("auto_enable", [True, False]) + def test_structure_tool_params_passthrough( + self, mock_is_struct, mock_exec_struct, auto_enable + ): from shared.workflow.execution.service import ( WorkerWorkflowExecutionService, ) @@ -904,10 +907,66 @@ def test_structure_tool_params_passthrough(self, mock_is_struct, mock_exec_struc mock_exec_service = MagicMock() mock_exec_service.tool_instances = [MagicMock()] - service._build_and_execute_workflow(mock_exec_service, "test.pdf") + service._build_and_execute_workflow( + mock_exec_service, "test.pdf", auto_enable_highlight=auto_enable + ) # Verify _execute_structure_tool_workflow was called - mock_exec_struct.assert_called_once_with(mock_exec_service, "test.pdf", False) + mock_exec_struct.assert_called_once_with( + mock_exec_service, "test.pdf", auto_enable + ) + + @pytest.mark.parametrize( + "review_config, expected", + [({"auto_enable_highlight": True}, True), ({}, False)], + ) + @patch( + "shared.workflow.execution.service.WorkerWorkflowExecutionService." + "_build_and_execute_workflow" + ) + @patch( + "shared.workflow.execution.service.WorkerWorkflowExecutionService." + "_prepare_workflow_input_file", + return_value=None, + ) + @patch( + "shared.workflow.execution.service.WorkerWorkflowExecutionService." + "_compile_workflow", + return_value=True, + ) + def test_auto_enable_highlight_read_from_review_config( + self, mock_compile, mock_prepare, mock_build, review_config, expected + ): + from shared.workflow.execution.service import ( + WorkerWorkflowExecutionService, + ) + + context = MagicMock() + context.file_data.manual_review_config = review_config + + assert WorkerWorkflowExecutionService()._execute_workflow_with_service( + execution_service=MagicMock(), + file_processing_context=context, + file_name="invoice.pdf", + workflow_file_execution_id="", + execution_id="exec-1", + workflow_id="wf-1", + ) + assert mock_build.call_args.kwargs["auto_enable_highlight"] is expected + + @pytest.mark.parametrize( + "util, expected", + [ + (MagicMock(has_confidence_rules=MagicMock(return_value=True)), True), + (MagicMock(has_confidence_rules=MagicMock(return_value=False)), False), + # Older plugin without the method + (object(), False), + ], + ) + def test_has_confidence_rules_helper(self, util, expected): + from shared.utils.manual_review_factory import has_confidence_rules + + assert has_confidence_rules(util, "wf-1") is expected @patch( "shared.workflow.execution.service.WorkerWorkflowExecutionService." From a5c76fe4d50cf6bda1c2154ccc9b78d8696c892e Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 22:24:57 +0530 Subject: [PATCH 03/10] UN-4435 test: run the general orchestrator test to completion The batch-data stub returned None, so the orchestrator raised at batch_data.files and a try/except with a wrong comment hid it. The stub now returns a batch with no files and the try/except is gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/tests/test_barrier.py | 39 +++++++++++++++-------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py index 8516c0c39e..3153febe9c 100644 --- a/workers/tests/test_barrier.py +++ b/workers/tests/test_barrier.py @@ -853,12 +853,13 @@ def test_general_sets_flag_on_global_file_data(self, monkeypatch, has_rules): lambda **kwargs: [MagicMock(name="batch_1")], ) configs = [] + + def create_batch_data(**kwargs): + configs.append(dict(kwargs["file_data"].manual_review_config)) + return MagicMock(files=[]) + monkeypatch.setattr( - general_tasks, - "_create_batch_data_general", - lambda **kwargs: configs.append( - dict(kwargs["file_data"].manual_review_config) - ), + general_tasks, "_create_batch_data_general", create_batch_data ) ui_log = MagicMock(name="_log_batch_statistics_to_ui") monkeypatch.setattr(general_tasks, "_log_batch_statistics_to_ui", ui_log) @@ -868,23 +869,17 @@ def test_general_sets_flag_on_global_file_data(self, monkeypatch, has_rules): MagicMock(return_value=MagicMock(id="chord-id")), ) - try: - general_tasks._orchestrate_file_processing_general( - api_client=api_client, - workflow_id="wf-1", - execution_id="exec-1", - source_files={"f1": MagicMock(name="file_1")}, - pipeline_id="pipe-1", - scheduled=False, - execution_mode=None, - use_file_history=False, - organization_id="org_test", - ) - except Exception: - # Post-chord status updates hit the mocked client; the flag is - # already captured by then. - if not configs: - raise + general_tasks._orchestrate_file_processing_general( + api_client=api_client, + workflow_id="wf-1", + execution_id="exec-1", + source_files={"f1": MagicMock(name="file_1")}, + pipeline_id="pipe-1", + scheduled=False, + execution_mode=None, + use_file_history=False, + organization_id="org_test", + ) util.has_confidence_rules.assert_called_once_with("wf-1") assert configs[0].get(AUTO_ENABLE_HIGHLIGHT, False) is has_rules From b829d3c961c7781f340db491728b0440bd3dc5fa Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 23:17:52 +0530 Subject: [PATCH 04/10] UN-4435 fix: check each flow's own HITL rules; skip agentic - ETL/task runs check the DB rules only; API deployments check the API rules only. These are the rules each flow applies. - The override moves from the workflow service into structure_tool_task, after the agentic branch. Agentic runs keep the user's own setting: workflow runs give them no highlight data, so confidence cannot work. - The "highlight enabled automatically" message is now logged per file, only when the override applies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 10 +- .../file_processing/structure_tool_task.py | 11 ++ workers/general/tasks.py | 9 +- workers/shared/utils/manual_review_factory.py | 11 +- workers/shared/workflow/execution/service.py | 6 +- workers/tests/test_barrier.py | 55 ++++--- workers/tests/test_structure_tool_pipeline.py | 152 +++++++++++++++--- 7 files changed, 177 insertions(+), 77 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index d625620afc..d04e7d1f21 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -24,7 +24,6 @@ from shared.processing.files import FileProcessingUtils from shared.utils.manual_review_factory import ( AUTO_ENABLE_HIGHLIGHT, - HIGHLIGHT_AUTO_ENABLED_MESSAGE, get_manual_review_workflow_util, has_confidence_rules, ) @@ -672,15 +671,8 @@ def _run_workflow_api( ) auto_enable_highlight = has_confidence_rules( - get_manual_review_workflow_util(api_client, schema_name), workflow_id + get_manual_review_workflow_util(api_client, schema_name), workflow_id, "API" ) - if auto_enable_highlight: - _log_api_statistics_to_ui( - execution_id=execution_id, - organization_id=schema_name, - pipeline_id=pipeline_id, - message=HIGHLIGHT_AUTO_ENABLED_MESSAGE, - ) # Create batch tasks following the exact Django pattern batch_tasks = [] diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py index 36c875b7e6..34f7b1ec63 100644 --- a/workers/file_processing/structure_tool_task.py +++ b/workers/file_processing/structure_tool_task.py @@ -28,6 +28,10 @@ ) from shared.enums.task_enums import TaskName from shared.infrastructure.context import StateStore +from shared.utils.manual_review_factory import ( + AUTO_ENABLE_HIGHLIGHT, + HIGHLIGHT_AUTO_ENABLED_MESSAGE, +) from unstract.sdk1.constants import ToolEnv, UsageKwargs from unstract.sdk1.execution.context import ExecutionContext @@ -365,6 +369,13 @@ def _execute_structure_tool_impl(params: dict) -> dict: execution_data_dir=execution_data_dir, ) + # HITL confidence rules need highlight data. Override a copy so the + # user's saved setting is unchanged. Agentic projects are skipped above: + # workflow runs give them no highlight data, so confidence can't work. + if params.get(AUTO_ENABLE_HIGHLIGHT): + tool_instance_metadata = {**tool_instance_metadata, _SK.ENABLE_HIGHLIGHT: True} + shim.stream_log(HIGHLIGHT_AUTO_ENABLED_MESSAGE) + # ---- Step 3: Profile overrides ---- _handle_profile_overrides(exec_metadata, platform_helper, tool_metadata) diff --git a/workers/general/tasks.py b/workers/general/tasks.py index 5bb433bdd9..ddcd44fc2e 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -44,7 +44,6 @@ from shared.processing.types import FileDataValidator, TypeConverter from shared.utils.manual_review_factory import ( AUTO_ENABLE_HIGHLIGHT, - HIGHLIGHT_AUTO_ENABLED_MESSAGE, get_manual_review_service, has_confidence_rules, ) @@ -983,14 +982,8 @@ def _orchestrate_file_processing_general( total_files=len(source_files), ) logger.debug("Global file data configured for manual review") - if has_confidence_rules(workflow_util, workflow_id): + if has_confidence_rules(workflow_util, workflow_id, "DB"): global_file_data.manual_review_config[AUTO_ENABLE_HIGHLIGHT] = True - _log_batch_statistics_to_ui( - execution_id=execution_id, - organization_id=organization_id, - pipeline_id=pipeline_id, - message=HIGHLIGHT_AUTO_ENABLED_MESSAGE, - ) # Pre-calculate file decisions for ALL files based on total count - not per batch! q_file_no_list = global_file_data.manual_review_config.get("q_file_no_list", []) logger.info( diff --git a/workers/shared/utils/manual_review_factory.py b/workers/shared/utils/manual_review_factory.py index 7bd5371bac..c53efb3ee0 100644 --- a/workers/shared/utils/manual_review_factory.py +++ b/workers/shared/utils/manual_review_factory.py @@ -390,15 +390,14 @@ def get_manual_review_workflow_util(api_client: Any, organization_id: str) -> An return _create_null_workflow_util() -def has_confidence_rules(workflow_util: Any, workflow_id: str) -> bool: - """Whether the workflow's HITL rules contain a confidence condition. +def has_confidence_rules(workflow_util: Any, workflow_id: str, rule_type: str) -> bool: + """Whether the workflow's HITL rules of ``rule_type`` ("DB" or "API") + contain a confidence condition. - False in OSS, and with an older plugin that lacks the method. The plugin - checks both DB and API rules whichever flow runs; a leftover rule of the - other type can turn highlight on needlessly. Accepted to keep one check. + False in OSS, and with a plugin that lacks the method. """ check = getattr(workflow_util, "has_confidence_rules", None) - return bool(check and check(workflow_id)) + return bool(check and check(workflow_id, rule_type)) def has_manual_review_plugin(): diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py index d7adb43cc1..e39af89f4b 100644 --- a/workers/shared/workflow/execution/service.py +++ b/workers/shared/workflow/execution/service.py @@ -1045,11 +1045,6 @@ def _execute_structure_tool_workflow( file_handler = execution_service.file_handler tool_instance_metadata = tool_instance.metadata - if auto_enable_highlight: - # HITL confidence rules need highlight data. Override a copy so the - # user's saved setting is unchanged. Agentic projects produce no - # confidence, so for them this only adds source refs and cost. - tool_instance_metadata = {**tool_instance_metadata, "enable_highlight": True} # Read metadata from METADATA.json for file_hash and exec_metadata metadata = {} @@ -1082,6 +1077,7 @@ def _execute_structure_tool_workflow( "messaging_channel": getattr(execution_service, "messaging_channel", ""), "file_hash": metadata.get("source_hash", ""), "exec_metadata": metadata, + AUTO_ENABLE_HIGHLIGHT: auto_enable_highlight, } # Call synchronously (same process, in-band) diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py index 3153febe9c..52f221308d 100644 --- a/workers/tests/test_barrier.py +++ b/workers/tests/test_barrier.py @@ -776,20 +776,30 @@ def test_manual_review_required_invokes_decision_helper_per_batch( class TestAutoEnableHighlightOrchestration: """HITL confidence rules set ``auto_enable_highlight`` (UN-4435). - The orchestrators check the rules once and put the flag in + The orchestrators check their own flow's rules once (API deployments the + API rules, the rest the DB rules) and put the flag in ``manual_review_config``, which the file workers read. """ - @pytest.mark.parametrize("has_rules", [True, False]) - def test_api_deployment_sets_flag_on_every_batch(self, monkeypatch, has_rules): - from shared.utils.manual_review_factory import ( - AUTO_ENABLE_HIGHLIGHT, - HIGHLIGHT_AUTO_ENABLED_MESSAGE, + @staticmethod + def _util(confidence_rule_type): + """A workflow util with a confidence rule of one type, or none.""" + util = MagicMock(name="workflow_util") + util.has_confidence_rules.side_effect = ( + lambda workflow_id, rule_type: rule_type == confidence_rule_type ) + return util + + @pytest.mark.parametrize( + "confidence_rule_type, has_rules", [("API", True), ("DB", False), (None, False)] + ) + def test_api_deployment_sets_flag_on_every_batch( + self, monkeypatch, confidence_rule_type, has_rules + ): + from shared.utils.manual_review_factory import AUTO_ENABLE_HIGHLIGHT mocks = _setup_workflow_api_mocks(monkeypatch, num_batches=2) - util = MagicMock(name="workflow_util") - util.has_confidence_rules.return_value = has_rules + util = self._util(confidence_rule_type) monkeypatch.setattr( mocks.api_tasks, "get_manual_review_workflow_util", lambda *a: util ) @@ -806,8 +816,6 @@ def test_api_deployment_sets_flag_on_every_batch(self, monkeypatch, has_rules): dict(kwargs["file_data"].manual_review_config) ), ) - ui_log = MagicMock(name="_log_api_statistics_to_ui") - monkeypatch.setattr(mocks.api_tasks, "_log_api_statistics_to_ui", ui_log) mocks.api_tasks._run_workflow_api( api_client=mocks.api_client, @@ -822,23 +830,21 @@ def test_api_deployment_sets_flag_on_every_batch(self, monkeypatch, has_rules): task_id="task-1", ) - util.has_confidence_rules.assert_called_once_with("wf-1") + util.has_confidence_rules.assert_called_once_with("wf-1", "API") assert len(configs) == 2 assert all(c.get(AUTO_ENABLE_HIGHLIGHT, False) is has_rules for c in configs) - messages = [c.kwargs.get("message") for c in ui_log.call_args_list] - assert (HIGHLIGHT_AUTO_ENABLED_MESSAGE in messages) is has_rules - @pytest.mark.parametrize("has_rules", [True, False]) - def test_general_sets_flag_on_global_file_data(self, monkeypatch, has_rules): + @pytest.mark.parametrize( + "confidence_rule_type, has_rules", [("DB", True), ("API", False), (None, False)] + ) + def test_general_sets_flag_on_global_file_data( + self, monkeypatch, confidence_rule_type, has_rules + ): from general import tasks as general_tasks - from shared.utils.manual_review_factory import ( - AUTO_ENABLE_HIGHLIGHT, - HIGHLIGHT_AUTO_ENABLED_MESSAGE, - ) + from shared.utils.manual_review_factory import AUTO_ENABLE_HIGHLIGHT api_client = MagicMock(name="api_client") - util = MagicMock(name="workflow_util") - util.has_confidence_rules.return_value = has_rules + util = self._util(confidence_rule_type) util.create_workflow_file_data_with_manual_review.return_value = MagicMock( manual_review_config={} ) @@ -861,8 +867,7 @@ def create_batch_data(**kwargs): monkeypatch.setattr( general_tasks, "_create_batch_data_general", create_batch_data ) - ui_log = MagicMock(name="_log_batch_statistics_to_ui") - monkeypatch.setattr(general_tasks, "_log_batch_statistics_to_ui", ui_log) + monkeypatch.setattr(general_tasks, "_log_batch_statistics_to_ui", MagicMock()) monkeypatch.setattr( general_tasks.WorkflowOrchestrationUtils, "create_chord_execution", @@ -881,10 +886,8 @@ def create_batch_data(**kwargs): organization_id="org_test", ) - util.has_confidence_rules.assert_called_once_with("wf-1") + util.has_confidence_rules.assert_called_once_with("wf-1", "DB") assert configs[0].get(AUTO_ENABLE_HIGHLIGHT, False) is has_rules - messages = [c.kwargs.get("message") for c in ui_log.call_args_list] - assert (HIGHLIGHT_AUTO_ENABLED_MESSAGE in messages) is has_rules if __name__ == "__main__": diff --git a/workers/tests/test_structure_tool_pipeline.py b/workers/tests/test_structure_tool_pipeline.py index 00e021d4df..04824bb354 100644 --- a/workers/tests/test_structure_tool_pipeline.py +++ b/workers/tests/test_structure_tool_pipeline.py @@ -966,7 +966,9 @@ def test_auto_enable_highlight_read_from_review_config( def test_has_confidence_rules_helper(self, util, expected): from shared.utils.manual_review_factory import has_confidence_rules - assert has_confidence_rules(util, "wf-1") is expected + assert has_confidence_rules(util, "wf-1", "DB") is expected + if isinstance(util, MagicMock): + util.has_confidence_rules.assert_called_once_with("wf-1", "DB") @patch( "shared.workflow.execution.service.WorkerWorkflowExecutionService." @@ -1022,36 +1024,22 @@ def test_source_file_name_uses_real_filename_not_sentinel( assert params["source_file_name"] == "invoice.pdf" assert params["source_file_name"] != "SOURCE" - @pytest.mark.parametrize( - "user_highlight, auto_enable, expected", - [ - (False, True, True), # HITL confidence rule present - (True, False, True), # user setting respected - (False, False, False), - ], - ) + @pytest.mark.parametrize("auto_enable", [True, False]) @patch( "shared.workflow.execution.service.WorkerWorkflowExecutionService." "_get_platform_service_api_key", return_value="sk-test", ) @patch("file_processing.structure_tool_task.execute_structure_tool") - def test_auto_enable_highlight_overrides_a_copy( - self, mock_execute_struct, mock_get_key, user_highlight, auto_enable, expected + def test_auto_enable_highlight_passed_in_params( + self, mock_execute_struct, mock_get_key, auto_enable ): - """HITL confidence rules turn highlight on for the run only. - - Word confidence and the saved tool instance metadata stay as the - user set them. - """ + """The service passes the flag on and leaves the metadata alone.""" from shared.workflow.execution.service import ( WorkerWorkflowExecutionService, ) - saved_metadata = { - "enable_highlight": user_highlight, - "enable_word_confidence": False, - } + saved_metadata = {"enable_highlight": False} mock_exec_service = MagicMock() mock_exec_service.tool_instances = [MagicMock(metadata=saved_metadata)] mock_exec_service.file_handler.get_workflow_metadata.return_value = {} @@ -1061,10 +1049,128 @@ def test_auto_enable_highlight_overrides_a_copy( mock_exec_service, "invoice.pdf", auto_enable_highlight=auto_enable ) - sent = mock_execute_struct.call_args[0][0]["tool_instance_metadata"] - assert sent["enable_highlight"] is expected - assert sent["enable_word_confidence"] is False + params = mock_execute_struct.call_args[0][0] + assert params["auto_enable_highlight"] is auto_enable + assert params["tool_instance_metadata"] == {"enable_highlight": False} + + +class TestAutoEnableHighlight: + """``auto_enable_highlight`` turns highlight on for the run (UN-4435).""" + + @pytest.mark.parametrize( + "user_highlight, auto_enable, expected", + [ + (False, True, True), # HITL confidence rule present + (True, False, True), # user setting respected + (False, False, False), + ], + ) + @patch(_PATCH_SHIM) + @patch(_PATCH_FILE_STORAGE) + @patch(_PATCH_PLATFORM_HELPER) + @patch(_PATCH_DISPATCHER) + def test_regular_overrides_a_copy( + self, + mock_get_executor_dispatcher, + mock_create_ph, + mock_get_fs, + mock_shim_cls, + base_params, + tool_metadata_regular, + mock_fs, + mock_platform_helper, + user_highlight, + auto_enable, + expected, + ): + """Word confidence and the saved metadata stay as the user set them.""" + from file_processing.structure_tool_task import ( + _execute_structure_tool_impl as execute_structure_tool, + ) + from shared.utils.manual_review_factory import ( + HIGHLIGHT_AUTO_ENABLED_MESSAGE, + ) + + mock_get_fs.return_value = mock_fs + mock_create_ph.return_value = mock_platform_helper + mock_platform_helper.get_prompt_studio_tool.return_value = { + "tool_metadata": tool_metadata_regular, + } + dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True) + mock_get_executor_dispatcher.return_value = dispatcher_instance + dispatcher_instance.dispatch.return_value = _make_pipeline_result() + + saved_metadata = base_params["tool_instance_metadata"] + saved_metadata["enable_highlight"] = user_highlight + saved_metadata["enable_word_confidence"] = False + base_params["auto_enable_highlight"] = auto_enable + + execute_structure_tool(base_params) + + ep = dispatcher_instance.dispatch.call_args[0][0].executor_params + assert ep["extract_params"]["enable_highlight"] is expected + assert ep["answer_params"]["tool_settings"]["enable_word_confidence"] is False assert saved_metadata["enable_highlight"] is user_highlight + messages = [ + c.args[0] for c in mock_shim_cls.return_value.stream_log.call_args_list + ] + assert (HIGHLIGHT_AUTO_ENABLED_MESSAGE in messages) is auto_enable + + @patch("unstract.sdk1.x2txt.X2Text") + @patch(_PATCH_SHIM) + @patch(_PATCH_FILE_STORAGE) + @patch(_PATCH_PLATFORM_HELPER) + @patch(_PATCH_DISPATCHER) + def test_agentic_keeps_user_setting( + self, + mock_get_executor_dispatcher, + mock_create_ph, + mock_get_fs, + mock_shim_cls, + mock_x2text_cls, + base_params, + mock_fs, + mock_platform_helper, + ): + """Agentic runs get no highlight data, so the flag is ignored.""" + from file_processing.structure_tool_task import ( + _execute_structure_tool_impl as execute_structure_tool, + ) + from shared.utils.manual_review_factory import ( + HIGHLIGHT_AUTO_ENABLED_MESSAGE, + ) + + mock_get_fs.return_value = mock_fs + mock_create_ph.return_value = mock_platform_helper + mock_platform_helper.get_prompt_studio_tool.return_value = None + mock_platform_helper.get_agentic_studio_tool.return_value = { + "tool_metadata": { + "name": "Agentic Project", + "project_id": "ap-001", + "json_schema": {"field": "string"}, + }, + } + dispatcher_instance = create_autospec(PgExecutionDispatcher, instance=True) + mock_get_executor_dispatcher.return_value = dispatcher_instance + dispatcher_instance.dispatch.return_value = ExecutionResult( + success=True, data={"output": {"field": "value"}} + ) + mock_x2text_instance = MagicMock() + mock_x2text_instance.process.return_value = MagicMock( + extracted_text="extracted text" + ) + mock_x2text_cls.return_value = mock_x2text_instance + + base_params["tool_instance_metadata"]["enable_highlight"] = False + base_params["auto_enable_highlight"] = True + + execute_structure_tool(base_params) + + assert mock_x2text_instance.process.call_args.kwargs["enable_highlight"] is False + messages = [ + c.args[0] for c in mock_shim_cls.return_value.stream_log.call_args_list + ] + assert HIGHLIGHT_AUTO_ENABLED_MESSAGE not in messages class TestHelperFunctions: From ad56d13e7e500b3a2a8f438d7a0ecda0d16d9bf3 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 23:28:10 +0530 Subject: [PATCH 05/10] UN-4435 fix: check API rules only for a hitl_queue_name request The destination evaluates API rules only when the request carries a hitl_queue_name, and a hitl_packet_id skips rule evaluation. Highlight on any other API call was cost nothing would read. Record in code the known limits: run-wide highlight, no confidence from non-LLMWhisperer V2 extractors or summarize-as-source, and the API + file-history path that evaluates DB rules. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 12 ++++++++-- .../file_processing/structure_tool_task.py | 3 +++ workers/general/tasks.py | 3 +++ workers/tests/test_barrier.py | 22 ++++++++++++++----- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index d04e7d1f21..eba7e7bf8f 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -670,8 +670,16 @@ def _run_workflow_api( total_files=total_files, ) - auto_enable_highlight = has_confidence_rules( - get_manual_review_workflow_util(api_client, schema_name), workflow_id, "API" + # The destination evaluates API rules only for a hitl_queue_name request; + # a hitl_packet_id skips rule evaluation, so highlight would go unused. + # ponytail: the API + file-history path evaluates DB rules, which this + # check does not cover; add a "DB" check there if that path needs it. + auto_enable_highlight = ( + bool(kwargs.get("hitl_queue_name")) + and not kwargs.get("hitl_packet_id") + and has_confidence_rules( + get_manual_review_workflow_util(api_client, schema_name), workflow_id, "API" + ) ) # Create batch tasks following the exact Django pattern diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py index 34f7b1ec63..ecc7987aca 100644 --- a/workers/file_processing/structure_tool_task.py +++ b/workers/file_processing/structure_tool_task.py @@ -372,6 +372,9 @@ def _execute_structure_tool_impl(params: dict) -> dict: # HITL confidence rules need highlight data. Override a copy so the # user's saved setting is unchanged. Agentic projects are skipped above: # workflow runs give them no highlight data, so confidence can't work. + # Known limit: with an extractor other than LLMWhisperer V2, no highlight + # plugin, or summarize-as-source, highlight still gives no confidence and + # rules still score 100; this line is logged anyway. if params.get(AUTO_ENABLE_HIGHLIGHT): tool_instance_metadata = {**tool_instance_metadata, _SK.ENABLE_HIGHLIGHT: True} shim.stream_log(HIGHLIGHT_AUTO_ENABLED_MESSAGE) diff --git a/workers/general/tasks.py b/workers/general/tasks.py index ddcd44fc2e..5de4613d5c 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -982,6 +982,9 @@ def _orchestrate_file_processing_general( total_files=len(source_files), ) logger.debug("Global file data configured for manual review") + # Run-wide: rules are evaluated after extraction, so every file needs + # confidence data. Each file gets the highlight prompt postamble and + # highlight/confidence metadata, including files not sent to review. if has_confidence_rules(workflow_util, workflow_id, "DB"): global_file_data.manual_review_config[AUTO_ENABLE_HIGHLIGHT] = True # Pre-calculate file decisions for ALL files based on total count - not per batch! diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py index 52f221308d..14816646ec 100644 --- a/workers/tests/test_barrier.py +++ b/workers/tests/test_barrier.py @@ -777,8 +777,8 @@ class TestAutoEnableHighlightOrchestration: """HITL confidence rules set ``auto_enable_highlight`` (UN-4435). The orchestrators check their own flow's rules once (API deployments the - API rules, the rest the DB rules) and put the flag in - ``manual_review_config``, which the file workers read. + API rules, only for a ``hitl_queue_name`` request; the rest the DB rules) + and put the flag in ``manual_review_config``, which the file workers read. """ @staticmethod @@ -791,10 +791,18 @@ def _util(confidence_rule_type): return util @pytest.mark.parametrize( - "confidence_rule_type, has_rules", [("API", True), ("DB", False), (None, False)] + "confidence_rule_type, hitl_kwargs, has_rules", + [ + ("API", {"hitl_queue_name": "q"}, True), + ("DB", {"hitl_queue_name": "q"}, False), + (None, {"hitl_queue_name": "q"}, False), + # The destination evaluates no API rules for these requests. + ("API", {}, False), + ("API", {"hitl_queue_name": "q", "hitl_packet_id": "p"}, False), + ], ) def test_api_deployment_sets_flag_on_every_batch( - self, monkeypatch, confidence_rule_type, has_rules + self, monkeypatch, confidence_rule_type, hitl_kwargs, has_rules ): from shared.utils.manual_review_factory import AUTO_ENABLE_HIGHLIGHT @@ -828,9 +836,13 @@ def test_api_deployment_sets_flag_on_every_batch( pipeline_id="pipe-1", use_file_history=False, task_id="task-1", + **hitl_kwargs, ) - util.has_confidence_rules.assert_called_once_with("wf-1", "API") + if hitl_kwargs.get("hitl_queue_name") and not hitl_kwargs.get("hitl_packet_id"): + util.has_confidence_rules.assert_called_once_with("wf-1", "API") + else: + util.has_confidence_rules.assert_not_called() assert len(configs) == 2 assert all(c.get(AUTO_ENABLE_HIGHLIGHT, False) is has_rules for c in configs) From b0e221841da5f411d8ea89255d4137b3e29abd02 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 28 Oct 2026 23:36:07 +0530 Subject: [PATCH 06/10] UN-4435 fix: tighten highlight comments; drop leftover local Narrow the run-wide comment to what holds for every file, name the hitl_packet_id exclusion in the test docstring, record that forced highlight adds metadata keys to include_metadata API responses, and pass tool_instance.metadata inline again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 2 ++ workers/general/tasks.py | 4 ++-- workers/shared/workflow/execution/service.py | 4 +--- workers/tests/test_barrier.py | 5 +++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index eba7e7bf8f..6f6f9da7ac 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -674,6 +674,8 @@ def _run_workflow_api( # a hitl_packet_id skips rule evaluation, so highlight would go unused. # ponytail: the API + file-history path evaluates DB rules, which this # check does not cover; add a "DB" check there if that path needs it. + # Highlight adds confidence_data, line_numbers and whisper_hash to the + # result metadata, which include_metadata callers receive. auto_enable_highlight = ( bool(kwargs.get("hitl_queue_name")) and not kwargs.get("hitl_packet_id") diff --git a/workers/general/tasks.py b/workers/general/tasks.py index 5de4613d5c..bf80de2b04 100644 --- a/workers/general/tasks.py +++ b/workers/general/tasks.py @@ -983,8 +983,8 @@ def _orchestrate_file_processing_general( ) logger.debug("Global file data configured for manual review") # Run-wide: rules are evaluated after extraction, so every file needs - # confidence data. Each file gets the highlight prompt postamble and - # highlight/confidence metadata, including files not sent to review. + # confidence data, including files not sent to review. Where highlight + # can't produce it, see the override in structure_tool_task.py. if has_confidence_rules(workflow_util, workflow_id, "DB"): global_file_data.manual_review_config[AUTO_ENABLE_HIGHLIGHT] = True # Pre-calculate file decisions for ALL files based on total count - not per batch! diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py index e39af89f4b..cf22199d72 100644 --- a/workers/shared/workflow/execution/service.py +++ b/workers/shared/workflow/execution/service.py @@ -1044,8 +1044,6 @@ def _execute_structure_tool_workflow( tool_instance = execution_service.tool_instances[0] file_handler = execution_service.file_handler - tool_instance_metadata = tool_instance.metadata - # Read metadata from METADATA.json for file_hash and exec_metadata metadata = {} try: @@ -1063,7 +1061,7 @@ def _execute_structure_tool_workflow( "workflow_id": execution_service.workflow_id, "execution_id": execution_service.execution_id, "file_execution_id": execution_service.file_execution_id, - "tool_instance_metadata": tool_instance_metadata, + "tool_instance_metadata": tool_instance.metadata, "platform_service_api_key": platform_api_key, "input_file_path": str(file_handler.infile), "output_dir_path": str(file_handler.execution_dir), diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py index 14816646ec..c40a62c3f1 100644 --- a/workers/tests/test_barrier.py +++ b/workers/tests/test_barrier.py @@ -777,8 +777,9 @@ class TestAutoEnableHighlightOrchestration: """HITL confidence rules set ``auto_enable_highlight`` (UN-4435). The orchestrators check their own flow's rules once (API deployments the - API rules, only for a ``hitl_queue_name`` request; the rest the DB rules) - and put the flag in ``manual_review_config``, which the file workers read. + API rules, only for a ``hitl_queue_name`` request without a + ``hitl_packet_id``; the rest the DB rules) and put the flag in + ``manual_review_config``, which the file workers read. """ @staticmethod From 49e7b43b3587164b02661c149b82375a82874e02 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 10:12:59 +0530 Subject: [PATCH 07/10] UN-4435 fix: document extraction change from forced highlight Correct the metadata-key wording and drop ticket IDs from test docstrings. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 5 +++-- workers/file_processing/structure_tool_task.py | 3 +++ workers/tests/test_barrier.py | 2 +- workers/tests/test_structure_tool_pipeline.py | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index 6f6f9da7ac..67a97257a8 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -674,8 +674,9 @@ def _run_workflow_api( # a hitl_packet_id skips rule evaluation, so highlight would go unused. # ponytail: the API + file-history path evaluates DB rules, which this # check does not cover; add a "DB" check there if that path needs it. - # Highlight adds confidence_data, line_numbers and whisper_hash to the - # result metadata, which include_metadata callers receive. + # Highlight adds confidence_data to the result metadata and fills + # highlight_data, line_numbers and whisper_hash, which include_metadata + # callers receive. auto_enable_highlight = ( bool(kwargs.get("hitl_queue_name")) and not kwargs.get("hitl_packet_id") diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py index ecc7987aca..2accb93488 100644 --- a/workers/file_processing/structure_tool_task.py +++ b/workers/file_processing/structure_tool_task.py @@ -375,6 +375,9 @@ def _execute_structure_tool_impl(params: dict) -> dict: # Known limit: with an extractor other than LLMWhisperer V2, no highlight # plugin, or summarize-as-source, highlight still gives no confidence and # rules still score 100; this line is logged anyway. + # Highlight also changes extraction: the prompt gets the line-number + # postamble and LLMWhisperer V2 adds line numbers, so answers may differ + # from a run with highlight off. if params.get(AUTO_ENABLE_HIGHLIGHT): tool_instance_metadata = {**tool_instance_metadata, _SK.ENABLE_HIGHLIGHT: True} shim.stream_log(HIGHLIGHT_AUTO_ENABLED_MESSAGE) diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py index c40a62c3f1..0a276134c4 100644 --- a/workers/tests/test_barrier.py +++ b/workers/tests/test_barrier.py @@ -774,7 +774,7 @@ def test_manual_review_required_invokes_decision_helper_per_batch( class TestAutoEnableHighlightOrchestration: - """HITL confidence rules set ``auto_enable_highlight`` (UN-4435). + """HITL confidence rules set ``auto_enable_highlight``. The orchestrators check their own flow's rules once (API deployments the API rules, only for a ``hitl_queue_name`` request without a diff --git a/workers/tests/test_structure_tool_pipeline.py b/workers/tests/test_structure_tool_pipeline.py index 04824bb354..80dcded85b 100644 --- a/workers/tests/test_structure_tool_pipeline.py +++ b/workers/tests/test_structure_tool_pipeline.py @@ -1055,7 +1055,7 @@ def test_auto_enable_highlight_passed_in_params( class TestAutoEnableHighlight: - """``auto_enable_highlight`` turns highlight on for the run (UN-4435).""" + """``auto_enable_highlight`` turns highlight on for the run.""" @pytest.mark.parametrize( "user_highlight, auto_enable, expected", From b5ee69bf2b9a946f5c6af101f91897e098121bb0 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 10:19:34 +0530 Subject: [PATCH 08/10] UN-4435 fix: correct highlight_data exposure note; mark pending decisions Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 5 +++-- workers/file_processing/structure_tool_task.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index 67a97257a8..4c2bd8c180 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -675,8 +675,9 @@ def _run_workflow_api( # ponytail: the API + file-history path evaluates DB rules, which this # check does not cover; add a "DB" check there if that path needs it. # Highlight adds confidence_data to the result metadata and fills - # highlight_data, line_numbers and whisper_hash, which include_metadata - # callers receive. + # line_numbers and whisper_hash, which include_metadata callers receive + # (highlight_data too, when the org's API highlight flag is on). Whether + # to strip these keys is pending a decision. auto_enable_highlight = ( bool(kwargs.get("hitl_queue_name")) and not kwargs.get("hitl_packet_id") diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py index 2accb93488..56239eb6c7 100644 --- a/workers/file_processing/structure_tool_task.py +++ b/workers/file_processing/structure_tool_task.py @@ -377,7 +377,8 @@ def _execute_structure_tool_impl(params: dict) -> dict: # rules still score 100; this line is logged anyway. # Highlight also changes extraction: the prompt gets the line-number # postamble and LLMWhisperer V2 adds line numbers, so answers may differ - # from a run with highlight off. + # from a run with highlight off. Getting confidence without the prompt + # postamble is pending a decision. if params.get(AUTO_ENABLE_HIGHLIGHT): tool_instance_metadata = {**tool_instance_metadata, _SK.ENABLE_HIGHLIGHT: True} shim.stream_log(HIGHLIGHT_AUTO_ENABLED_MESSAGE) From 8f6792159933a64bbe91c1f2752d13f1cd79b943 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 12:39:10 +0530 Subject: [PATCH 09/10] UN-4435 fix: record settled highlight decisions in comments Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 9 +++++---- workers/file_processing/structure_tool_task.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index 4c2bd8c180..ed54d8baa6 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -672,12 +672,13 @@ def _run_workflow_api( # The destination evaluates API rules only for a hitl_queue_name request; # a hitl_packet_id skips rule evaluation, so highlight would go unused. - # ponytail: the API + file-history path evaluates DB rules, which this - # check does not cover; add a "DB" check there if that path needs it. + # ponytail: the API + use_file_history path evaluates DB rules, which + # this check does not cover. use_file_history is internal-only; add a + # "DB" check there if it is ever exposed. # Highlight adds confidence_data to the result metadata and fills # line_numbers and whisper_hash, which include_metadata callers receive - # (highlight_data too, when the org's API highlight flag is on). Whether - # to strip these keys is pending a decision. + # (highlight_data too, when the org's API highlight flag is on). The keys + # are kept on purpose, so callers can see why a document was routed. auto_enable_highlight = ( bool(kwargs.get("hitl_queue_name")) and not kwargs.get("hitl_packet_id") diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py index 56239eb6c7..ef069c2a5b 100644 --- a/workers/file_processing/structure_tool_task.py +++ b/workers/file_processing/structure_tool_task.py @@ -377,8 +377,8 @@ def _execute_structure_tool_impl(params: dict) -> dict: # rules still score 100; this line is logged anyway. # Highlight also changes extraction: the prompt gets the line-number # postamble and LLMWhisperer V2 adds line numbers, so answers may differ - # from a run with highlight off. Getting confidence without the prompt - # postamble is pending a decision. + # from a run with highlight off. This is accepted: confidence needs the + # line numbers the postamble asks for. if params.get(AUTO_ENABLE_HIGHLIGHT): tool_instance_metadata = {**tool_instance_metadata, _SK.ENABLE_HIGHLIGHT: True} shim.stream_log(HIGHLIGHT_AUTO_ENABLED_MESSAGE) From a2a8ccdff17f54d1ed7b3d3e0708b181717d6495 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 29 Oct 2026 12:44:13 +0530 Subject: [PATCH 10/10] UN-4435 fix: describe use_file_history as undocumented, not internal Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017CeVWyH8qfPJXCthxkH3Jc --- workers/api-deployment/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py index ed54d8baa6..43afdbff27 100644 --- a/workers/api-deployment/tasks.py +++ b/workers/api-deployment/tasks.py @@ -673,8 +673,9 @@ def _run_workflow_api( # The destination evaluates API rules only for a hitl_queue_name request; # a hitl_packet_id skips rule evaluation, so highlight would go unused. # ponytail: the API + use_file_history path evaluates DB rules, which - # this check does not cover. use_file_history is internal-only; add a - # "DB" check there if it is ever exposed. + # this check does not cover, so a DB confidence rule there gets no + # highlight. use_file_history is hidden from the API schema and meant for + # demos, but a caller can still send it; add a "DB" check if it matters. # Highlight adds confidence_data to the result metadata and fills # line_numbers and whisper_hash, which include_metadata callers receive # (highlight_data too, when the org's API highlight flag is on). The keys