From 50fc1fa8a53d65b1f3e0790a0a43e466ccf04ec6 Mon Sep 17 00:00:00 2001 From: Milana Gurbanova Date: Tue, 14 Jul 2026 19:21:46 +0200 Subject: [PATCH 1/2] fix(langchain): normalize tool definitions to OpenAI function format Tool definitions from invocation_params["tools"] were forwarded into the generation input without format normalization. Setups that bypass bind_tools() (model.bind(tools=...), tools passed as invoke kwargs, or custom model integrations) deliver raw BaseTool instances, which were serialized as their attribute dict (args_schema, return_direct, ...) and not recognized as tool definitions by the Langfuse UI. The CallbackHandler now normalizes each tool definition before appending it: OpenAI-format dicts pass through unchanged, everything else is converted via langchain_core's convert_to_openai_tool with a fallback to the original value. Fixes #11850 --- langfuse/langchain/CallbackHandler.py | 9 +- langfuse/langchain/utils.py | 26 ++++++ tests/unit/test_langchain.py | 117 ++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 2 deletions(-) diff --git a/langfuse/langchain/CallbackHandler.py b/langfuse/langchain/CallbackHandler.py index 011370e20..43e93b385 100644 --- a/langfuse/langchain/CallbackHandler.py +++ b/langfuse/langchain/CallbackHandler.py @@ -33,7 +33,7 @@ LangfuseTool, ) from langfuse._utils import _get_timestamp -from langfuse.langchain.utils import _extract_model_name +from langfuse.langchain.utils import _extract_model_name, _normalize_tool_definition from langfuse.logger import langfuse_logger from langfuse.types import TraceContext @@ -1189,7 +1189,12 @@ def __on_llm_action( try: tools = kwargs.get("invocation_params", {}).get("tools", None) if tools and isinstance(tools, list): - prompts.extend([{"role": "tool", "content": tool} for tool in tools]) + prompts.extend( + [ + {"role": "tool", "content": _normalize_tool_definition(tool)} + for tool in tools + ] + ) model_name = self._parse_model_and_log_errors( serialized=serialized, metadata=metadata, kwargs=kwargs diff --git a/langfuse/langchain/utils.py b/langfuse/langchain/utils.py index dcf78c342..42ae3f67c 100644 --- a/langfuse/langchain/utils.py +++ b/langfuse/langchain/utils.py @@ -9,6 +9,32 @@ # - langchain_community is loaded as a dependency of langchain, but extremely unreliable. Decided to not depend on it. +def _normalize_tool_definition(tool: Any) -> Any: + """Best-effort normalization of a tool definition into the OpenAI function-tool format. + + LangChain integrations are expected to pass tool definitions through + `invocation_params["tools"]` in the OpenAI format + `{"type": "function", "function": {...}}`. Some code paths (custom chat + models, `model.bind(tools=...)` instead of `bind_tools`) forward raw + `BaseTool` instances instead, which would otherwise be serialized as their + attribute dict (`args_schema`, `return_direct`, ...) and not be recognized + as tools by the Langfuse UI. + """ + if ( + isinstance(tool, dict) + and tool.get("type") == "function" + and isinstance(tool.get("function"), dict) + ): + return tool + + try: + from langchain_core.utils.function_calling import convert_to_openai_tool + + return convert_to_openai_tool(tool) + except Exception: + return tool + + def _extract_model_name( serialized: Optional[Dict[str, Any]], **kwargs: Any, diff --git a/tests/unit/test_langchain.py b/tests/unit/test_langchain.py index 6f8fe458b..0e63fb6b1 100644 --- a/tests/unit/test_langchain.py +++ b/tests/unit/test_langchain.py @@ -9,11 +9,14 @@ from langchain_core.output_parsers import StrOutputParser from langchain_core.outputs import ChatGeneration, ChatResult, Generation, LLMResult from langchain_core.prompts import ChatPromptTemplate +from langchain_core.tools import StructuredTool from langchain_openai import ChatOpenAI, OpenAI from opentelemetry import context as otel_context +from pydantic import BaseModel, Field from langfuse._client.attributes import LangfuseOtelSpanAttributes from langfuse.langchain import CallbackHandler +from langfuse.langchain.utils import _normalize_tool_definition callback_handler_module = importlib.import_module("langfuse.langchain.CallbackHandler") @@ -940,3 +943,117 @@ def import_without_langgraph_command( ) importlib.reload(callback_handler_module) + + +class _ExampleToolArgs(BaseModel): + query: str = Field(description="What to search for") + + +def _make_example_tool() -> StructuredTool: + return StructuredTool.from_function( + func=lambda query: query, + name="example_tool", + description="Example tool description", + args_schema=_ExampleToolArgs, + ) + + +def _invoke_chat_model_with_tools(langfuse_memory_client, bind): + response = ChatResult( + generations=[ChatGeneration(message=AIMessage(content="ok"), text="ok")], + llm_output={"token_usage": {}, "model_name": "gpt-4o-mini"}, + ) + + with patch.object(ChatOpenAI, "_generate", return_value=response): + handler = CallbackHandler() + + with langfuse_memory_client.start_as_current_observation(name="parent"): + bind(ChatOpenAI(api_key="test", temperature=0)).invoke( + [HumanMessage(content="hello")], + config={"callbacks": [handler]}, + ) + + langfuse_memory_client.flush() + + +def _tool_definition_messages(generation_span, json_attr): + input_messages = json_attr( + generation_span, LangfuseOtelSpanAttributes.OBSERVATION_INPUT + ) + return [message for message in input_messages if message.get("role") == "tool"] + + +def test_normalize_tool_definition_passes_openai_format_through(): + openai_tool = { + "type": "function", + "function": {"name": "example_tool", "parameters": {"type": "object"}}, + } + + assert _normalize_tool_definition(openai_tool) is openai_tool + + +def test_normalize_tool_definition_converts_base_tool_instances(): + normalized = _normalize_tool_definition(_make_example_tool()) + + assert normalized["type"] == "function" + assert normalized["function"]["name"] == "example_tool" + assert normalized["function"]["description"] == "Example tool description" + assert "query" in normalized["function"]["parameters"]["properties"] + + +def test_normalize_tool_definition_wraps_bare_function_schemas(): + bare_function = { + "name": "example_tool", + "description": "Example tool description", + "parameters": {"type": "object", "properties": {}}, + } + + normalized = _normalize_tool_definition(bare_function) + + assert normalized["type"] == "function" + assert normalized["function"]["name"] == "example_tool" + + +def test_normalize_tool_definition_returns_unconvertible_values_unchanged(): + unconvertible = object() + + assert _normalize_tool_definition(unconvertible) is unconvertible + + +def test_raw_bound_tools_are_normalized_in_generation_input( + langfuse_memory_client, get_span, json_attr +): + # .bind() (unlike .bind_tools()) forwards raw tool objects into + # invocation_params -- the path that produced unrenderable tool + # definitions in the Langfuse UI. + _invoke_chat_model_with_tools( + langfuse_memory_client, lambda model: model.bind(tools=[_make_example_tool()]) + ) + + tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr) + + assert len(tool_messages) == 1 + tool_definition = tool_messages[0]["content"] + assert tool_definition["type"] == "function" + assert tool_definition["function"]["name"] == "example_tool" + assert "query" in tool_definition["function"]["parameters"]["properties"], ( + "args schema should be exposed as OpenAI-style parameters" + ) + + +def test_bind_tools_tool_definitions_stay_in_openai_format( + langfuse_memory_client, get_span, json_attr +): + # Regression guard for the standard path: bind_tools() already provides + # OpenAI-format definitions, which must pass through unchanged. + _invoke_chat_model_with_tools( + langfuse_memory_client, lambda model: model.bind_tools([_make_example_tool()]) + ) + + tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr) + + assert len(tool_messages) == 1 + tool_definition = tool_messages[0]["content"] + assert tool_definition["type"] == "function" + assert tool_definition["function"]["name"] == "example_tool" + assert "query" in tool_definition["function"]["parameters"]["properties"] From 4587d8dfff94bf00a8cd93d0df382404db2a3c17 Mon Sep 17 00:00:00 2001 From: Milana Gurbanova Date: Wed, 15 Jul 2026 08:55:14 +0200 Subject: [PATCH 2/2] fix(langchain): log tool normalization fallback at debug level --- langfuse/langchain/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/langfuse/langchain/utils.py b/langfuse/langchain/utils.py index 42ae3f67c..08ae8835c 100644 --- a/langfuse/langchain/utils.py +++ b/langfuse/langchain/utils.py @@ -3,6 +3,8 @@ import re from typing import Any, Dict, List, Literal, Optional, cast +from langfuse.logger import langfuse_logger + # NOTE ON DEPENDENCIES: # - since Jan 2024, there is https://pypi.org/project/langchain-openai/ which is a separate package and imports openai models. # Decided to not make this a dependency of langfuse as few people will have this. Need to match these models manually @@ -32,6 +34,10 @@ def _normalize_tool_definition(tool: Any) -> Any: return convert_to_openai_tool(tool) except Exception: + langfuse_logger.debug( + "Could not normalize tool definition to OpenAI format; keeping original.", + exc_info=True, + ) return tool