fix(langchain): normalize tool definitions to OpenAI function format

langfuse/langchain/CallbackHandler.py

@@ -33,7 +33,7 @@
     LangfuseTool,
 )
 from langfuse._utils import _get_timestamp
-from langfuse.langchain.utils import _extract_model_name
+from langfuse.langchain.utils import _extract_model_name, _normalize_tool_definition
 from langfuse.logger import langfuse_logger
 from langfuse.types import TraceContext
 
@@ -1189,7 +1189,12 @@ def __on_llm_action(
         try:
             tools = kwargs.get("invocation_params", {}).get("tools", None)
             if tools and isinstance(tools, list):
-                prompts.extend([{"role": "tool", "content": tool} for tool in tools])
+                prompts.extend(
+                    [
+                        {"role": "tool", "content": _normalize_tool_definition(tool)}
+                        for tool in tools
+                    ]
+                )
 
             model_name = self._parse_model_and_log_errors(
                 serialized=serialized, metadata=metadata, kwargs=kwargs

langfuse/langchain/utils.py

@@ -3,12 +3,44 @@
 import re
 from typing import Any, Dict, List, Literal, Optional, cast
 
+from langfuse.logger import langfuse_logger
+
 # NOTE ON DEPENDENCIES:
 # - since Jan 2024, there is https://pypi.org/project/langchain-openai/ which is a separate package and imports openai models.
 #   Decided to not make this a dependency of langfuse as few people will have this. Need to match these models manually
 # - langchain_community is loaded as a dependency of langchain, but extremely unreliable. Decided to not depend on it.
 
 
+def _normalize_tool_definition(tool: Any) -> Any:
+    """Best-effort normalization of a tool definition into the OpenAI function-tool format.
+
+    LangChain integrations are expected to pass tool definitions through
+    `invocation_params["tools"]` in the OpenAI format
+    `{"type": "function", "function": {...}}`. Some code paths (custom chat
+    models, `model.bind(tools=...)` instead of `bind_tools`) forward raw
+    `BaseTool` instances instead, which would otherwise be serialized as their
+    attribute dict (`args_schema`, `return_direct`, ...) and not be recognized
+    as tools by the Langfuse UI.
+    """
+    if (
+        isinstance(tool, dict)
+        and tool.get("type") == "function"
+        and isinstance(tool.get("function"), dict)
+    ):
+        return tool
+
+    try:
+        from langchain_core.utils.function_calling import convert_to_openai_tool
+
+        return convert_to_openai_tool(tool)
+    except Exception:
+        langfuse_logger.debug(
+            "Could not normalize tool definition to OpenAI format; keeping original.",
+            exc_info=True,
+        )
+        return tool
+
+
 def _extract_model_name(
     serialized: Optional[Dict[str, Any]],
     **kwargs: Any,

tests/unit/test_langchain.py

@@ -9,11 +9,14 @@
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.outputs import ChatGeneration, ChatResult, Generation, LLMResult
 from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.tools import StructuredTool
 from langchain_openai import ChatOpenAI, OpenAI
 from opentelemetry import context as otel_context
+from pydantic import BaseModel, Field
 
 from langfuse._client.attributes import LangfuseOtelSpanAttributes
 from langfuse.langchain import CallbackHandler
+from langfuse.langchain.utils import _normalize_tool_definition
 
 callback_handler_module = importlib.import_module("langfuse.langchain.CallbackHandler")
 
@@ -940,3 +943,117 @@ def import_without_langgraph_command(
         )
 
     importlib.reload(callback_handler_module)
+
+
+class _ExampleToolArgs(BaseModel):
+    query: str = Field(description="What to search for")
+
+
+def _make_example_tool() -> StructuredTool:
+    return StructuredTool.from_function(
+        func=lambda query: query,
+        name="example_tool",
+        description="Example tool description",
+        args_schema=_ExampleToolArgs,
+    )
+
+
+def _invoke_chat_model_with_tools(langfuse_memory_client, bind):
+    response = ChatResult(
+        generations=[ChatGeneration(message=AIMessage(content="ok"), text="ok")],
+        llm_output={"token_usage": {}, "model_name": "gpt-4o-mini"},
+    )
+
+    with patch.object(ChatOpenAI, "_generate", return_value=response):
+        handler = CallbackHandler()
+
+        with langfuse_memory_client.start_as_current_observation(name="parent"):
+            bind(ChatOpenAI(api_key="test", temperature=0)).invoke(
+                [HumanMessage(content="hello")],
+                config={"callbacks": [handler]},
+            )
+
+    langfuse_memory_client.flush()
+
+
+def _tool_definition_messages(generation_span, json_attr):
+    input_messages = json_attr(
+        generation_span, LangfuseOtelSpanAttributes.OBSERVATION_INPUT
+    )
+    return [message for message in input_messages if message.get("role") == "tool"]
+
+
+def test_normalize_tool_definition_passes_openai_format_through():
+    openai_tool = {
+        "type": "function",
+        "function": {"name": "example_tool", "parameters": {"type": "object"}},
+    }
+
+    assert _normalize_tool_definition(openai_tool) is openai_tool
+
+
+def test_normalize_tool_definition_converts_base_tool_instances():
+    normalized = _normalize_tool_definition(_make_example_tool())
+
+    assert normalized["type"] == "function"
+    assert normalized["function"]["name"] == "example_tool"
+    assert normalized["function"]["description"] == "Example tool description"
+    assert "query" in normalized["function"]["parameters"]["properties"]
+
+
+def test_normalize_tool_definition_wraps_bare_function_schemas():
+    bare_function = {
+        "name": "example_tool",
+        "description": "Example tool description",
+        "parameters": {"type": "object", "properties": {}},
+    }
+
+    normalized = _normalize_tool_definition(bare_function)
+
+    assert normalized["type"] == "function"
+    assert normalized["function"]["name"] == "example_tool"
+
+
+def test_normalize_tool_definition_returns_unconvertible_values_unchanged():
+    unconvertible = object()
+
+    assert _normalize_tool_definition(unconvertible) is unconvertible
+
+
+def test_raw_bound_tools_are_normalized_in_generation_input(
+    langfuse_memory_client, get_span, json_attr
+):
+    # .bind() (unlike .bind_tools()) forwards raw tool objects into
+    # invocation_params -- the path that produced unrenderable tool
+    # definitions in the Langfuse UI.
+    _invoke_chat_model_with_tools(
+        langfuse_memory_client, lambda model: model.bind(tools=[_make_example_tool()])
+    )
+
+    tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr)
+
+    assert len(tool_messages) == 1
+    tool_definition = tool_messages[0]["content"]
+    assert tool_definition["type"] == "function"
+    assert tool_definition["function"]["name"] == "example_tool"
+    assert "query" in tool_definition["function"]["parameters"]["properties"], (
+        "args schema should be exposed as OpenAI-style parameters"
+    )
+
+
+def test_bind_tools_tool_definitions_stay_in_openai_format(
+    langfuse_memory_client, get_span, json_attr
+):
+    # Regression guard for the standard path: bind_tools() already provides
+    # OpenAI-format definitions, which must pass through unchanged.
+    _invoke_chat_model_with_tools(
+        langfuse_memory_client, lambda model: model.bind_tools([_make_example_tool()])
+    )
+
+    tool_messages = _tool_definition_messages(get_span("ChatOpenAI"), json_attr)
+
+    assert len(tool_messages) == 1
+    tool_definition = tool_messages[0]["content"]
+    assert tool_definition["type"] == "function"
+    assert tool_definition["function"]["name"] == "example_tool"
+    assert "query" in tool_definition["function"]["parameters"]["properties"]