diff --git a/hugo/config/_default/menus/main.en.yaml b/hugo/config/_default/menus/main.en.yaml index 91d1c806ca0..6125f3471d6 100644 --- a/hugo/config/_default/menus/main.en.yaml +++ b/hugo/config/_default/menus/main.en.yaml @@ -5519,245 +5519,246 @@ menu: parent: llm_obs identifier: llm_obs_quickstart weight: 1 - - name: Instrumentation - url: llm_observability/instrumentation/ + - name: Terms and Concepts + url: llm_observability/quickstart/terms/ + parent: llm_obs_quickstart + identifier: llm_obs_terms + weight: 101 + - name: Build with AI + url: llm_observability/build_with_ai/ parent: llm_obs - identifier: llm_obs_instrumentation + identifier: llm_obs_build_with_ai weight: 2 - - name: Automatic - url: llm_observability/instrumentation/auto_instrumentation + - name: Instrument + url: llm_observability/instrument/ + parent: llm_obs + identifier: llm_obs_instrumentation + weight: 3 + - name: Automatic Instrumentation + url: llm_observability/instrument/auto_instrumentation parent: llm_obs_instrumentation identifier: llm_obs_instrumentation_auto - weight: 201 + weight: 301 - name: SDK Reference - url: llm_observability/instrumentation/sdk + url: llm_observability/instrument/sdk parent: llm_obs_instrumentation identifier: llm_obs_instrumentation_sdk - weight: 202 - - name: HTTP API - url: llm_observability/instrumentation/api + weight: 302 + - name: API Reference + url: llm_observability/instrument/api parent: llm_obs_instrumentation identifier: llm_obs_instrumentation_api - weight: 203 - - name: OpenTelemetry - url: llm_observability/instrumentation/otel_instrumentation + weight: 303 + - name: OpenTelemetry Instrumentation + url: llm_observability/instrument/otel_instrumentation parent: llm_obs_instrumentation identifier: llm_obs_instrumentation_otel - weight: 204 - - name: Tracing Proxy Services - url: llm_observability/trace_proxy_services + weight: 304 + - name: Agentic Instrumentation + url: llm_observability/instrument/agentic/ parent: llm_obs_instrumentation - identifier: llm_obs_instrumentation_trace_proxy_services - weight: 205 - - name: Lapdog - url: llm_observability/lapdog - parent: llm_obs - identifier: llm_obs_lapdog - weight: 3 - - name: Monitoring - url: llm_observability/monitoring + identifier: llm_obs_instrumentation_agentic + weight: 305 + - name: Python + url: llm_observability/instrument/agentic/python + parent: llm_obs_instrumentation_agentic + identifier: llm_obs_instrumentation_agentic_python + weight: 30501 + - name: Node.js + url: llm_observability/instrument/agentic/nodejs + parent: llm_obs_instrumentation_agentic + identifier: llm_obs_instrumentation_agentic_nodejs + weight: 30502 + - name: Java + url: llm_observability/instrument/agentic/java + parent: llm_obs_instrumentation_agentic + identifier: llm_obs_instrumentation_agentic_java + weight: 30503 + - name: Agent Observability and APM + url: llm_observability/instrument/agent_observability_and_apm + parent: llm_obs_instrumentation + identifier: llm_obs_correlate_apm + weight: 310 + - name: Prompt Tracking + url: llm_observability/instrument/prompt_tracking + parent: llm_obs_instrumentation + identifier: llm_obs_prompt_tracking + weight: 311 + - name: Investigate + url: llm_observability/investigate/ parent: llm_obs identifier: llm_obs_monitoring weight: 4 - name: Querying spans and traces - url: llm_observability/monitoring/querying + url: llm_observability/investigate/querying parent: llm_obs_monitoring identifier: llm_obs_monitoring_querying - weight: 301 - - name: Correlate with APM - url: llm_observability/monitoring/llm_observability_and_apm - parent: llm_obs_monitoring - identifier: llm_obs_correlate_apm - weight: 302 - - name: Patterns - url: llm_observability/monitoring/patterns/ - parent: llm_obs_monitoring - identifier: llm_obs_patterns - weight: 303 - - name: Agent Monitoring - url: llm_observability/monitoring/agent_monitoring - parent: llm_obs_monitoring - identifier: llm_obs_agent_monitoring - weight: 304 + weight: 401 - name: Cost - url: llm_observability/monitoring/cost + url: llm_observability/investigate/cost parent: llm_obs_monitoring identifier: llm_obs_monitoring_cost - - name: MCP Clients - url: llm_observability/monitoring/mcp_client - parent: llm_obs_monitoring - identifier: llm_obs_mcp_client - weight: 305 - - name: Prompt Tracking - url: llm_observability/monitoring/prompt_tracking - parent: llm_obs_monitoring - identifier: llm_obs_prompt_tracking - weight: 306 - - name: Prompt Management - url: llm_observability/monitoring/prompt_management - parent: llm_obs_monitoring - identifier: llm_obs_prompt_management - weight: 307 + weight: 402 - name: Metrics - url: llm_observability/monitoring/metrics + url: llm_observability/investigate/metrics parent: llm_obs_monitoring identifier: llm_obs_monitoring_metrics - weight: 308 - - name: Automation Rules - url: llm_observability/monitoring/automation_rules - parent: llm_obs_monitoring - identifier: llm_obs_monitoring_automation_rules - weight: 309 - - name: Evaluations - url: llm_observability/evaluations/ - parent: llm_obs - identifier: llm_obs_evaluations - weight: 5 - - name: Custom LLM-as-a-Judge - url: llm_observability/evaluations/custom_llm_as_a_judge_evaluations - parent: llm_obs_evaluations - identifier: llm_obs_custom_llm_as_a_judge_evaluations - weight: 400 - - name: Language Mismatch - url: llm_observability/evaluations/managed_evaluations/language_mismatch - parent: llm_obs_evaluations - identifier: llm_obs_language_mismatch - weight: 401 - - name: Sensitive Data Scanner - url: llm_observability/evaluations/sensitive_data_scanner - parent: llm_obs_evaluations - identifier: llm_obs_sensitive_data_scanner - weight: 402 - - name: External Evaluations - url: llm_observability/evaluations/external_evaluations - parent: llm_obs_evaluations - identifier: llm_obs_external_evaluations weight: 403 - - name: End-User Feedback - url: llm_observability/evaluations/end_user_feedback - parent: llm_obs_evaluations - identifier: llm_obs_end_user_feedback - weight: 4030 - - name: Developer Guide - url: llm_observability/evaluations/evaluation_developer_guide - parent: llm_obs_evaluations - identifier: llm_obs_evaluation_developer_guide - weight: 4031 - - name: Connect Your LLM Provider - url: llm_observability/evaluations/custom_llm_as_a_judge_evaluations/connect_to_account - parent: llm_obs_custom_llm_as_a_judge_evaluations - identifier: llm_obs_custom_llm_as_a_judge_evaluations_connect_to_account - weight: 40100 - - name: Template Evaluations - url: llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations - parent: llm_obs_custom_llm_as_a_judge_evaluations - identifier: llm_obs_custom_llm_as_a_judge_evaluations_template - weight: 40101 - - name: Session-Level Evaluations - url: llm_observability/evaluations/custom_llm_as_a_judge_evaluations/session_level_evaluations - parent: llm_obs_custom_llm_as_a_judge_evaluations - identifier: llm_obs_custom_llm_as_a_judge_evaluations_session_level - weight: 40102 - - name: Trace-Level Evaluations - url: llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations - parent: llm_obs_custom_llm_as_a_judge_evaluations - identifier: llm_obs_custom_llm_as_a_judge_evaluations_trace_level - weight: 40103 - - name: Prompt Templating - url: llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating - parent: llm_obs_custom_llm_as_a_judge_evaluations - identifier: llm_obs_custom_llm_as_a_judge_evaluations_prompt_templating - weight: 40104 - - name: NeMo - url: llm_observability/evaluations/submit_nemo_evaluations - parent: llm_obs_external_evaluations - identifier: llm_obs_submit_nemo_evaluations - weight: 40201 - - name: DeepEval Evaluations - url: llm_observability/evaluations/deepeval_evaluations - parent: llm_obs_external_evaluations - identifier: llm_obs_deepeval_evaluations - weight: 40301 - - name: Pydantic Evaluations - url: llm_observability/evaluations/pydantic_evaluations - parent: llm_obs_external_evaluations - identifier: llm_obs_pydantic_evaluations - weight: 40302 + - name: Patterns + url: llm_observability/investigate/patterns/ + parent: llm_obs_monitoring + identifier: llm_obs_patterns + weight: 404 - name: Annotation Queues - url: llm_observability/evaluations/annotation_queues - parent: llm_obs_evaluations + url: llm_observability/investigate/annotation_queues + parent: llm_obs_monitoring identifier: llm_obs_annotation_queues - weight: 403 - - name: Compatibility - url: llm_observability/evaluations/evaluation_compatibility - parent: llm_obs_evaluations - identifier: llm_obs_evaluation_compatibility - weight: 404 + weight: 405 - name: Export API - url: llm_observability/evaluations/export_api - parent: llm_obs_evaluations + url: llm_observability/investigate/export_api + parent: llm_obs_monitoring identifier: llm_obs_evaluations_export_api - weight: 405 - - name: Experiments - url: llm_observability/experiments + weight: 406 + - name: Improve + url: llm_observability/improve/ parent: llm_obs - identifier: llm_obs_experiments - weight: 6 - - name: Setup and Usage - url: llm_observability/experiments/setup - parent: llm_obs_experiments - identifier: llm_obs_experiments_setup + identifier: llm_obs_improve + weight: 5 + - name: Playground + url: llm_observability/improve/playground + parent: llm_obs_improve + identifier: llm_obs_playground weight: 501 - name: Datasets - url: llm_observability/experiments/datasets - parent: llm_obs_experiments + url: llm_observability/improve/datasets + parent: llm_obs_improve identifier: llm_obs_experiments_datasets weight: 502 + - name: Experiments + url: llm_observability/improve/experiments + parent: llm_obs_improve + identifier: llm_obs_experiments + weight: 503 + - name: Setup and Usage + url: llm_observability/improve/experiments/setup + parent: llm_obs_experiments + identifier: llm_obs_experiments_setup + weight: 50301 - name: Analyzing Results - url: llm_observability/experiments/analyzing_results + url: llm_observability/improve/experiments/analyzing_results parent: llm_obs_experiments identifier: llm_obs_experiments_analyzing_results - weight: 503 + weight: 50302 - name: Advanced Experiment Runs - url: llm_observability/experiments/advanced_runs + url: llm_observability/improve/experiments/advanced_runs parent: llm_obs_experiments identifier: llm_obs_experiments_advanced_runs - weight: 504 - - name: Experiments API - url: llm_observability/experiments/api - parent: llm_obs_experiments - identifier: llm_obs_experiments_api - weight: 505 + weight: 50303 - name: Prompt Optimization - url: llm_observability/experiments/prompt_optimization + url: llm_observability/improve/experiments/prompt_optimization parent: llm_obs_experiments identifier: llm_obs_experiments_prompt_optimization - weight: 506 - - name: Playground - url: llm_observability/playground + weight: 50304 + - name: Experiments API + url: llm_observability/improve/experiments/api parent: llm_obs_experiments - identifier: llm_obs_playground - weight: 507 - - name: MCP Server - url: llm_observability/mcp_server + identifier: llm_obs_experiments_api + weight: 50305 + - name: Configure + url: llm_observability/configure/ parent: llm_obs - identifier: llm_obs_mcp_server - weight: 7 - - name: Data Security and RBAC - url: llm_observability/data_security_and_rbac + identifier: llm_obs_configure + weight: 6 + - name: Evaluations + url: llm_observability/configure/evaluations/ + parent: llm_obs_configure + identifier: llm_obs_evaluations + weight: 601 + - name: Custom LLM-as-a-Judge + url: llm_observability/configure/evaluations/llm_as_a_judge_evaluations + parent: llm_obs_evaluations + identifier: llm_obs_custom_llm_as_a_judge_evaluations + weight: 60101 + - name: Language Mismatch + url: llm_observability/configure/evaluations/language_mismatch + parent: llm_obs_evaluations + identifier: llm_obs_language_mismatch + weight: 60102 + - name: End-User Feedback + url: llm_observability/configure/evaluations/end_user_feedback + parent: llm_obs_evaluations + identifier: llm_obs_end_user_feedback + weight: 60104 + - name: Compatibility + url: llm_observability/configure/evaluations/compatibility + parent: llm_obs_evaluations + identifier: llm_obs_evaluation_compatibility + weight: 60106 + - name: External Evaluations + url: llm_observability/configure/evaluations/external_evaluations + parent: llm_obs_evaluations + identifier: llm_obs_external_evaluations + weight: 60107 + - name: Prompt Management + url: llm_observability/configure/prompt_management + parent: llm_obs_configure + identifier: llm_obs_prompt_management + weight: 602 + - name: Automation Rules + url: llm_observability/configure/automation_rules + parent: llm_obs_configure + identifier: llm_obs_monitoring_automation_rules + weight: 603 + - name: Data Privacy, Security, and RBAC + url: llm_observability/data_privacy_security_and_rbac parent: llm_obs identifier: llm_obs_data_security_and_rbac - weight: 8 - - name: Terms and Concepts - url: llm_observability/terms/ + weight: 7 + - name: Lapdog + url: llm_observability/lapdog parent: llm_obs - identifier: llm_obs_terms - weight: 9 + identifier: llm_obs_lapdog + weight: 8 - name: Guides url: llm_observability/guide/ parent: llm_obs identifier: llm_obs_guide - weight: 10 + weight: 9 + - name: Using the Datadog-CrewAI Integration + url: llm_observability/guide/crewai_guide + parent: llm_obs_guide + identifier: llm_obs_guide_crewai + weight: 901 + - name: Trace an LLM Application in AWS Lambda + url: llm_observability/guide/trace-an-llm-application-in-aws-lambda + parent: llm_obs_guide + identifier: llm_obs_guide_aws_lambda + weight: 902 + - name: Evaluation Developer Guide + url: llm_observability/configure/evaluations/evaluation_developer_guide + parent: llm_obs_guide + identifier: llm_obs_guide_evaluation_developer_guide + weight: 903 + - name: Instrument a Next.js Application + url: llm_observability/guide/nextjs_guide + parent: llm_obs_guide + identifier: llm_obs_guide_nextjs + weight: 903 + - name: Monitor Proxy Services + url: llm_observability/guide/monitor_proxy_services + parent: llm_obs_guide + identifier: llm_obs_guide_monitor_proxy_services + weight: 904 + - name: Agent Monitoring + url: llm_observability/guide/agent_monitoring + parent: llm_obs_guide + identifier: llm_obs_guide_agent_monitoring + weight: 905 + - name: Monitor MCP Clients + url: llm_observability/guide/monitor_mcp_client + parent: llm_obs_guide + identifier: llm_obs_guide_monitor_mcp_client + weight: 906 - name: GPU Monitoring url: gpu_monitoring/ pre: gpu-monitoring-wui diff --git a/hugo/content/en/account_management/rbac/data_access.md b/hugo/content/en/account_management/rbac/data_access.md index bee9fc060e8..a60ec948f57 100644 --- a/hugo/content/en/account_management/rbac/data_access.md +++ b/hugo/content/en/account_management/rbac/data_access.md @@ -267,4 +267,4 @@ When querying data through Datadog APIs with restrictions enabled, users without [10]: /account_management/rbac/data_access/#supported-telemetry [11]: /logs/guide/logs-rbac/?tab=ui#restrict-access-to-logs [12]: /dashboards/sharing/shared_dashboards/ -[13]: /llm_observability/instrumentation/otel_instrumentation/ +[13]: /llm_observability/instrument/otel_instrumentation/ diff --git a/hugo/content/en/api/latest/using-the-api/_index.md b/hugo/content/en/api/latest/using-the-api/_index.md index d901c331789..3dfde62b4ea 100644 --- a/hugo/content/en/api/latest/using-the-api/_index.md +++ b/hugo/content/en/api/latest/using-the-api/_index.md @@ -104,7 +104,7 @@ You can also use the Datadog API to manage your account programmatically: [33]: /api/v1/key-management/ [34]: /api/v1/usage-metering/ [35]: /api/v1/ip-ranges/ -[36]: /llm_observability/evaluations/export_api +[36]: /llm_observability/investigate/export_api [37]: /api/latest/cloudflare-integration/ [38]: /api/latest/fastly-integration/ [39]: /api/latest/jira-integration/ diff --git a/hugo/content/en/llm_observability/_index.md b/hugo/content/en/llm_observability/_index.md index 1c08c93bb18..22d37a09909 100644 --- a/hugo/content/en/llm_observability/_index.md +++ b/hugo/content/en/llm_observability/_index.md @@ -132,12 +132,12 @@ See the [Setup documentation][5] for instructions on instrumenting your LLM appl {{< partial name="whats-next/whats-next.html" >}} [1]: https://app.datadoghq.com/llm/traces -[2]: /llm_observability/terms +[2]: /llm_observability/quickstart/terms [3]: /llm_observability/setup/sdk [4]: /llm_observability/setup/api [5]: /llm_observability/setup [6]: /llm_observability/quickstart [7]: https://app.datadoghq.com/dash/integration/llm_operational_insights [8]: /llm_observability/setup/auto_instrumentation -[9]: /llm_observability/evaluations/managed_evaluations -[10]: /llm_observability/monitoring/patterns +[9]: /llm_observability/configure/evaluations/managed_evaluations +[10]: /llm_observability/investigate/patterns diff --git a/hugo/content/en/llm_observability/build_with_ai/_index.md b/hugo/content/en/llm_observability/build_with_ai/_index.md new file mode 100644 index 00000000000..221b49223a2 --- /dev/null +++ b/hugo/content/en/llm_observability/build_with_ai/_index.md @@ -0,0 +1,65 @@ +--- +title: Build with AI +description: Use the Datadog MCP server, CLI, and Claude Code skills to build and analyze LLM applications from your development environment. +--- + +Datadog supports coding agent workflows through the Datadog MCP server, the Pup CLI, and Claude Code skills. Use them to investigate your Agent Observability data and iterate on your LLM application without leaving your development environment. + +## Get started + +### Install the skills + +{{< code-block lang="shell" >}} +npx skills add datadog-labs/agent-skills/agent-observability --full-depth -y +{{< /code-block >}} + +### Choose a data backend + +The skills read your Agent Observability data through either the Datadog MCP server or the Pup CLI. Set up one of them. Each skill detects the MCP server at startup and falls back to the Pup CLI when the MCP server is unavailable. + +{{% collapse-content title="Option A: Datadog MCP server" level="h4" expanded=true id="option-a-mcp-server" %}} + +{{< site-region region="us,us3,us5,eu,ap1,ap2,uk1" >}} +Connect the MCP server to your Claude Code session: + +
claude mcp add --scope user --transport http datadog-llmo-mcp \
+  '{{< region-param key="mcp_server_endpoint" >}}?toolsets=llmobs,core'
+ +To run the MCP server as a local binary instead, install the Datadog MCP CLI: + +{{< code-block lang="shell" >}} +curl -sSL https://coterm.datadoghq.com/mcp-cli/install.sh | bash +datadog_mcp_cli login +claude mcp add datadog --scope user -- ~/.local/bin/datadog_mcp_cli +{{< /code-block >}} +{{< /site-region >}} + +{{< site-region region="gov,gov2" >}} +
This product is not supported for your selected site ({{< region-param key="dd_site_name" >}}).
+{{< /site-region >}} + +For the full setup, including authentication options and the available tools, see [MCP Server][2]. + +{{% /collapse-content %}} + +{{% collapse-content title="Option B: Pup CLI" level="h4" expanded=false id="option-b-pup-cli" %}} + +Install the Pup CLI with Homebrew (macOS/Linux) and authenticate: + +{{< code-block lang="shell" >}} +brew tap datadog-labs/pack +brew install datadog-labs/pack/pup +pup auth login +{{< /code-block >}} + +For other installation methods, supported commands, and authentication options, see [Pup CLI][1]. + +{{% /collapse-content %}} + +{{< whatsnext desc="Build with the Datadog MCP server and skills:" >}} + {{< nextlink href="/llm_observability/build_with_ai/mcp_server" >}}MCP Server{{< /nextlink >}} + {{< nextlink href="/llm_observability/build_with_ai/claude_code_skills" >}}Claude Code Skills{{< /nextlink >}} +{{< /whatsnext >}} + +[1]: /cli/ +[2]: /llm_observability/build_with_ai/mcp_server diff --git a/hugo/content/en/llm_observability/guide/claude_code_skills.md b/hugo/content/en/llm_observability/build_with_ai/claude_code_skills.md similarity index 99% rename from hugo/content/en/llm_observability/guide/claude_code_skills.md rename to hugo/content/en/llm_observability/build_with_ai/claude_code_skills.md index 7ff7f835774..f5f94c3e769 100644 --- a/hugo/content/en/llm_observability/guide/claude_code_skills.md +++ b/hugo/content/en/llm_observability/build_with_ai/claude_code_skills.md @@ -1,11 +1,13 @@ --- title: Analyze LLM Applications with Claude Code Skills +aliases: +- /llm_observability/guide/claude_code_skills/ description: Use Datadog's Claude Code skills to classify sessions, diagnose failures, compare experiments, generate Python experiment code, and bootstrap evaluators against your live production data. further_reading: - - link: '/llm_observability/evaluations/' + - link: '/llm_observability/configure/evaluations/' tag: 'Documentation' text: 'Agent Observability Evaluations' - - link: '/llm_observability/experiments/' + - link: '/llm_observability/improve/experiments/' tag: 'Documentation' text: 'LLM Experiments' - link: '/llm_observability/guide/evaluation_developer_guide' @@ -296,7 +298,7 @@ If you are new to evaluating an LLM application, the recommended flow is: [1]: https://claude.ai/code [2]: /llm_observability/setup/ -[3]: /llm_observability/experiments/ +[3]: /llm_observability/improve/experiments/ [4]: /llm_observability/guide/evaluation_developer_guide [5]: https://datadoghq.atlassian.net/wiki/spaces/BITSAI/pages/5226692942/pup+CLI [6]: https://github.com/datadog-labs/agent-skills diff --git a/hugo/content/en/llm_observability/mcp_server.md b/hugo/content/en/llm_observability/build_with_ai/mcp_server.md similarity index 99% rename from hugo/content/en/llm_observability/mcp_server.md rename to hugo/content/en/llm_observability/build_with_ai/mcp_server.md index ee1a2887b5d..0762824a699 100644 --- a/hugo/content/en/llm_observability/mcp_server.md +++ b/hugo/content/en/llm_observability/build_with_ai/mcp_server.md @@ -1,17 +1,19 @@ --- title: Agent Observability MCP and Skills +aliases: +- /llm_observability/mcp_server/ description: "Connect AI agents to your Agent Observability traces and experiments using the Datadog MCP Server." further_reading: - link: "mcp_server" tag: "Documentation" text: "Datadog MCP Server" -- link: "/llm_observability/experiments" +- link: "/llm_observability/improve/experiments" tag: "Documentation" text: "Set up and use Agent Observability Experiments" -- link: "/llm_observability/monitoring" +- link: "/llm_observability/investigate" tag: "Documentation" text: "Monitor your application with Agent Observability" -- link: "/llm_observability/guide/claude_code_skills" +- link: "/llm_observability/build_with_ai/claude_code_skills" tag: "Guide" text: "Analyze LLM Applications with Claude Code Skills" --- @@ -483,4 +485,4 @@ For custom visualizations that go beyond standard Datadog widgets, like comparis [6]: /account_management/api-app-keys/ [7]: /account_management/org_settings/service_accounts/ [8]: https://github.com/datadog-labs/agent-skills -[9]: /llm_observability/guide/claude_code_skills +[9]: /llm_observability/build_with_ai/claude_code_skills diff --git a/hugo/content/en/llm_observability/configure/_index.md b/hugo/content/en/llm_observability/configure/_index.md new file mode 100644 index 00000000000..4d73cd7987e --- /dev/null +++ b/hugo/content/en/llm_observability/configure/_index.md @@ -0,0 +1,12 @@ +--- +title: Configure +description: Configure evaluations, prompt management, and automation rules for Agent Observability. +--- + +Configure the Agent Observability capabilities that Datadog runs for you, including evaluations, automation rules, and prompt management. Datadog hosts and scales them, so you do not operate your own evaluation, routing, or prompt-serving infrastructure. + +{{< whatsnext desc="Configuration options:" >}} + {{< nextlink href="/llm_observability/configure/evaluations/" >}}Evaluations{{< /nextlink >}} + {{< nextlink href="/llm_observability/configure/prompt_management" >}}Prompt Management{{< /nextlink >}} + {{< nextlink href="/llm_observability/configure/automation_rules" >}}Automation Rules{{< /nextlink >}} +{{< /whatsnext >}} diff --git a/hugo/content/en/llm_observability/monitoring/automation_rules.md b/hugo/content/en/llm_observability/configure/automation_rules.md similarity index 95% rename from hugo/content/en/llm_observability/monitoring/automation_rules.md rename to hugo/content/en/llm_observability/configure/automation_rules.md index 7b72038583e..668a462df6d 100644 --- a/hugo/content/en/llm_observability/monitoring/automation_rules.md +++ b/hugo/content/en/llm_observability/configure/automation_rules.md @@ -1,11 +1,13 @@ --- title: Automation Rules +aliases: +- /llm_observability/monitoring/automation_rules/ description: Route Agent Observability traces into annotation queues or datasets automatically using filter-based rules. further_reading: - - link: /llm_observability/evaluations/annotation_queues + - link: /llm_observability/investigate/annotation_queues tag: Documentation text: Set up annotation queues for human review - - link: /llm_observability/experiments/datasets + - link: /llm_observability/improve/datasets tag: Documentation text: Build datasets from production traces - link: /api/latest/llm-observability/ @@ -89,6 +91,6 @@ The Trace Explorer supports a wider field set than automations. The button's too {{< partial name="whats-next/whats-next.html" >}} [1]: https://app.datadoghq.com/llm/traces -[2]: /llm_observability/evaluations/annotation_queues/?tab=usingautomationrules -[3]: /llm_observability/experiments/datasets/?tab=fromproductiontraces +[2]: /llm_observability/investigate/annotation_queues/?tab=usingautomationrules +[3]: /llm_observability/improve/datasets/?tab=fromproductiontraces [4]: https://app.datadoghq.com/llm/settings/automations diff --git a/hugo/content/en/llm_observability/evaluations/_index.md b/hugo/content/en/llm_observability/configure/evaluations/_index.md similarity index 87% rename from hugo/content/en/llm_observability/evaluations/_index.md rename to hugo/content/en/llm_observability/configure/evaluations/_index.md index 8a7774cdcb4..c5aad7a63d2 100644 --- a/hugo/content/en/llm_observability/evaluations/_index.md +++ b/hugo/content/en/llm_observability/configure/evaluations/_index.md @@ -4,6 +4,7 @@ description: Learn how to configure Evaluations for your LLM application. aliases: - /tracing/llm_observability/evaluations/ - /llm_observability/configuration/ + - /llm_observability/evaluations/ further_reading: - link: https://www.datadoghq.com/blog/llm-prompt-tracking @@ -66,15 +67,15 @@ Agent Observability offers an [Export API][9] that you can use to retrieve spans {{< partial name="whats-next/whats-next.html" >}} -[1]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations -[2]: /llm_observability/evaluations/managed_evaluations -[3]: /llm_observability/evaluations/external_evaluations -[5]: /llm_observability/evaluations/submit_nemo_evaluations +[1]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations +[2]: /llm_observability/configure/evaluations/managed_evaluations +[3]: /llm_observability/configure/evaluations/external_evaluations +[5]: /llm_observability/configure/evaluations/external_evaluations/nemo [6]: /security/sensitive_data_scanner/ [7]: /account_management/rbac/permissions/#llm-observability [8]: https://app.datadoghq.com/llm/evaluations -[9]: /llm_observability/evaluations/export_api +[9]: /llm_observability/investigate/export_api [10]: /llm_observability/guide/evaluation_developer_guide -[11]: /llm_observability/evaluations/annotation_queues +[11]: /llm_observability/investigate/annotation_queues [12]: /security/sensitive_data_scanner/scanning_rules/library_rules/ -[13]: /llm_observability/evaluations/end_user_feedback +[13]: /llm_observability/configure/evaluations/end_user_feedback diff --git a/hugo/content/en/llm_observability/evaluations/evaluation_compatibility.md b/hugo/content/en/llm_observability/configure/evaluations/compatibility.md similarity index 67% rename from hugo/content/en/llm_observability/evaluations/evaluation_compatibility.md rename to hugo/content/en/llm_observability/configure/evaluations/compatibility.md index 2f94db919c9..1dd6b6bc071 100644 --- a/hugo/content/en/llm_observability/evaluations/evaluation_compatibility.md +++ b/hugo/content/en/llm_observability/configure/evaluations/compatibility.md @@ -1,5 +1,7 @@ --- title: Evaluation compatibility +aliases: +- /llm_observability/evaluations/evaluation_compatibility/ description: Learn about the compatibility requirements for evaluations. --- @@ -43,14 +45,14 @@ Existing templates for custom LLM-as-a-judge evaluations are supported for the f | [Goal Completeness][3] | Fully supported | All third party LLM providers | LLM only | -[1]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#tool-selection -[2]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#tool-argument-correctness -[3]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#goal-completeness -[4]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#hallucination -[5]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#failure-to-answer -[6]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#sentiment -[7]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#toxicity -[8]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#prompt-injection -[9]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#topic-relevancy -[10]: /llm_observability/evaluations/managed_evaluations#language-mismatch -[11]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations#define-the-evaluation-output +[1]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#tool-selection +[2]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#tool-argument-correctness +[3]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#goal-completeness +[4]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#hallucination +[5]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#failure-to-answer +[6]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#sentiment +[7]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#toxicity +[8]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#prompt-injection +[9]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#topic-relevancy +[10]: /llm_observability/configure/evaluations/managed_evaluations#language-mismatch +[11]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations#define-the-evaluation-output diff --git a/hugo/content/en/llm_observability/evaluations/end_user_feedback.md b/hugo/content/en/llm_observability/configure/evaluations/end_user_feedback.md similarity index 86% rename from hugo/content/en/llm_observability/evaluations/end_user_feedback.md rename to hugo/content/en/llm_observability/configure/evaluations/end_user_feedback.md index 1a124ef1abf..a1066dcddaf 100644 --- a/hugo/content/en/llm_observability/evaluations/end_user_feedback.md +++ b/hugo/content/en/llm_observability/configure/evaluations/end_user_feedback.md @@ -1,14 +1,16 @@ --- title: End-User Feedback +aliases: +- /llm_observability/evaluations/end_user_feedback/ description: Submit end-user feedback to Agent Observability and connect it to spans, traces, sessions, or external entities. further_reading: - - link: '/llm_observability/instrumentation/api/#evaluations-api' + - link: '/llm_observability/instrument/api/#evaluations-api' tag: 'Documentation' text: 'Learn about the Evaluations API' - - link: '/llm_observability/evaluations/external_evaluations' + - link: '/llm_observability/configure/evaluations/external_evaluations' tag: 'Documentation' text: 'Learn about submitting external evaluations' - - link: '/llm_observability/evaluations/annotation_queues' + - link: '/llm_observability/investigate/annotation_queues' tag: 'Documentation' text: 'Learn about Annotation Queues' --- @@ -115,9 +117,9 @@ To create a dashboard widget for feedback, create the widget as you would for an {{< partial name="whats-next/whats-next.html" >}} -[1]: /llm_observability/evaluations/external_evaluations -[2]: /llm_observability/evaluations/annotation_queues -[3]: /llm_observability/instrumentation/api/#evaluations-api -[4]: /llm_observability/instrumentation/sdk/?tab=python#enriching-spans -[5]: /llm_observability/instrumentation/api/?tab=model#spans-api -[6]: /llm_observability/monitoring/querying/#feedback-queries +[1]: /llm_observability/configure/evaluations/external_evaluations +[2]: /llm_observability/investigate/annotation_queues +[3]: /llm_observability/instrument/api/#evaluations-api +[4]: /llm_observability/instrument/sdk/?tab=python#enriching-spans +[5]: /llm_observability/instrument/api/?tab=model#spans-api +[6]: /llm_observability/investigate/querying/#feedback-queries diff --git a/hugo/content/en/llm_observability/evaluations/evaluation_developer_guide.md b/hugo/content/en/llm_observability/configure/evaluations/evaluation_developer_guide.md similarity index 97% rename from hugo/content/en/llm_observability/evaluations/evaluation_developer_guide.md rename to hugo/content/en/llm_observability/configure/evaluations/evaluation_developer_guide.md index ecfdb782612..22b4351cd74 100644 --- a/hugo/content/en/llm_observability/evaluations/evaluation_developer_guide.md +++ b/hugo/content/en/llm_observability/configure/evaluations/evaluation_developer_guide.md @@ -2,15 +2,16 @@ title: Evaluation Developer Guide aliases: - /llm_observability/guide/evaluation_developer_guide + - /llm_observability/evaluations/evaluation_developer_guide/ description: Learn how to build custom evaluators using the Agent Observability SDK. further_reading: - - link: '/llm_observability/evaluations/external_evaluations' + - link: '/llm_observability/configure/evaluations/external_evaluations' tag: 'Documentation' text: 'Learn about submitting external evaluations' - link: '/llm_observability/setup/sdk/python' tag: 'Documentation' text: 'Learn about the Agent Observability SDK for Python' - - link: '/llm_observability/instrumentation/api' + - link: '/llm_observability/instrument/api' tag: 'Documentation' text: 'Learn about the HTTP API Reference' --- @@ -645,7 +646,7 @@ except RemoteEvaluatorError as e: ## Using evaluators in production -
This section covers evaluations you run and submit manually from your application code. To have Datadog run evaluations automatically on production traces, see Custom LLM-as-a-Judge Evaluations instead.
+
This section covers evaluations you run and submit manually from your application code. To have Datadog run evaluations automatically on production traces, see Custom LLM-as-a-Judge Evaluations instead.
To submit evaluations from your application code, construct the `EvaluatorContext` yourself, call the evaluator, and submit the result with `LLMObs.submit_evaluation()`. You can also submit evaluations through the HTTP API. @@ -775,11 +776,11 @@ When submitting evaluations for [OpenTelemetry-instrumented spans][3], include t {{< partial name="whats-next/whats-next.html" >}} -[1]: /llm_observability/evaluations/external_evaluations -[2]: /llm_observability/instrumentation/api/#evaluations-api -[3]: /llm_observability/instrumentation/otel_instrumentation -[4]: /llm_observability/experiments -[5]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations -[6]: /llm_observability/evaluations/deepeval_evaluations/ -[7]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations#configure-the-prompt -[8]: /llm_observability/evaluations/pydantic_evaluations +[1]: /llm_observability/configure/evaluations/external_evaluations +[2]: /llm_observability/instrument/api/#evaluations-api +[3]: /llm_observability/instrument/otel_instrumentation +[4]: /llm_observability/improve/experiments +[5]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations +[6]: /llm_observability/configure/evaluations/external_evaluations/deepeval/ +[7]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations#configure-the-prompt +[8]: /llm_observability/configure/evaluations/external_evaluations/pydantic diff --git a/hugo/content/en/llm_observability/evaluations/external_evaluations.md b/hugo/content/en/llm_observability/configure/evaluations/external_evaluations/_index.md similarity index 79% rename from hugo/content/en/llm_observability/evaluations/external_evaluations.md rename to hugo/content/en/llm_observability/configure/evaluations/external_evaluations/_index.md index 8ef7373c841..79c18ada96f 100644 --- a/hugo/content/en/llm_observability/evaluations/external_evaluations.md +++ b/hugo/content/en/llm_observability/configure/evaluations/external_evaluations/_index.md @@ -5,6 +5,8 @@ aliases: - /tracing/llm_observability/submit_evaluations - /llm_observability/submit_evaluations - /llm_observability/evaluations/submit_evaluations + - /llm_observability/configure/evaluations/submit_evaluations + - /llm_observability/evaluations/external_evaluations/ further_reading: - link: '/llm_observability/guide/evaluation_developer_guide' tag: 'Documentation' @@ -15,10 +17,10 @@ further_reading: - link: '/llm_observability/setup/api' tag: 'Documentation' text: 'Learn about the Evaluations API' - - link: '/llm_observability/evaluations/submit_nemo_evaluations' + - link: '/llm_observability/configure/evaluations/external_evaluations/nemo' tag: 'Documentation' text: 'Learn about submitting evaluations from NVIDIA NeMo' - - link: '/llm_observability/evaluations/end_user_feedback' + - link: '/llm_observability/configure/evaluations/end_user_feedback' tag: 'Documentation' text: 'Learn about submitting end-user feedback' --- @@ -40,7 +42,7 @@ Evaluation labels must be unique for a given LLM application (ml_app -
For feedback submitted by your users such as thumbs-up or thumbs-down ratings, accepted changes, free-text comments, and other signals, see End-User Feedback.
+
For feedback submitted by your users such as thumbs-up or thumbs-down ratings, accepted changes, free-text comments, and other signals, see End-User Feedback.
## Submitting external evaluations with the SDK @@ -83,7 +85,7 @@ def llm_call(): You can use the evaluations API provided by Agent Observability to send evaluations associated with spans, traces, or sessions to Datadog. See the [Evaluations API][2] for more details on the API specifications. For building reusable evaluators, see the [Evaluation Developer Guide][5]. -To submit evaluations for OpenTelemetry spans directly to the Evaluations API, you must include the source:otel tag in the evaluation. Additionally, span_id and trace_id values must be provided as **decimal** strings. If your OpenTelemetry instrumentation produces hexadecimal IDs, convert them to decimal before submitting. For example, in Python: str(int(hex_span_id, 16)). +To submit evaluations for OpenTelemetry spans directly to the Evaluations API, you must include the source:otel tag in the evaluation. Additionally, span_id and trace_id values must be provided as **decimal** strings. If your OpenTelemetry instrumentation produces hexadecimal IDs, convert them to decimal before submitting. For example, in Python: str(int(hex_span_id, 16)). ### Example @@ -117,6 +119,14 @@ To submit evaluations for }} + {{< nextlink href="/llm_observability/configure/evaluations/external_evaluations/deepeval" >}}DeepEval Evaluations{{< /nextlink >}} + {{< nextlink href="/llm_observability/configure/evaluations/external_evaluations/pydantic" >}}Pydantic Evaluations{{< /nextlink >}} + {{< nextlink href="/llm_observability/configure/evaluations/external_evaluations/nemo" >}}NeMo Evaluations{{< /nextlink >}} +{{< /whatsnext >}} + ## Further Reading {{< partial name="whats-next/whats-next.html" >}} diff --git a/hugo/content/en/llm_observability/evaluations/deepeval_evaluations.md b/hugo/content/en/llm_observability/configure/evaluations/external_evaluations/deepeval.md similarity index 91% rename from hugo/content/en/llm_observability/evaluations/deepeval_evaluations.md rename to hugo/content/en/llm_observability/configure/evaluations/external_evaluations/deepeval.md index 24a21a8be71..3b664dbc27d 100644 --- a/hugo/content/en/llm_observability/evaluations/deepeval_evaluations.md +++ b/hugo/content/en/llm_observability/configure/evaluations/external_evaluations/deepeval.md @@ -1,8 +1,10 @@ --- title: DeepEval Evaluations +aliases: +- /llm_observability/evaluations/deepeval_evaluations/ description: Use DeepEval evaluations with Agent Observability Experiments. further_reading: -- link: "/llm_observability/evaluations/external_evaluations" +- link: "/llm_observability/configure/evaluations/external_evaluations" tag: "Documentation" text: "Submit Evaluations" --- @@ -102,9 +104,9 @@ After you run an experiment with a DeepEval evaluation, you can view the DeepEva {{< partial name="whats-next/whats-next.html" >}} -[1]: /llm_observability/experiments -[2]: /llm_observability/experiments/setup#create-an-experiment +[1]: /llm_observability/improve/experiments +[2]: /llm_observability/improve/experiments/setup#create-an-experiment [3]: https://deepeval.com/docs/metrics-introduction -[4]: /llm_observability/experiments/setup#create-a-dataset -[5]: /llm_observability/experiments/datasets +[4]: /llm_observability/improve/experiments/setup#create-a-dataset +[5]: /llm_observability/improve/datasets [6]: https://github.com/DataDog/llm-observability/blob/main/experiments/eval-integrations/1-deepeval-demo.py \ No newline at end of file diff --git a/hugo/content/en/llm_observability/evaluations/submit_nemo_evaluations.md b/hugo/content/en/llm_observability/configure/evaluations/external_evaluations/nemo.md similarity index 98% rename from hugo/content/en/llm_observability/evaluations/submit_nemo_evaluations.md rename to hugo/content/en/llm_observability/configure/evaluations/external_evaluations/nemo.md index 472289b044b..c76e0611d82 100644 --- a/hugo/content/en/llm_observability/evaluations/submit_nemo_evaluations.md +++ b/hugo/content/en/llm_observability/configure/evaluations/external_evaluations/nemo.md @@ -3,8 +3,9 @@ title: NeMo Evaluations description: Learn how to submit NVIDIA NeMo Evaluator model evaluation scores to Agent Observability to monitor benchmarking results alongside LLM traces. aliases: - /llm_observability/submit_nemo_evaluations/ + - /llm_observability/evaluations/submit_nemo_evaluations/ further_reading: -- link: "/llm_observability/evaluations/external_evaluations" +- link: "/llm_observability/configure/evaluations/external_evaluations" tag: "Documentation" text: "Submit Evaluations" --- diff --git a/hugo/content/en/llm_observability/evaluations/pydantic_evaluations.md b/hugo/content/en/llm_observability/configure/evaluations/external_evaluations/pydantic.md similarity index 93% rename from hugo/content/en/llm_observability/evaluations/pydantic_evaluations.md rename to hugo/content/en/llm_observability/configure/evaluations/external_evaluations/pydantic.md index bbab3849f07..f07cb6bfe06 100644 --- a/hugo/content/en/llm_observability/evaluations/pydantic_evaluations.md +++ b/hugo/content/en/llm_observability/configure/evaluations/external_evaluations/pydantic.md @@ -1,8 +1,10 @@ --- title: Pydantic Evaluations +aliases: +- /llm_observability/evaluations/pydantic_evaluations/ description: Use Pydantic evaluations with Agent Observability Experiments. further_reading: -- link: "/llm_observability/evaluations/external_evaluations" +- link: "/llm_observability/configure/evaluations/external_evaluations" tag: "Documentation" text: "Submit Evaluations" --- @@ -137,9 +139,9 @@ After you run an experiment with a Pydantic evaluation, you can view the Pydanti {{< partial name="whats-next/whats-next.html" >}} -[1]: /llm_observability/experiments -[2]: /llm_observability/experiments/setup#create-an-experiment +[1]: /llm_observability/improve/experiments +[2]: /llm_observability/improve/experiments/setup#create-an-experiment [3]: https://ai.pydantic.dev/evals/ -[4]: /llm_observability/experiments/setup#create-a-dataset -[5]: /llm_observability/experiments/datasets +[4]: /llm_observability/improve/experiments/setup#create-a-dataset +[5]: /llm_observability/improve/datasets [6]: https://github.com/DataDog/llm-observability/blob/main/experiments/eval-integrations/2-pydantic-demo.py diff --git a/hugo/content/en/llm_observability/evaluations/managed_evaluations/language_mismatch.md b/hugo/content/en/llm_observability/configure/evaluations/language_mismatch.md similarity index 88% rename from hugo/content/en/llm_observability/evaluations/managed_evaluations/language_mismatch.md rename to hugo/content/en/llm_observability/configure/evaluations/language_mismatch.md index c89d62546b1..d5704ffd4d4 100644 --- a/hugo/content/en/llm_observability/evaluations/managed_evaluations/language_mismatch.md +++ b/hugo/content/en/llm_observability/configure/evaluations/language_mismatch.md @@ -2,7 +2,7 @@ title: Language Mismatch description: Learn about Datadog's Language Mismatch evaluation. further_reading: -- link: "/llm_observability/terms/" +- link: "/llm_observability/quickstart/terms/" tag: "Documentation" text: "Learn about Agent Observability terms and concepts" - link: "/llm_observability/setup" @@ -10,7 +10,10 @@ further_reading: text: "Learn how to set up Agent Observability" aliases: - /llm_observability/evaluations/quality_evaluations + - /llm_observability/configure/evaluations/quality_evaluations - /llm_observability/evaluations/managed_evaluations/quality_evaluations + - /llm_observability/configure/evaluations/managed_evaluations/quality_evaluations + - /llm_observability/evaluations/managed_evaluations/language_mismatch/ --- This check identifies instances where the LLM generates responses in a different language or dialect than the one used by the user, which can lead to confusion or miscommunication. This check ensures that the LLM's responses are clear, relevant, and appropriate for the user's linguistic preferences and needs. diff --git a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/_index.md b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/_index.md similarity index 97% rename from hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/_index.md rename to hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/_index.md index 5742c8d9ba8..9511543c50f 100644 --- a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/_index.md +++ b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/_index.md @@ -1,5 +1,7 @@ --- title: Custom LLM-as-a-Judge Evaluations +aliases: +- /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/ description: How to create custom LLM-as-a-judge evaluations, and how to use these evaluation results across Agent Observability. further_reading: - link: "https://www.datadoghq.com/blog/manage-ai-cost-and-performance-with-datadog/" @@ -11,13 +13,13 @@ further_reading: - link: "https://www.datadoghq.com/blog/llm-evaluation-framework-best-practices/" tag: "Blog" text: "Building an LLM evaluation framework: best practices" -- link: "/llm_observability/terms/" +- link: "/llm_observability/quickstart/terms/" tag: "Documentation" text: "Learn about Agent Observability terms and concepts" - link: "/llm_observability/setup" tag: "Documentation" text: "Learn how to set up Agent Observability" -- link: "/llm_observability/evaluations/managed_evaluations" +- link: "/llm_observability/configure/evaluations/managed_evaluations" tag: "Documentation" text: "Learn about managed evaluations" - link: "https://huggingface.co/learn/cookbook/llm_judge" @@ -593,21 +595,21 @@ You can use basic CRUD operations to manipulate managed evaluation configs, afte {{< partial name="whats-next/whats-next.html" >}} [1]: https://app.datadoghq.com/llm/evaluations -[2]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/connect_to_account +[2]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/connect_to_account [3]: /events/explorer/facets/ [4]: /monitors/ [5]: https://arxiv.org/abs/2504.00050 -[6]: /llm_observability/evaluations/evaluation_compatibility -[7]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations/ -[8]: /llm_observability/experiments +[6]: /llm_observability/configure/evaluations/compatibility +[7]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations/ +[8]: /llm_observability/improve/experiments [9]: /llm_observability/guide/evaluation_developer_guide/#using-managed-evaluators [10]: https://app.datadoghq.com/dash/integration/llm_evaluations_token_usage [11]: /api/latest/llm-observability/#get-a-custom-evaluator-configuration [12]: /api/latest/llm-observability/#create-or-update-a-custom-evaluator-configuration [13]: /api/latest/llm-observability/#delete-a-custom-evaluator-configuration [14]: /account_management/api-app-keys -[15]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating -[16]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations -[17]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/session_level_evaluations +[15]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/prompt_templating +[16]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/trace_level_evaluations +[17]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/session_level_evaluations [18]: https://docs.cloud.google.com/gemini-enterprise-agent-platform/resources/locations diff --git a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/connect_to_account.md b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/connect_to_account.md similarity index 98% rename from hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/connect_to_account.md rename to hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/connect_to_account.md index cb188a30339..df98c42bec5 100644 --- a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/connect_to_account.md +++ b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/connect_to_account.md @@ -1,8 +1,10 @@ --- title: Connect your LLM provider account +aliases: +- /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/connect_to_account/ description: How to connect to your LLM provider account to support judge LLM based evaluations further_reading: -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations" tag: "Documentation" text: "Learn about custom LLM-as-a-judge evaluations" --- diff --git a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating.md b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/prompt_templating.md similarity index 92% rename from hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating.md rename to hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/prompt_templating.md index 4a57ebf0a5c..e4c7680195b 100644 --- a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating.md +++ b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/prompt_templating.md @@ -1,14 +1,16 @@ --- title: Prompt Templating +aliases: +- /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating/ description: Reference for the templating used in custom LLM-as-a-judge evaluation prompts—variables, array operators, span and trace filters, session paths, and resolution rules. further_reading: -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations" tag: "Documentation" text: "Custom LLM-as-a-Judge Evaluations" -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/session_level_evaluations" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/session_level_evaluations" tag: "Documentation" text: "Session-Level Evaluations" -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/trace_level_evaluations" tag: "Documentation" text: "Trace-Level Evaluations" --- @@ -179,6 +181,6 @@ For example, given a span where `meta.input.messages` is: {{< partial name="whats-next/whats-next.html" >}} -[1]: /llm_observability/instrumentation/sdk/#tracking-user-sessions -[2]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/session_level_evaluations -[3]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations +[1]: /llm_observability/instrument/sdk/#tracking-user-sessions +[2]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/session_level_evaluations +[3]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/trace_level_evaluations diff --git a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/session_level_evaluations.md b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/session_level_evaluations.md similarity index 90% rename from hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/session_level_evaluations.md rename to hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/session_level_evaluations.md index d7e0c985469..6556f009e3a 100644 --- a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/session_level_evaluations.md +++ b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/session_level_evaluations.md @@ -1,17 +1,19 @@ --- title: Session-Level Evaluations +aliases: +- /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/session_level_evaluations/ description: Run a custom LLM-as-a-judge across an entire user session, with examples of when to use session scope over trace or span scope. further_reading: -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations" tag: "Documentation" text: "Custom LLM-as-a-Judge Evaluations" -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/trace_level_evaluations" tag: "Documentation" text: "Trace-Level Evaluations" -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/prompt_templating" tag: "Documentation" text: "Prompt Templating" -- link: "/llm_observability/instrumentation/sdk/#tracking-user-sessions" +- link: "/llm_observability/instrument/sdk/#tracking-user-sessions" tag: "Documentation" text: "Tracking user sessions" --- @@ -20,7 +22,7 @@ A session-level evaluation runs once per [user session][9], with every trace—a Session scope answers questions about agent performance and user behavior across an entire interaction—questions that trace-level and span-level judges cannot answer from a single request or span. -
Session-level evaluations require spans to be tagged with a session_id. See Tracking user sessions to instrument your application.
+
Session-level evaluations require spans to be tagged with a session_id. See Tracking user sessions to instrument your application.
## Configure a session-level evaluation @@ -191,13 +193,13 @@ Configuring evaluations requires the `Agent Observability Write` [permission][4] {{< partial name="whats-next/whats-next.html" >}} [1]: https://app.datadoghq.com/llm/evaluations -[2]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations -[3]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating +[2]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations +[3]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/prompt_templating [4]: /account_management/rbac/permissions/#llm-observability -[5]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/#define-the-evaluation-output +[5]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/#define-the-evaluation-output [6]: /events/explorer/facets/ [7]: /monitors/ -[8]: /llm_observability/evaluations/annotation_queues -[9]: /llm_observability/instrumentation/sdk/#tracking-user-sessions -[10]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations -[11]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations/#goal-completeness +[8]: /llm_observability/investigate/annotation_queues +[9]: /llm_observability/instrument/sdk/#tracking-user-sessions +[10]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/trace_level_evaluations +[11]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations/#goal-completeness diff --git a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations.md b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations.md similarity index 92% rename from hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations.md rename to hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations.md index bc9b12b0053..c2e242729ff 100644 --- a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations.md +++ b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations.md @@ -2,7 +2,7 @@ title: LLM-as-a-Judge Evaluation Templates description: Learn how to create LLM-as-a-Judge evaluations from templates for your LLM applications. further_reading: -- link: "/llm_observability/terms/" +- link: "/llm_observability/quickstart/terms/" tag: "Documentation" text: "Learn about Agent Observability terms and concepts" - link: "/llm_observability/setup" @@ -13,9 +13,14 @@ further_reading: text: "Detect hallucinations in your RAG LLM applications with Datadog LLM Observability" aliases: - /llm_observability/evaluations/agent_evaluations + - /llm_observability/configure/evaluations/agent_evaluations - /llm_observability/evaluations/managed_evaluations/agent_evaluations + - /llm_observability/configure/evaluations/managed_evaluations/agent_evaluations - /llm_observability/evaluations/session_level_evaluations + - /llm_observability/configure/evaluations/session_level_evaluations - /llm_observability/evaluations/managed_evaluations/session_level_evaluations + - /llm_observability/configure/evaluations/managed_evaluations/session_level_evaluations + - /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations/ --- Datadog provides LLM-as-a-judge templates for the following evaluations: [Failure to Answer][16], [Goal Completeness][22], [Hallucination][25], [Prompt Injection][14], [Sentiment][12], [Tool Argument Correctness][23], [Tool Selection][24], [Topic Relevancy][15], and [Toxicity][13]. After you select a template, you can modify any aspect of the evaluation. @@ -393,18 +398,18 @@ result = triage_agent.run_sync( [9]: https://arxiv.org/pdf/2309.11998 [10]: /security/sensitive_data_scanner/ [11]: https://app.datadoghq.com/llm/evaluations -[12]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#sentiment -[13]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#toxicity -[14]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#prompt-injection -[15]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#topic-relevancy -[16]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#failure-to-answer -[17]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/ -[18]: /llm_observability/evaluations/managed_evaluations/#create-new-evaluations -[19]: /llm_observability/evaluations/managed_evaluations/#edit-existing-evaluations +[12]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#sentiment +[13]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#toxicity +[14]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#prompt-injection +[15]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#topic-relevancy +[16]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#failure-to-answer +[17]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/ +[18]: /llm_observability/configure/evaluations/managed_evaluations/#create-new-evaluations +[19]: /llm_observability/configure/evaluations/managed_evaluations/#edit-existing-evaluations [20]: https://github.com/DataDog/llm-observability/blob/main/evaluation_examples/1-tool-selection-demo.py [21]: https://github.com/DataDog/llm-observability/blob/main/evaluation_examples/2-tool-argument-correctness-demo.py -[22]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#goal-completeness -[23]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#tool-argument-correctness -[24]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#tool-selection -[25]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations#hallucination -[26]: /llm_observability/instrumentation/sdk?tab=python#prompt-tracking +[22]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#goal-completeness +[23]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#tool-argument-correctness +[24]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#tool-selection +[25]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations#hallucination +[26]: /llm_observability/instrument/sdk?tab=python#prompt-tracking diff --git a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations.md b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/trace_level_evaluations.md similarity index 94% rename from hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations.md rename to hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/trace_level_evaluations.md index 57d3c9142d1..5b10c0b00f8 100644 --- a/hugo/content/en/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations.md +++ b/hugo/content/en/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/trace_level_evaluations.md @@ -1,14 +1,16 @@ --- title: Trace-Level Evaluations +aliases: +- /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/trace_level_evaluations/ description: Run a custom LLM-as-a-judge across an entire trace, with examples of when to use trace scope over span scope. further_reading: -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations" tag: "Documentation" text: "Custom LLM-as-a-Judge Evaluations" -- link: "/llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating" +- link: "/llm_observability/configure/evaluations/llm_as_a_judge_evaluations/prompt_templating" tag: "Documentation" text: "Prompt Templating" -- link: "/llm_observability/terms/" +- link: "/llm_observability/quickstart/terms/" tag: "Documentation" text: "Agent Observability terms and concepts" --- @@ -219,10 +221,10 @@ Configuring evaluations requires the `Agent Observability Write` [permission][4] {{< partial name="whats-next/whats-next.html" >}} [1]: https://app.datadoghq.com/llm/evaluations -[2]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/template_evaluations -[3]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/prompt_templating +[2]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/template_evaluations +[3]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/prompt_templating [4]: /account_management/rbac/permissions/#llm-observability -[5]: /llm_observability/evaluations/custom_llm_as_a_judge_evaluations/#define-the-evaluation-output +[5]: /llm_observability/configure/evaluations/llm_as_a_judge_evaluations/#define-the-evaluation-output [6]: /events/explorer/facets/ [7]: /monitors/ -[8]: /llm_observability/evaluations/annotation_queues +[8]: /llm_observability/investigate/annotation_queues diff --git a/hugo/content/en/llm_observability/evaluations/managed_evaluations/_index.md b/hugo/content/en/llm_observability/configure/evaluations/managed_evaluations/_index.md similarity index 80% rename from hugo/content/en/llm_observability/evaluations/managed_evaluations/_index.md rename to hugo/content/en/llm_observability/configure/evaluations/managed_evaluations/_index.md index 2cee1a42198..4311889fe6c 100644 --- a/hugo/content/en/llm_observability/evaluations/managed_evaluations/_index.md +++ b/hugo/content/en/llm_observability/configure/evaluations/managed_evaluations/_index.md @@ -5,7 +5,7 @@ further_reading: - link: https://www.datadoghq.com/blog/llm-aws-strands tag: Blog text: Gain visibility into Strands Agents workflows with Datadog LLM Observability -- link: "/llm_observability/terms/" +- link: "/llm_observability/quickstart/terms/" tag: "Documentation" text: "Learn about Agent Observability terms and concepts" - link: "/llm_observability/setup" @@ -13,6 +13,8 @@ further_reading: text: "Learn how to set up Agent Observability" aliases: - /llm_observability/evaluations/ootb_evaluations + - /llm_observability/configure/evaluations/ootb_evaluations + - /llm_observability/evaluations/managed_evaluations/ --- ## Overview @@ -46,6 +48,6 @@ After you click {{< ui >}}Save and Publish{{< /ui >}}, the evaluation goes live. {{< partial name="whats-next/whats-next.html" >}} [1]: https://app.datadoghq.com/llm/evaluations -[2]: /llm_observability/evaluations/evaluation_compatibility -[3]: /llm_observability/evaluations/managed_evaluations/quality_evaluations#language-mismatch -[4]: /llm_observability/evaluations/managed_evaluations/security_and_safety_evaluations#sensitive-data-scanning +[2]: /llm_observability/configure/evaluations/compatibility +[3]: /llm_observability/configure/evaluations/managed_evaluations/quality_evaluations#language-mismatch +[4]: /llm_observability/configure/evaluations/managed_evaluations/security_and_safety_evaluations#sensitive-data-scanning diff --git a/hugo/content/en/llm_observability/evaluations/managed_evaluations/security_and_safety_evaluations.md b/hugo/content/en/llm_observability/configure/evaluations/managed_evaluations/security_and_safety_evaluations.md similarity index 85% rename from hugo/content/en/llm_observability/evaluations/managed_evaluations/security_and_safety_evaluations.md rename to hugo/content/en/llm_observability/configure/evaluations/managed_evaluations/security_and_safety_evaluations.md index aaadea928bb..93c3daf3041 100644 --- a/hugo/content/en/llm_observability/evaluations/managed_evaluations/security_and_safety_evaluations.md +++ b/hugo/content/en/llm_observability/configure/evaluations/managed_evaluations/security_and_safety_evaluations.md @@ -2,7 +2,7 @@ title: Sensitive Data Scanner description: Learn how to configure managed evaluations for your LLM applications. further_reading: -- link: "/llm_observability/terms/" +- link: "/llm_observability/quickstart/terms/" tag: "Documentation" text: "Learn about Agent Observability terms and concepts" - link: "/llm_observability/setup" @@ -10,6 +10,8 @@ further_reading: text: "Learn how to set up Agent Observability" aliases: - /llm_observability/evaluations/sensitive_data_scanner + - /llm_observability/configure/evaluations/sensitive_data_scanner + - /llm_observability/evaluations/managed_evaluations/security_and_safety_evaluations/ --- This check ensures that sensitive information is handled appropriately and securely, reducing the risk of data breaches or unauthorized access. diff --git a/hugo/content/en/llm_observability/monitoring/prompt_management.md b/hugo/content/en/llm_observability/configure/prompt_management.md similarity index 95% rename from hugo/content/en/llm_observability/monitoring/prompt_management.md rename to hugo/content/en/llm_observability/configure/prompt_management.md index 246b082aaa1..5231d209b2b 100644 --- a/hugo/content/en/llm_observability/monitoring/prompt_management.md +++ b/hugo/content/en/llm_observability/configure/prompt_management.md @@ -1,15 +1,17 @@ --- title: Prompt Management +aliases: +- /llm_observability/monitoring/prompt_management/ description: Create, version, and retrieve managed prompts in Python applications with Prompt Management. further_reading: - - link: "/llm_observability/monitoring/prompt_tracking" + - link: "/llm_observability/instrument/prompt_tracking" tag: "Documentation" text: "Prompt Tracking" - - link: "/llm_observability/playground" + - link: "/llm_observability/improve/playground" tag: "Documentation" text: "Playground" - - link: "/llm_observability/instrumentation/sdk/?tab=python" + - link: "/llm_observability/instrument/sdk/?tab=python" tag: "Documentation" text: "Agent Observability SDK" @@ -46,7 +48,7 @@ pip install --upgrade ddtrace Integrate a managed prompt with a coding agent of your choice by pasting in the following prompt: ```text -Follow the instructions at https://docs.datadoghq.com/llm_observability/instrumentation/agentic.md to integrate the Datadog managed prompt into this application for environment and track its use in Agent Observability. +Follow the instructions at https://docs.datadoghq.com/llm_observability/instrument/agentic.md to integrate the Datadog managed prompt into this application for environment and track its use in Agent Observability. Prompt variables: @@ -323,12 +325,12 @@ To retrieve an exact version regardless of any targeting rule, pass `version` as {{< partial name="whats-next/whats-next.html" >}} -[1]: /llm_observability/monitoring/prompt_tracking +[1]: /llm_observability/instrument/prompt_tracking [2]: /getting_started/site/ [3]: /account_management/api-app-keys/#api-keys [4]: /account_management/api-app-keys/#application-keys -[5]: /llm_observability/instrumentation/sdk/?tab=python -[6]: /llm_observability/instrumentation/auto_instrumentation/?tab=python -[7]: /llm_observability/instrumentation/sdk/?tab=python#manual-instrumentation +[5]: /llm_observability/instrument/sdk/?tab=python +[6]: /llm_observability/instrument/auto_instrumentation/?tab=python +[7]: /llm_observability/instrument/sdk/?tab=python#manual-instrumentation [8]: /api/latest/llm-observability/ [9]: /api/latest/feature-flags/list-environments/ diff --git a/hugo/content/en/llm_observability/data_security_and_rbac.md b/hugo/content/en/llm_observability/data_privacy_security_and_rbac.md similarity index 94% rename from hugo/content/en/llm_observability/data_security_and_rbac.md rename to hugo/content/en/llm_observability/data_privacy_security_and_rbac.md index 9e2cd30fc14..b7062d174c4 100644 --- a/hugo/content/en/llm_observability/data_security_and_rbac.md +++ b/hugo/content/en/llm_observability/data_privacy_security_and_rbac.md @@ -1,5 +1,7 @@ --- -title: Data Security and RBAC +title: Data Privacy, Security, and RBAC +aliases: +- /llm_observability/data_security_and_rbac/ description: Restrict access to sensitive Agent Observability data using data access controls, redact data with span processors, and integrate with Sensitive Data Scanner. further_reading: - link: "/account_management/rbac/data_access" @@ -38,6 +40,6 @@ By proactively scanning for sensitive data, Agent Observability ensures that con {{< partial name="whats-next/whats-next.html" >}} [1]: /account_management/rbac/data_access -[2]: /llm_observability/instrumentation/sdk/#span-processing +[2]: /llm_observability/instrument/sdk/#span-processing [3]: /security/sensitive_data_scanner/ diff --git a/hugo/content/en/llm_observability/guide/_index.md b/hugo/content/en/llm_observability/guide/_index.md index fe052902476..5396a5d9c56 100644 --- a/hugo/content/en/llm_observability/guide/_index.md +++ b/hugo/content/en/llm_observability/guide/_index.md @@ -9,14 +9,15 @@ cascade: --- {{< whatsnext desc="Agent Observability Guides:" >}} - {{< nextlink href="/llm_observability/quickstart#trace-an-llm-application" >}}Trace an LLM Application{{< /nextlink >}} - {{< nextlink href="/llm_observability/quickstart#trace-an-llm-application-in-aws-lambda" >}}Trace an LLM Application in AWS Lambda{{< /nextlink >}} - {{< nextlink href="/llm_observability/trace_proxy_services" >}}Trace Proxy and Gateway Services{{< /nextlink >}} - {{< nextlink href="/llm_observability/evaluations/" >}}Evaluations{{< /nextlink >}} - {{< nextlink href="/llm_observability/guide/llm_observability_and_apm" >}}Using Agent Observability and APM{{< /nextlink >}} - {{< nextlink href="/llm_observability/monitoring/mcp_client" >}}Monitor MCP Clients{{< /nextlink >}} + {{< nextlink href="/llm_observability/quickstart/" >}}Trace an LLM Application{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/trace-an-llm-application-in-aws-lambda" >}}Trace an LLM Application in AWS Lambda{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/monitor_proxy_services" >}}Trace Proxy and Gateway Services{{< /nextlink >}} + {{< nextlink href="/llm_observability/configure/evaluations/" >}}Evaluations{{< /nextlink >}} + {{< nextlink href="/llm_observability/instrument/agent_observability_and_apm" >}}Using Agent Observability and APM{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/monitor_mcp_client" >}}Monitor MCP Clients{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/crewai_guide" >}}Using the Datadog-CrewAI integration for Agent Observability{{< /nextlink >}} {{< nextlink href="/llm_observability/guide/nextjs_guide" >}}Instrument a Next.js Application for Agent Observability{{< /nextlink >}} - {{< nextlink href="/llm_observability/guide/evaluation_developer_guide" >}}Evaluation Developer Guide: Build custom evaluators{{< /nextlink >}} - {{< nextlink href="/llm_observability/guide/claude_code_skills" >}}Analyze LLM Applications with Claude Code Skills{{< /nextlink >}} + {{< nextlink href="/llm_observability/guide/agent_monitoring" >}}Agent Monitoring{{< /nextlink >}} + {{< nextlink href="/llm_observability/configure/evaluations/evaluation_developer_guide" >}}Evaluation Developer Guide: Build custom evaluators{{< /nextlink >}} + {{< nextlink href="/llm_observability/build_with_ai/claude_code_skills" >}}Analyze LLM Applications with Claude Code Skills{{< /nextlink >}} {{< /whatsnext >}} diff --git a/hugo/content/en/llm_observability/monitoring/agent_monitoring.md b/hugo/content/en/llm_observability/guide/agent_monitoring.md similarity index 97% rename from hugo/content/en/llm_observability/monitoring/agent_monitoring.md rename to hugo/content/en/llm_observability/guide/agent_monitoring.md index 19a2553cb48..92e30233e68 100644 --- a/hugo/content/en/llm_observability/monitoring/agent_monitoring.md +++ b/hugo/content/en/llm_observability/guide/agent_monitoring.md @@ -3,6 +3,7 @@ title: Agent Monitoring description: Monitor, troubleshoot, and improve agentic applications built on OpenAI Agents SDK, LangGraph, or CrewAI using Agent Observability. aliases: - /llm_observability/agent_monitoring + - /llm_observability/monitoring/agent_monitoring/ further_reading: - link: "https://www.datadoghq.com/blog/openai-agents-llm-observability/" tag: "Blog" diff --git a/hugo/content/en/llm_observability/guide/crewai_guide.md b/hugo/content/en/llm_observability/guide/crewai_guide.md index 7fc18f2e783..f7b1cc81acf 100644 --- a/hugo/content/en/llm_observability/guide/crewai_guide.md +++ b/hugo/content/en/llm_observability/guide/crewai_guide.md @@ -97,8 +97,8 @@ Additionally, you can view the execution graph view of the trace, which shows th {{< img src="llm_observability/guides/crewai/execution_graph.png" alt="The same trace in Agent Observability, toggled to 'Execution Graph' view. Control and data flow of the trace is displayed." style="width:100%;" >}} [1]: https://docs.crewai.com/en/introduction -[2]: /llm_observability/instrumentation/auto_instrumentation -[3]: /llm_observability/monitoring/agent_monitoring +[2]: /llm_observability/instrument/auto_instrumentation +[3]: /llm_observability/guide/agent_monitoring [4]: /account_management/api-app-keys/#api-keys -[5]: /llm_observability/instrumentation/sdk?tab=python#application-naming-guidelines +[5]: /llm_observability/instrument/sdk?tab=python#application-naming-guidelines [6]: https://app.datadoghq.com/llm/traces diff --git a/hugo/content/en/llm_observability/monitoring/mcp_client.md b/hugo/content/en/llm_observability/guide/monitor_mcp_client.md similarity index 98% rename from hugo/content/en/llm_observability/monitoring/mcp_client.md rename to hugo/content/en/llm_observability/guide/monitor_mcp_client.md index e0e9396df85..c1b1a988cc0 100644 --- a/hugo/content/en/llm_observability/monitoring/mcp_client.md +++ b/hugo/content/en/llm_observability/guide/monitor_mcp_client.md @@ -1,5 +1,7 @@ --- title: MCP Clients +aliases: +- /llm_observability/monitoring/mcp_client/ description: Learn how to instrument and monitor MCP clients with Agent Observability. further_reading: @@ -45,7 +47,7 @@ If you are using the official MCP Python SDK to connect to an MCP server with an For additional configuration options, see the [Agent Observability SDK setup guide][1]. -[1]: /llm_observability/instrumentation/sdk?tab=python#setup +[1]: /llm_observability/instrument/sdk?tab=python#setup {{% /tab %}} {{% tab "Node.js" %}} @@ -75,7 +77,7 @@ If you are using the official MCP JavaScript SDK to connect to an MCP server wit For additional configuration options, see the [Agent Observability SDK setup guide][1]. -[1]: /llm_observability/instrumentation/sdk?tab=nodejs#setup +[1]: /llm_observability/instrument/sdk?tab=nodejs#setup {{% /tab %}} {{< /tabs >}} @@ -458,5 +460,5 @@ java -javaagent:dd-java-agent.jar -Ddd.llmobs.enabled=true -Ddd.llmobs.ml-app=}} [1]: https://github.com/modelcontextprotocol/python-sdk -[2]: /llm_observability/instrumentation/sdk +[2]: /llm_observability/instrument/sdk [3]: https://github.com/modelcontextprotocol/typescript-sdk \ No newline at end of file diff --git a/hugo/content/en/llm_observability/trace_proxy_services.md b/hugo/content/en/llm_observability/guide/monitor_proxy_services.md similarity index 99% rename from hugo/content/en/llm_observability/trace_proxy_services.md rename to hugo/content/en/llm_observability/guide/monitor_proxy_services.md index f6003723c40..9b324e104f9 100644 --- a/hugo/content/en/llm_observability/trace_proxy_services.md +++ b/hugo/content/en/llm_observability/guide/monitor_proxy_services.md @@ -1,5 +1,7 @@ --- title: Tracing Proxy Services +aliases: +- /llm_observability/trace_proxy_services/ description: Learn how to use Agent Observability to trace LLM calls through proxy or gateway services as part of a complete end-to-end trace. --- diff --git a/hugo/content/en/llm_observability/guide/nextjs_guide.md b/hugo/content/en/llm_observability/guide/nextjs_guide.md index 4279617b30c..76f90e2d758 100644 --- a/hugo/content/en/llm_observability/guide/nextjs_guide.md +++ b/hugo/content/en/llm_observability/guide/nextjs_guide.md @@ -2,10 +2,10 @@ title: Instrument a Next.js Application for Agent Observability description: Learn how to set up Agent Observability in a Next.js application using auto-instrumentation to monitor and trace your LLM calls. further_reading: - - link: '/llm_observability/instrumentation/auto_instrumentation' + - link: '/llm_observability/instrument/auto_instrumentation' tag: 'Documentation' text: 'Supported auto-instrumentation frameworks and libraries' - - link: '/llm_observability/instrumentation/sdk' + - link: '/llm_observability/instrument/sdk' tag: 'Documentation' text: 'Agent Observability SDK Reference for manual instrumentation' --- @@ -125,9 +125,9 @@ Each trace shows token usage, model, latency, and the full input and output for The Datadog SDK requires the Node.js runtime. Ensure your Route Handlers and Server Actions that make LLM calls do not set `export const runtime = 'edge'`. [1]: /llm_observability/ -[2]: /llm_observability/instrumentation/auto_instrumentation/?tab=nodejs +[2]: /llm_observability/instrument/auto_instrumentation/?tab=nodejs [3]: https://app.datadoghq.com/organization-settings/api-keys [4]: https://nextjs.org/docs/app/building-your-application/optimizing/instrumentation -[5]: /llm_observability/instrumentation/auto_instrumentation/?tab=nodejs +[5]: /llm_observability/instrument/auto_instrumentation/?tab=nodejs [6]: https://nextjs.org/docs/app/building-your-application/routing/route-handlers [7]: https://app.datadoghq.com/llm/traces diff --git a/hugo/content/en/llm_observability/guide/trace-an-llm-application-in-aws-lambda.md b/hugo/content/en/llm_observability/guide/trace-an-llm-application-in-aws-lambda.md new file mode 100644 index 00000000000..c5052f21c40 --- /dev/null +++ b/hugo/content/en/llm_observability/guide/trace-an-llm-application-in-aws-lambda.md @@ -0,0 +1,76 @@ +--- +title: Trace an LLM Application in AWS Lambda +description: Instrument an existing AWS Lambda function with Agent Observability by using the Datadog Extension and language layers. +further_reading: +- link: "/llm_observability/instrument/sdk/" + tag: "Documentation" + text: "Agent Observability SDK Reference" +--- + +To instrument an existing AWS Lambda function with Agent Observability, you can use the Datadog Extension and respective language layers. + +1. Open a Cloudshell in the AWS console. +2. Install the Datadog CLI client +```shell +npm install -g @datadog/datadog-ci +``` +3. Set the Datadog API key and site +```shell +export DD_API_KEY= +export DD_SITE= +``` +If you already have or prefer to use a secret in Secrets Manager, you can set the API key by using the secret ARN: +```shell +export DATADOG_API_KEY_SECRET_ARN= +``` +4. Install your Lambda function with Agent Observability (this requires at least version 77 of the Datadog Extension layer) +{{< tabs >}} +{{% tab "Python" %}} +```shell +datadog-ci lambda instrument -f -r -v {{< latest-lambda-layer-version layer="python" >}} -e {{< latest-lambda-layer-version layer="extension" >}} --llmobs +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```shell +datadog-ci lambda instrument -f -r -v {{< latest-lambda-layer-version layer="node" >}} -e {{< latest-lambda-layer-version layer="extension" >}} --llmobs +``` +{{% /tab %}} + +{{% tab "Java" %}} +```shell +datadog-ci lambda instrument -f -r -v {{< latest-lambda-layer-version layer="dd-trace-java" >}} -e {{< latest-lambda-layer-version layer="extension" >}} --llmobs +``` +{{% /tab %}} +{{< /tabs >}} + +4. Invoke your Lambda function and verify that Agent Observability traces are visible in the Datadog UI. + +Manually flush Agent Observability traces by using the `flush` method before the Lambda function returns. + +{{< tabs >}} +{{% tab "Python" %}} +```python +from ddtrace.llmobs import LLMObs +def handler(): + # function body + LLMObs.flush() +``` +{{% /tab %}} + +{{% tab "Node.js" %}} +```javascript +import tracer from 'dd-trace'; +const llmobs = tracer.llmobs; + +export const handler = async (event) => { + // your function body + llmobs.flush(); +}; +``` +{{% /tab %}} +{{< /tabs >}} + +## Further Reading + +{{< partial name="whats-next/whats-next.html" >}} diff --git a/hugo/content/en/llm_observability/improve/_index.md b/hugo/content/en/llm_observability/improve/_index.md new file mode 100644 index 00000000000..316a9371f60 --- /dev/null +++ b/hugo/content/en/llm_observability/improve/_index.md @@ -0,0 +1,12 @@ +--- +title: Improve +description: Iterate on your LLM applications and agents with the playground, datasets, and experiments. +--- + +Iterate on your LLM application with datasets, experiments, and the playground. Test prompt and model changes against saved inputs, and compare the results before you ship them. + +{{< whatsnext desc="Iterate on your application:" >}} + {{< nextlink href="/llm_observability/improve/playground" >}}Playground{{< /nextlink >}} + {{< nextlink href="/llm_observability/improve/datasets" >}}Datasets{{< /nextlink >}} + {{< nextlink href="/llm_observability/improve/experiments" >}}Experiments{{< /nextlink >}} +{{< /whatsnext >}} diff --git a/hugo/content/en/llm_observability/experiments/datasets.md b/hugo/content/en/llm_observability/improve/datasets.md similarity index 98% rename from hugo/content/en/llm_observability/experiments/datasets.md rename to hugo/content/en/llm_observability/improve/datasets.md index bedce559f70..e3b5b616e41 100644 --- a/hugo/content/en/llm_observability/experiments/datasets.md +++ b/hugo/content/en/llm_observability/improve/datasets.md @@ -1,8 +1,10 @@ --- title: Datasets +aliases: +- /llm_observability/experiments/datasets/ description: Using datasets in Agent Observability Experiments, including how to create, retrieve, and manage datasets, as well as information about versioning. further_reading: - - link: /llm_observability/monitoring/automation_rules + - link: /llm_observability/configure/automation_rules tag: Documentation text: Route traces into datasets automatically with Automation Rules --- @@ -128,7 +130,7 @@ After creating an automation, manage it from [{{< ui >}}AI Observability{{< /ui [2]: https://app.datadoghq.com/llm/traces [3]: https://app.datadoghq.com/llm/settings/automations -[5]: /llm_observability/monitoring/automation_rules/#supported-filter-fields +[5]: /llm_observability/configure/automation_rules/#supported-filter-fields {{% /tab %}} {{< /tabs >}} diff --git a/hugo/content/en/llm_observability/experiments/_index.md b/hugo/content/en/llm_observability/improve/experiments/_index.md similarity index 96% rename from hugo/content/en/llm_observability/experiments/_index.md rename to hugo/content/en/llm_observability/improve/experiments/_index.md index 8130082602d..89c3e29aac6 100644 --- a/hugo/content/en/llm_observability/experiments/_index.md +++ b/hugo/content/en/llm_observability/improve/experiments/_index.md @@ -3,8 +3,9 @@ title: Experiments description: An overview of Agent Observability Experiments feature. aliases: - /llm_observability/experiments_preview + - /llm_observability/experiments/ further_reading: - - link: /llm_observability/experiments/setup + - link: /llm_observability/improve/experiments/setup tag: "Documentation" text: Set up and use Agent Observability Experiments - link: "https://www.datadoghq.com/blog/llm-experiments/" diff --git a/hugo/content/en/llm_observability/experiments/advanced_runs.md b/hugo/content/en/llm_observability/improve/experiments/advanced_runs.md similarity index 98% rename from hugo/content/en/llm_observability/experiments/advanced_runs.md rename to hugo/content/en/llm_observability/improve/experiments/advanced_runs.md index 50482c220cd..5aa92e7207d 100644 --- a/hugo/content/en/llm_observability/experiments/advanced_runs.md +++ b/hugo/content/en/llm_observability/improve/experiments/advanced_runs.md @@ -1,5 +1,7 @@ --- title: Advanced Experiment Runs +aliases: +- /llm_observability/experiments/advanced_runs/ description: Run experiments multiple times to account for model variability on a subset of your dataset, and automate experiment execution in CI/CD pipelines. --- @@ -268,6 +270,6 @@ GET /api/v2/llm-obs/v1/experiments?filter[experiment]=my-pipeline&filter[metadat GET /api/v2/llm-obs/v1/experiments?filter[experiment]=my-pipeline&filter[metadata]={"commit":"abc123"} ``` -[1]: /llm_observability/instrumentation/sdk?tab=python -[2]: /llm_observability/experiments/api +[1]: /llm_observability/instrument/sdk?tab=python +[2]: /llm_observability/improve/experiments/api [3]: https://app.datadoghq.com/llm/experiments diff --git a/hugo/content/en/llm_observability/experiments/analyzing_results.md b/hugo/content/en/llm_observability/improve/experiments/analyzing_results.md similarity index 99% rename from hugo/content/en/llm_observability/experiments/analyzing_results.md rename to hugo/content/en/llm_observability/improve/experiments/analyzing_results.md index 9e1d942d4c6..3fe75ce9d25 100644 --- a/hugo/content/en/llm_observability/experiments/analyzing_results.md +++ b/hugo/content/en/llm_observability/improve/experiments/analyzing_results.md @@ -1,5 +1,7 @@ --- title: Analyze Your Experiments Results +aliases: +- /llm_observability/experiments/analyzing_results/ description: How to analyze Agent Observability Experiments results. --- diff --git a/hugo/content/en/llm_observability/experiments/api.md b/hugo/content/en/llm_observability/improve/experiments/api.md similarity index 99% rename from hugo/content/en/llm_observability/experiments/api.md rename to hugo/content/en/llm_observability/improve/experiments/api.md index 3826782186f..cf99ea1320b 100644 --- a/hugo/content/en/llm_observability/experiments/api.md +++ b/hugo/content/en/llm_observability/improve/experiments/api.md @@ -1,5 +1,7 @@ --- title: Experiments API +aliases: +- /llm_observability/experiments/api/ description: Reference the Agent Observability Experiments HTTP API. --- @@ -556,6 +558,6 @@ Empty body on success. {{% /collapse-content %}} -[1]: /llm_observability/experiments/setup +[1]: /llm_observability/improve/experiments/setup [2]: https://github.com/DataDog/llm-observability/tree/main/experiments [3]: https://www.postman.com/ diff --git a/hugo/content/en/llm_observability/experiments/prompt_optimization.md b/hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md similarity index 99% rename from hugo/content/en/llm_observability/experiments/prompt_optimization.md rename to hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md index e42e4c0cf35..59e58904473 100644 --- a/hugo/content/en/llm_observability/experiments/prompt_optimization.md +++ b/hugo/content/en/llm_observability/improve/experiments/prompt_optimization.md @@ -1,12 +1,14 @@ --- title: Prompt Optimization +aliases: +- /llm_observability/experiments/prompt_optimization/ description: Automatically improve LLM prompts through iterative refinement and AI-powered evaluation. private: true further_reading: - - link: /llm_observability/experiments/setup + - link: /llm_observability/improve/experiments/setup tag: "Documentation" text: Set up and use Agent Observability Experiments - - link: /llm_observability/experiments/datasets + - link: /llm_observability/improve/datasets tag: "Documentation" text: Create and manage datasets - link: "https://www.datadoghq.com/blog/llm-experiments/" @@ -519,5 +521,5 @@ Avoid using cheaper models (GPT-3.5-turbo, Claude Haiku) as they lack the reason [1]: https://github.com/DataDog/dd-trace-py [2]: https://app.datadoghq.com/organization-settings/api-keys -[3]: /llm_observability/experiments/datasets?tab=csv +[3]: /llm_observability/improve/datasets?tab=csv [4]: https://github.com/DataDog/llm-observability/tree/main/experiments/scripts diff --git a/hugo/content/en/llm_observability/experiments/setup.md b/hugo/content/en/llm_observability/improve/experiments/setup.md similarity index 97% rename from hugo/content/en/llm_observability/experiments/setup.md rename to hugo/content/en/llm_observability/improve/experiments/setup.md index cfd9aa73246..0af87588630 100644 --- a/hugo/content/en/llm_observability/experiments/setup.md +++ b/hugo/content/en/llm_observability/improve/experiments/setup.md @@ -1,5 +1,7 @@ --- title: Setup and Usage +aliases: +- /llm_observability/experiments/setup/ description: How to set up Agent Observability Experiments and start running experiments. --- @@ -338,9 +340,9 @@ To create an experiment: Note: LLM Experiments traces are retained for 90 days. -[1]: /llm_observability/experiments/datasets -[2]: /llm_observability/instrumentation/custom_instrumentation?tab=decorators#trace-an-llm-application -[3]: /llm_observability/instrumentation/auto_instrumentation?tab=python +[1]: /llm_observability/improve/datasets +[2]: /llm_observability/instrument/custom_instrumentation?tab=decorators#trace-an-llm-application +[3]: /llm_observability/instrument/auto_instrumentation?tab=python [4]: /llm_observability/guide/evaluation_developer_guide -[5]: /llm_observability/monitoring/llm_observability_and_apm/ -[6]: /llm_observability/instrumentation/otel_instrumentation +[5]: /llm_observability/instrument/agent_observability_and_apm/ +[6]: /llm_observability/instrument/otel_instrumentation diff --git a/hugo/content/en/llm_observability/playground.md b/hugo/content/en/llm_observability/improve/playground.md similarity index 97% rename from hugo/content/en/llm_observability/playground.md rename to hugo/content/en/llm_observability/improve/playground.md index 66d255142a6..b13bf76c12c 100644 --- a/hugo/content/en/llm_observability/playground.md +++ b/hugo/content/en/llm_observability/improve/playground.md @@ -1,14 +1,16 @@ --- title: Playground +aliases: +- /llm_observability/playground/ description: Use the Agent Observability Playground to test prompts interactively against any connected model provider and run experiments with evaluators from a browser-based interface. further_reading: - - link: "/llm_observability/experiments" + - link: "/llm_observability/improve/experiments" tag: "Documentation" text: "Agent Observability Experiments" - - link: "/llm_observability/experiments/datasets" + - link: "/llm_observability/improve/datasets" tag: "Documentation" text: "Datasets" - - link: "/llm_observability/experiments/analyzing_results" + - link: "/llm_observability/improve/experiments/analyzing_results" tag: "Documentation" text: "Analyzing Experiment Results" --- @@ -140,4 +142,4 @@ The experiment runs across all records in the dataset—not only the 20-record p [1]: https://app.datadoghq.com/llm/playground [2]: https://app.datadoghq.com/llm/testing/experiments [3]: https://app.datadoghq.com/llm/settings/integrations -[4]: /llm_observability/experiments/datasets +[4]: /llm_observability/improve/datasets diff --git a/hugo/content/en/llm_observability/instrumentation/_index.md b/hugo/content/en/llm_observability/instrument/_index.md similarity index 92% rename from hugo/content/en/llm_observability/instrumentation/_index.md rename to hugo/content/en/llm_observability/instrument/_index.md index b36b51606f3..e30ac5b4d10 100644 --- a/hugo/content/en/llm_observability/instrumentation/_index.md +++ b/hugo/content/en/llm_observability/instrument/_index.md @@ -1,5 +1,7 @@ --- -title: Agent Observability Instrumentation +title: Instrument +aliases: +- /llm_observability/instrumentation/ description: Overview of instrumentation options for Agent Observability, including SDK-based and API-based approaches for Python, Node.js, and Java. further_reading: - link: '/llm_observability/auto_instrumentation' @@ -76,9 +78,9 @@ To learn more, see the [HTTP API Documentation][3]. [1]: /llm_observability/auto_instrumentation -[2]: /llm_observability/instrumentation/sdk +[2]: /llm_observability/instrument/sdk [3]: /llm_observability/setup/api -[4]: /llm_observability/instrumentation/api/?tab=model#spans-api -[5]: /llm_observability/instrumentation/api/?tab=model#evaluations-api -[6]: /llm_observability/instrumentation/sdk#command-line-setup -[7]: /llm_observability/instrumentation/sdk#in-code-setup +[4]: /llm_observability/instrument/api/?tab=model#spans-api +[5]: /llm_observability/instrument/api/?tab=model#evaluations-api +[6]: /llm_observability/instrument/sdk#command-line-setup +[7]: /llm_observability/instrument/sdk#in-code-setup diff --git a/hugo/content/en/llm_observability/monitoring/llm_observability_and_apm.md b/hugo/content/en/llm_observability/instrument/agent_observability_and_apm.md similarity index 95% rename from hugo/content/en/llm_observability/monitoring/llm_observability_and_apm.md rename to hugo/content/en/llm_observability/instrument/agent_observability_and_apm.md index 48c0e3e21d9..c82dcdad251 100644 --- a/hugo/content/en/llm_observability/monitoring/llm_observability_and_apm.md +++ b/hugo/content/en/llm_observability/instrument/agent_observability_and_apm.md @@ -1,10 +1,11 @@ --- -title: Correlating Agent Observability and APM +title: Agent Observability and APM description: Learn how to navigate between Agent Observability spans and APM spans so you can gain insights into LLM-specific operations and your broader application ecosystem. aliases: - /llm_observability/guide/llm_observability_and_apm + - /llm_observability/monitoring/llm_observability_and_apm/ further_reading: -- link: "/llm_observability/terms/" +- link: "/llm_observability/quickstart/terms/" tag: "Documentation" text: "Learn about Agent Observability Spans" - link: "/glossary/#span/" @@ -67,5 +68,5 @@ To access LLM-specific insights, select an APM span in the [Trace Explorer][5] a [3]: /llm_observability/setup/api/ [4]: https://app.datadoghq.com/llm/traces [5]: https://app.datadoghq.com/apm/traces -[6]: /llm_observability/terms/#spans +[6]: /llm_observability/quickstart/terms/#spans [7]: /tracing \ No newline at end of file diff --git a/hugo/content/en/llm_observability/instrumentation/agentic/_index.md b/hugo/content/en/llm_observability/instrument/agentic/_index.md similarity index 81% rename from hugo/content/en/llm_observability/instrumentation/agentic/_index.md rename to hugo/content/en/llm_observability/instrument/agentic/_index.md index cc620d7b896..f8fa8218799 100644 --- a/hugo/content/en/llm_observability/instrumentation/agentic/_index.md +++ b/hugo/content/en/llm_observability/instrument/agentic/_index.md @@ -1,5 +1,7 @@ --- title: Agentic Instrumentation +aliases: +- /llm_observability/instrumentation/agentic/ --- ## Guidelines @@ -63,19 +65,19 @@ Follow the instructions for the detected language: | Language | Instructions | |----------|-------------| -| Python | [Python Application Agentic Instrumentation](/llm_observability/instrumentation/agentic/python.md) | -| Node.js | [Node.js Application Agentic Instrumentation](/llm_observability/instrumentation/agentic/nodejs.md) | -| Java | [Java Application Agentic Instrumentation](/llm_observability/instrumentation/agentic/java.md) | -| OpenTelemetry | [OpenTelemetry Instrumentation](/llm_observability/instrumentation/otel_instrumentation.md) | +| Python | [Python Application Agentic Instrumentation](/llm_observability/instrument/agentic/python.md) | +| Node.js | [Node.js Application Agentic Instrumentation](/llm_observability/instrument/agentic/nodejs.md) | +| Java | [Java Application Agentic Instrumentation](/llm_observability/instrument/agentic/java.md) | +| OpenTelemetry | [OpenTelemetry Instrumentation](/llm_observability/instrument/otel_instrumentation.md) | ## Instrument prompts and offer Prompt Management Prompt Tracking is part of the default Agent Observability instrumentation. Identify the prompt-construction boundary for each selected LLM call and preserve the prompt template separately from its dynamic variables. -1. If the user's request already specifies a Datadog managed prompt ID, follow the [Prompt Management agentic integration guide](/llm_observability/instrumentation/agentic/prompt_management.md). Do not ask whether to use Prompt Management again. +1. If the user's request already specifies a Datadog managed prompt ID, follow the [Prompt Management agentic integration guide](/llm_observability/instrument/agentic/prompt_management.md). Do not ask whether to use Prompt Management again. 2. Otherwise, identify the application's prompts and the dynamic variables used to format them. Preserve the existing provider, model, prompt content, and application behavior. -3. For a supported Python application, tell the user which prompts you identified and ask whether they want to manage those prompts with Datadog. If they agree, follow the [Prompt Management agentic integration guide](/llm_observability/instrumentation/agentic/prompt_management.md) to promote the selected local prompts and replace their local construction with managed-prompt retrieval. -4. If the user declines Prompt Management, or the application language is unsupported, instrument the selected prompts with structured prompt metadata by following the [Prompt Tracking instructions](/llm_observability/monitoring/prompt_tracking). Do not add runtime prompt retrieval. +3. For a supported Python application, tell the user which prompts you identified and ask whether they want to manage those prompts with Datadog. If they agree, follow the [Prompt Management agentic integration guide](/llm_observability/instrument/agentic/prompt_management.md) to promote the selected local prompts and replace their local construction with managed-prompt retrieval. +4. If the user declines Prompt Management, or the application language is unsupported, instrument the selected prompts with structured prompt metadata by following the [Prompt Tracking instructions](/llm_observability/instrument/prompt_tracking). Do not add runtime prompt retrieval. When Prompt Management replaces a local prompt, use the managed prompt's automatic tracking rather than attaching duplicate structured prompt metadata. @@ -92,3 +94,12 @@ https://app.{dd_site}/llm/applications?query=@ml_app:{application_name} Fill in the provided values: 1. `dd_site` - if the value was provided for [Datadog site](#datadog-site), use that value. Otherwise, use `datadoghq.com`. 2. `application_name` - use either the provided or inferred value from the [Agent Observability application name](#llm-observability-application-name) section. + +## Language-specific instructions + +{{< whatsnext desc="Instrument an application with a coding agent:" >}} + {{< nextlink href="/llm_observability/instrument/agentic/python" >}}Python{{< /nextlink >}} + {{< nextlink href="/llm_observability/instrument/agentic/nodejs" >}}Node.js{{< /nextlink >}} + {{< nextlink href="/llm_observability/instrument/agentic/java" >}}Java{{< /nextlink >}} + {{< nextlink href="/llm_observability/instrument/agentic/prompt_management" >}}Prompt Management Agentic Integration{{< /nextlink >}} +{{< /whatsnext >}} diff --git a/hugo/content/en/llm_observability/instrumentation/agentic/java.md b/hugo/content/en/llm_observability/instrument/agentic/java.md similarity index 90% rename from hugo/content/en/llm_observability/instrumentation/agentic/java.md rename to hugo/content/en/llm_observability/instrument/agentic/java.md index a7bf3da66c6..82ddeb71aef 100644 --- a/hugo/content/en/llm_observability/instrumentation/agentic/java.md +++ b/hugo/content/en/llm_observability/instrument/agentic/java.md @@ -1,5 +1,7 @@ --- -title: Java Application Agentic Instrumentation +title: Java +aliases: +- /llm_observability/instrumentation/agentic/java/ --- ## Instrumentation diff --git a/hugo/content/en/llm_observability/instrumentation/agentic/nodejs.md b/hugo/content/en/llm_observability/instrument/agentic/nodejs.md similarity index 96% rename from hugo/content/en/llm_observability/instrumentation/agentic/nodejs.md rename to hugo/content/en/llm_observability/instrument/agentic/nodejs.md index c5e9d7ad737..63629b49618 100644 --- a/hugo/content/en/llm_observability/instrumentation/agentic/nodejs.md +++ b/hugo/content/en/llm_observability/instrument/agentic/nodejs.md @@ -1,5 +1,7 @@ --- -title: Node.js Application Agentic Instrumentation +title: Node.js +aliases: +- /llm_observability/instrumentation/agentic/nodejs/ --- **Note**: The Datadog Node.js SDK **only** supports Node.js runtimes. If the user's application is in another Javascript runtime, do **not** use this instruction set. diff --git a/hugo/content/en/llm_observability/instrumentation/agentic/prompt_management.md b/hugo/content/en/llm_observability/instrument/agentic/prompt_management.md similarity index 97% rename from hugo/content/en/llm_observability/instrumentation/agentic/prompt_management.md rename to hugo/content/en/llm_observability/instrument/agentic/prompt_management.md index 704f17cd07e..5c4e1ab78ac 100644 --- a/hugo/content/en/llm_observability/instrumentation/agentic/prompt_management.md +++ b/hugo/content/en/llm_observability/instrument/agentic/prompt_management.md @@ -1,5 +1,7 @@ --- title: Prompt Management Agentic Integration +aliases: +- /llm_observability/instrumentation/agentic/prompt_management/ --- ## Goal @@ -9,11 +11,11 @@ Use an existing Datadog managed prompt or promote an application's local prompt, ## Select the workflow - **Use an existing managed prompt:** If the user's request includes a prompt ID, environment, and variable names, use them without asking whether to enable Prompt Management again. -- **Promote a local prompt:** Use this workflow only after the user opts in through the main [Agentic Instrumentation guide](/llm_observability/instrumentation/agentic). Promote the selected local chat prompt, deploy its first version to the requested environment, and then integrate runtime retrieval. +- **Promote a local prompt:** Use this workflow only after the user opts in through the main [Agentic Instrumentation guide](/llm_observability/instrument/agentic). Promote the selected local chat prompt, deploy its first version to the requested environment, and then integrate runtime retrieval. ## Guidelines -1. Prompt Management runtime retrieval is supported only for Python applications. If the target application is not Python, do not add runtime retrieval. Return to the main [Agentic Instrumentation guide](/llm_observability/instrumentation/agentic) and instrument the selected prompts with structured Prompt Tracking instead. Do not implement a direct HTTP client or rewrite the application in Python. +1. Prompt Management runtime retrieval is supported only for Python applications. If the target application is not Python, do not add runtime retrieval. Return to the main [Agentic Instrumentation guide](/llm_observability/instrument/agentic) and instrument the selected prompts with structured Prompt Tracking instead. Do not implement a direct HTTP client or rewrite the application in Python. 2. Inspect the application before modifying it. Identify its package manager, configuration and secret-management workflow, startup command, existing Datadog instrumentation, LLM provider, prompt construction, and provider call site. 3. For an existing managed prompt, use the prompt ID, environment, and variable names supplied in the user's prompt without asking the user to confirm them. For a promotion, derive a descriptive prompt ID from the selected prompt's purpose and ask the user to confirm it before creating the prompt. 4. If multiple prompt or provider call sites are plausible, ask the user which one to modify and wait for an answer before editing. diff --git a/hugo/content/en/llm_observability/instrumentation/agentic/python.md b/hugo/content/en/llm_observability/instrument/agentic/python.md similarity index 90% rename from hugo/content/en/llm_observability/instrumentation/agentic/python.md rename to hugo/content/en/llm_observability/instrument/agentic/python.md index a1999323d92..eadff51b96b 100644 --- a/hugo/content/en/llm_observability/instrumentation/agentic/python.md +++ b/hugo/content/en/llm_observability/instrument/agentic/python.md @@ -1,5 +1,7 @@ --- -title: Python Application Agentic Instrumentation +title: Python +aliases: +- /llm_observability/instrumentation/agentic/python/ --- ## Instrumentation diff --git a/hugo/content/en/llm_observability/instrumentation/api.md b/hugo/content/en/llm_observability/instrument/api.md similarity index 99% rename from hugo/content/en/llm_observability/instrumentation/api.md rename to hugo/content/en/llm_observability/instrument/api.md index 5f2c2a8a1a3..db6fd70b1ea 100644 --- a/hugo/content/en/llm_observability/instrumentation/api.md +++ b/hugo/content/en/llm_observability/instrument/api.md @@ -1,10 +1,11 @@ --- -title: HTTP API Reference +title: API Reference description: Reference documentation for the Agent Observability HTTP API, used to send LLM traces and spans to Datadog from applications in any language. aliases: - /tracing/llm_observability/api - /llm_observability/api - /llm_observability/setup/api + - /llm_observability/instrumentation/api/ further_reading: - link: https://www.datadoghq.com/blog/llm-otel-semantic-convention @@ -699,7 +700,7 @@ For feedback events, provide exactly one of `span_id`, `trace_id`, `session_id`, {{< partial name="whats-next/whats-next.html" >}} [1]: /llm_observability/setup/sdk/ -[2]: /llm_observability/terms/ +[2]: /llm_observability/quickstart/terms/ [3]: /getting_started/tagging/ -[4]: /llm_observability/evaluations/end_user_feedback -[5]: /llm_observability/instrumentation/sdk/?tab=python#enriching-spans +[4]: /llm_observability/configure/evaluations/end_user_feedback +[5]: /llm_observability/instrument/sdk/?tab=python#enriching-spans diff --git a/hugo/content/en/llm_observability/instrumentation/auto_instrumentation.md b/hugo/content/en/llm_observability/instrument/auto_instrumentation.md similarity index 97% rename from hugo/content/en/llm_observability/instrumentation/auto_instrumentation.md rename to hugo/content/en/llm_observability/instrument/auto_instrumentation.md index 11aeea66e03..fdbad78f599 100644 --- a/hugo/content/en/llm_observability/instrumentation/auto_instrumentation.md +++ b/hugo/content/en/llm_observability/instrument/auto_instrumentation.md @@ -1,13 +1,14 @@ --- -title: Automatic Instrumentation for Agent Observability +title: Automatic Instrumentation description: Enable automatic tracing for LLM frameworks and libraries in your Python or Node.js application using Agent Observability integrations. aliases: - /tracing/llm_observability/auto_instrumentation - /llm_observability/auto_instrumentation - /llm_observability/setup/auto_instrumentation - /llm_observability/sdk/auto_instrumentation + - /llm_observability/instrumentation/auto_instrumentation/ further_reading: - - link: '/llm_observability/instrumentation/sdk/' + - link: '/llm_observability/instrument/sdk/' tag: 'Documentation' text: 'Agent Observability SDK Reference' - link: https://www.datadoghq.com/blog/llm-prompt-tracking @@ -22,7 +23,7 @@ further_reading: Agent Observability can automatically trace and annotate calls to supported LLM frameworks and libraries through various [LLM integrations](#llm-integrations). When you [run your LLM application with the Agent Observability SDK][2], these LLM integrations are enabled by default and provide out-of-the-box traces and observability, without you having to change your code. -
Automatic instrumentation works for calls to supported frameworks and libraries. To trace other calls (for example: API calls, database queries, internal functions), see the Agent Observability SDK reference for how to add manual instrumentation.
+
Automatic instrumentation works for calls to supported frameworks and libraries. To trace other calls (for example: API calls, database queries, internal functions), see the Agent Observability SDK reference for how to add manual instrumentation.
### Supported frameworks and libraries @@ -77,7 +78,7 @@ node --import dd-trace/initialize.mjs app.js NODE_OPTIONS="--import dd-trace/initialize.mjs" node app.js ``` -[1]: /llm_observability/instrumentation/sdk/?tab=nodejs#command-line-setup +[1]: /llm_observability/instrument/sdk/?tab=nodejs#command-line-setup {{% /collapse-content %}} {{% collapse-content title="Support for bundled applications (esbuild, Webpack)" level="h4" expanded=false id="bundling-support" %}} @@ -119,7 +120,7 @@ See [Instrument a Next.js Application for Agent Observability][1] for properly s {{% /tab %}} {{< /tabs >}} -
Agent Observability also supports any framework that natively emits OpenTelemetry GenAI semantic convention v1.37+-compliant spans, without requiring the Datadog SDK. See OpenTelemetry Instrumentation for setup details and tested frameworks.
+
Agent Observability also supports any framework that natively emits OpenTelemetry GenAI semantic convention v1.37+-compliant spans, without requiring the Datadog SDK. See OpenTelemetry Instrumentation for setup details and tested frameworks.
## LLM integrations @@ -758,7 +759,7 @@ For setup instructions and a complete example, see [OpenTelemetry Instrumentatio [1]: https://github.com/strands-agents/sdk-python/releases/tag/v1.11.0 [2]: https://strandsagents.com [3]: https://opentelemetry.io/docs/specs/semconv/gen-ai/ -[4]: /llm_observability/instrumentation/otel_instrumentation#using-strands-agents +[4]: /llm_observability/instrument/otel_instrumentation#using-strands-agents {{% /tab %}} {{< /tabs >}} {{% /collapse-content %}} @@ -923,7 +924,7 @@ LLMObs.enable( ) ``` -[1]: /llm_observability/instrumentation/sdk?tab=python#in-code-setup +[1]: /llm_observability/instrument/sdk?tab=python#in-code-setup {{% /tab %}} {{% tab "Node.js" %}} @@ -939,7 +940,7 @@ const tracer = require('dd-trace').init({ const { llmobs } = tracer; ``` -[1]: /llm_observability/instrumentation/sdk?tab=nodejs#in-code-setup +[1]: /llm_observability/instrument/sdk?tab=nodejs#in-code-setup {{% /tab %}} {{< /tabs >}} @@ -965,7 +966,7 @@ LLMObs.enable( patch(langchain=True) ``` -[1]: /llm_observability/instrumentation/sdk?tab=python#in-code-setup +[1]: /llm_observability/instrument/sdk?tab=python#in-code-setup {{% /tab %}} {{% tab "Node.js" %}} @@ -983,7 +984,7 @@ const { llmobs } = tracer; tracer.use('langchain', true); ``` -[1]: /llm_observability/instrumentation/sdk?tab=nodejs#in-code-setup +[1]: /llm_observability/instrument/sdk?tab=nodejs#in-code-setup {{% /tab %}} {{< /tabs >}} @@ -1011,5 +1012,5 @@ For more specific control over library patching and the integration that starts {{< partial name="whats-next/whats-next.html" >}} -[1]: /llm_observability/instrumentation/sdk +[1]: /llm_observability/instrument/sdk [2]: /llm_observability/quickstart/ diff --git a/hugo/content/en/llm_observability/instrumentation/otel_instrumentation.md b/hugo/content/en/llm_observability/instrument/otel_instrumentation.md similarity index 98% rename from hugo/content/en/llm_observability/instrumentation/otel_instrumentation.md rename to hugo/content/en/llm_observability/instrument/otel_instrumentation.md index c109b21b12f..fedfbdc4976 100644 --- a/hugo/content/en/llm_observability/instrumentation/otel_instrumentation.md +++ b/hugo/content/en/llm_observability/instrument/otel_instrumentation.md @@ -1,5 +1,7 @@ --- title: OpenTelemetry Instrumentation +aliases: +- /llm_observability/instrumentation/otel_instrumentation/ description: Instrument LLM applications with OpenTelemetry using GenAI or OpenInference semantic conventions and send traces to Agent Observability without the Datadog SDK. --- @@ -17,15 +19,15 @@ Agent Observability supports ingesting OpenTelemetry traces that follow either t ### Evaluations -To send [external evaluations directly to the API](/llm_observability/evaluations/external_evaluations#submitting-external-evaluations-with-the-api) for OpenTelemetry spans, include the `source:otel` tag in the evaluation. When referencing spans, provide `span_id` and `trace_id` as decimal strings. OpenTelemetry uses hexadecimal IDs natively, so convert them to decimal before submitting evaluations. For example, use Python's `int(hex_span_id, 16)` to convert a hex span ID to its decimal equivalent. +To send [external evaluations directly to the API](/llm_observability/configure/evaluations/external_evaluations#submitting-external-evaluations-with-the-api) for OpenTelemetry spans, include the `source:otel` tag in the evaluation. When referencing spans, provide `span_id` and `trace_id` as decimal strings. OpenTelemetry uses hexadecimal IDs natively, so convert them to decimal before submitting evaluations. For example, use Python's `int(hex_span_id, 16)` to convert a hex span ID to its decimal equivalent. ### Prompt Tracking -For information on using Prompt Tracking with OpenTelemetry spans, see [Prompt Tracking - OpenTelemetry Instrumentation](/llm_observability/monitoring/prompt_tracking#opentelemetry-instrumentation). +For information on using Prompt Tracking with OpenTelemetry spans, see [Prompt Tracking - OpenTelemetry Instrumentation](/llm_observability/instrument/prompt_tracking#opentelemetry-instrumentation). ### Experiments -You can use OpenTelemetry spans inside [Agent Observability Experiments](/llm_observability/experiments/setup#using-opentelemetry-spans-inside-experiments). By setting `DD_TRACE_OTEL_ENABLED=1`, OTel spans created inside an experiment task automatically appear as children of the experiment span. +You can use OpenTelemetry spans inside [Agent Observability Experiments](/llm_observability/improve/experiments/setup#using-opentelemetry-spans-inside-experiments). By setting `DD_TRACE_OTEL_ENABLED=1`, OTel spans created inside an experiment task automatically appear as children of the experiment span. ### Span links @@ -939,7 +941,7 @@ with tracer.start_as_current_span("my-span") as span: [3]: https://app.datadoghq.com/llm/traces [4]: /help/ [5]: https://pypi.org/project/strands-agents/ -[6]: /llm_observability/evaluations/external_evaluations +[6]: /llm_observability/configure/evaluations/external_evaluations [7]: https://strandsagents.com/latest/ [8]: /account_management/rbac/data_access/ [9]: https://opentelemetry.io/docs/concepts/signals/traces/#span-links @@ -947,4 +949,4 @@ with tracer.start_as_current_span("my-span") as span: [11]: https://arize-ai.github.io/openinference/python/instrumentation/openinference-instrumentation-openai/ [12]: https://arize-ai.github.io/openinference/spec/semantic_conventions.html [13]: https://langfuse.com/integrations/native/opentelemetry -[14]: /llm_observability/monitoring/llm_observability_and_apm/ +[14]: /llm_observability/instrument/agent_observability_and_apm/ diff --git a/hugo/content/en/llm_observability/monitoring/prompt_tracking.md b/hugo/content/en/llm_observability/instrument/prompt_tracking.md similarity index 94% rename from hugo/content/en/llm_observability/monitoring/prompt_tracking.md rename to hugo/content/en/llm_observability/instrument/prompt_tracking.md index 7d2f7acef85..64e20b382b3 100644 --- a/hugo/content/en/llm_observability/monitoring/prompt_tracking.md +++ b/hugo/content/en/llm_observability/instrument/prompt_tracking.md @@ -1,5 +1,7 @@ --- title: Prompt Tracking +aliases: +- /llm_observability/monitoring/prompt_tracking/ description: Use Prompt Tracking to track your prompt templates and versions. further_reading: @@ -119,9 +121,9 @@ You can use the Agent Observability Trace Explorer to locate requests by prompt [1]: https://app.datadoghq.com/llm/traces [2]: https://app.datadoghq.com/llm/playground -[3]: /llm_observability/instrumentation/sdk/?tab=python#prompt-tracking -[4]: /llm_observability/instrumentation/api/?tab=model#prompt -[5]: /llm_observability/instrumentation/auto_instrumentation?tab=python#langchain -[6]: /llm_observability/instrumentation/sdk/?tab=nodejs#prompt-tracking -[7]: /llm_observability/instrumentation/otel_instrumentation -[8]: /llm_observability/monitoring/prompt_management +[3]: /llm_observability/instrument/sdk/?tab=python#prompt-tracking +[4]: /llm_observability/instrument/api/?tab=model#prompt +[5]: /llm_observability/instrument/auto_instrumentation?tab=python#langchain +[6]: /llm_observability/instrument/sdk/?tab=nodejs#prompt-tracking +[7]: /llm_observability/instrument/otel_instrumentation +[8]: /llm_observability/configure/prompt_management diff --git a/hugo/content/en/llm_observability/instrumentation/sdk.md b/hugo/content/en/llm_observability/instrument/sdk.md similarity index 96% rename from hugo/content/en/llm_observability/instrumentation/sdk.md rename to hugo/content/en/llm_observability/instrument/sdk.md index 410db6be757..b110dfc5b15 100644 --- a/hugo/content/en/llm_observability/instrumentation/sdk.md +++ b/hugo/content/en/llm_observability/instrument/sdk.md @@ -1,5 +1,5 @@ --- -title: Agent Observability SDK Reference +title: SDK Reference description: Reference documentation for the Agent Observability SDKs for Python, Node.js, and Java, covering automatic and manual instrumentation. aliases: - /tracing/llm_observability/sdk/python @@ -11,8 +11,10 @@ aliases: - /llm_observability/sdk/java - /llm_observability/sdk/ - /llm_observability/instrumentation/custom_instrumentation + - /llm_observability/instrument/custom_instrumentation - /tracing/llm_observability/trace_an_llm_application - /llm_observability/setup + - /llm_observability/instrumentation/sdk/ further_reading: - link: https://www.datadoghq.com/blog/llm-prompt-tracking @@ -245,7 +247,7 @@ LLMObs.enable( : optional - _boolean_ - **default**: `false`
When set to `True`, adds an argument to every MCP server tool requesting that the calling model describe why it chose to call the tool. The intent is recorded on the tool span. If not provided, this defaults to the value of `DD_MCP_CAPTURE_INTENT`. -[1]: /llm_observability/instrumentation/auto_instrumentation/ +[1]: /llm_observability/instrument/auto_instrumentation/ {{% /tab %}} {{% tab "Node.js" %}} @@ -310,73 +312,7 @@ Set the following values as environment variables. They cannot be configured pro {{% /collapse-content %}} -{{% collapse-content title="AWS Lambda Setup" level="h4" expanded=false id="aws-lambda-setup" %}} - -To instrument an existing AWS Lambda function with Agent Observability, you can use the Datadog Extension and respective language layers. - -1. Open a Cloudshell in the AWS console. -2. Install the Datadog CLI client -```shell -npm install -g @datadog/datadog-ci -``` -3. Set the Datadog API key and site -```shell -export DD_API_KEY= -export DD_SITE= -``` -If you already have or prefer to use a secret in Secrets Manager, you can set the API key by using the secret ARN: -```shell -export DATADOG_API_KEY_SECRET_ARN= -``` -4. Install your Lambda function with Agent Observability (this requires at least version 77 of the Datadog Extension layer) -{{< tabs >}} -{{% tab "Python" %}} -```shell -datadog-ci lambda instrument -f -r -v {{< latest-lambda-layer-version layer="python" >}} -e {{< latest-lambda-layer-version layer="extension" >}} --llmobs -``` -{{% /tab %}} - -{{% tab "Node.js" %}} -```shell -datadog-ci lambda instrument -f -r -v {{< latest-lambda-layer-version layer="node" >}} -e {{< latest-lambda-layer-version layer="extension" >}} --llmobs -``` -{{% /tab %}} - -{{% tab "Java" %}} -```shell -datadog-ci lambda instrument -f -r -v {{< latest-lambda-layer-version layer="dd-trace-java" >}} -e {{< latest-lambda-layer-version layer="extension" >}} --llmobs -``` -{{% /tab %}} -{{< /tabs >}} - -4. Invoke your Lambda function and verify that Agent Observability traces are visible in the Datadog UI. - -Manually flush Agent Observability traces by using the `flush` method before the Lambda function returns. - -{{< tabs >}} -{{% tab "Python" %}} -```python -from ddtrace.llmobs import LLMObs -def handler(): - # function body - LLMObs.flush() -``` -{{% /tab %}} - -{{% tab "Node.js" %}} -```javascript -import tracer from 'dd-trace'; -const llmobs = tracer.llmobs; - -export const handler = async (event) => { - // your function body - llmobs.flush(); -}; -``` -{{% /tab %}} -{{< /tabs >}} - -{{% /collapse-content %}} +For AWS Lambda, see [Trace an LLM Application in AWS Lambda][17]. After installing the SDK and running your application you should expect to see some data in Agent Observability from auto-instrumentation. Manual instrumentation can be used to capture custom built frameworks or operations from libraries that are not yet supported. @@ -387,7 +323,7 @@ After installing the SDK and running your application you should expect to see s Trace sampling sets the fraction of traces that Agent Observability retains. Because Agent Observability billing is based on the volume of spans you send, setting a sample rate is one way to control your Agent Observability cost. The SDK makes the sampling decision on the root span and applies it to all of that root span's child spans, including spans created in downstream services through [distributed tracing](#distributed-tracing). -Sampling does not affect your [Agent Observability metrics](/llm_observability/monitoring/metrics/), including [token and cost metrics](/llm_observability/monitoring/cost/) and other operational metrics. Because unsampled spans are dropped after Datadog ingests your traces, these metrics remain based on 100% of your application's instrumented traffic, regardless of the specified sample rate. Trace sampling is also independent of in-app controls such as [automation rules](/llm_observability/monitoring/automation_rules/) and [APM trace sampling](/tracing/trace_pipeline/ingestion_mechanisms/), which apply after ingestion. +Sampling does not affect your [Agent Observability metrics](/llm_observability/investigate/metrics/), including [token and cost metrics](/llm_observability/investigate/cost/) and other operational metrics. Because unsampled spans are dropped after Datadog ingests your traces, these metrics remain based on 100% of your application's instrumented traffic, regardless of the specified sample rate. Trace sampling is also independent of in-app controls such as [automation rules](/llm_observability/configure/automation_rules/) and [APM trace sampling](/tracing/trace_pipeline/ingestion_mechanisms/), which apply after ingestion. Configure the sample rate through either of two mechanisms: @@ -471,7 +407,7 @@ with LLMObs.llm(model="gpt-4o"): For a list of available span kinds, see the [Span Kinds documentation][1]. For more granular tracing of operations within functions, see [Tracing spans using inline methods](#tracing-spans-using-inline-methods). -[1]: /llm_observability/terms/ +[1]: /llm_observability/quickstart/terms/ {{% /tab %}} {{% tab "Node.js" %}} @@ -540,7 +476,7 @@ app.use(myAgentMiddleware) {{< /code-block >}} -[1]: /llm_observability/terms/ +[1]: /llm_observability/quickstart/terms/ [2]: /tracing/trace_collection/custom_instrumentation/nodejs/dd-api/?tab=wrapper {{% /tab %}} {{% tab "Java" %}} @@ -568,13 +504,13 @@ To finish a span, call `finish()` on a span object instance. If possible, wrap t } ``` -[1]: /llm_observability/terms/#span-kinds +[1]: /llm_observability/quickstart/terms/#span-kinds {{% /tab %}} {{< /tabs >}} ### LLM calls -
If you are using any LLM providers or frameworks that are supported by Datadog's LLM integrations, you do not need to manually start an LLM span to trace these operations.
+
If you are using any LLM providers or frameworks that are supported by Datadog's LLM integrations, you do not need to manually start an LLM span to trace these operations.
If you are manually instrumenting an LLM span, you must record token counts (such as input_tokens, output_tokens, and total_tokens) yourself by annotating the span. See Enriching spans for more information.
@@ -1354,7 +1290,7 @@ public class MyJavaClass { ## Enriching spans
-The metrics parameter here refers to numeric values attached as attributes on individual spans — not Datadog platform metrics. For certain recognized keys such as input_tokens, output_tokens, and total_tokens, Datadog uses these span attributes to generate corresponding platform metrics (such as ml_obs.span.llm.input.tokens) for use in dashboards and monitors. +The metrics parameter here refers to numeric values attached as attributes on individual spans — not Datadog platform metrics. For certain recognized keys such as input_tokens, output_tokens, and total_tokens, Datadog uses these span attributes to generate corresponding platform metrics (such as ml_obs.span.llm.input.tokens) for use in dashboards and monitors.
{{< tabs >}} @@ -1646,7 +1582,7 @@ Messages annotated with `audioParts` or `imageParts` render as inline audio play {{< img src="llm_observability/instrumentation/image_example.png" alt="An LLM span in the Agent Observability trace view. The input USER message shows the prompt 'What is in this image?' with an inline photo of a black puppy, and the output ASSISTANT message describes it as a black Labrador Retriever puppy on a wooden surface." style="width:100%;" >}} -For OpenAI audio chat completions, `audioParts` are also captured automatically by [Datadog's LLM integrations](/llm_observability/instrumentation/auto_instrumentation/)—no manual annotation required. Unlike `audioParts`, `imageParts` are not currently captured automatically and must be annotated manually; automatic capture is planned for a future release. +For OpenAI audio chat completions, `audioParts` are also captured automatically by [Datadog's LLM integrations](/llm_observability/instrument/auto_instrumentation/)—no manual annotation required. Unlike `audioParts`, `imageParts` are not currently captured automatically and must be annotated manually; automatic capture is planned for a future release. {{% /tab %}} {{% tab "Java" %}} @@ -2023,8 +1959,8 @@ Supported keys: - `template` (str): Template string with placeholders (for example, `"Translate {{text}} to {{lang}}"`). - `chat_template` (List[Message]): Multi-message template form. Provide a list of `{ "role": "", "content": "