diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index df36dfb7..fb71b188 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -310,21 +310,52 @@ export const tabNavigation: NavTab[] = [ title: 'Concepts', items: [ { title: 'Understanding Evaluation', href: '/docs/evaluation/concepts/understanding-evaluation' }, - { title: 'Eval Types', href: '/docs/evaluation/concepts/eval-types' }, - { title: 'Eval Templates', href: '/docs/evaluation/concepts/eval-templates' }, - { title: 'Judge Models', href: '/docs/evaluation/concepts/judge-models' }, - { title: 'Eval Results', href: '/docs/evaluation/concepts/eval-results' }, + { title: 'Eval types', href: '/docs/evaluation/concepts/eval-types' }, + { title: 'Eval templates & versions', href: '/docs/evaluation/concepts/eval-templates' }, + { title: 'Evaluator models', href: '/docs/evaluation/concepts/evaluator-models' }, + { title: 'Output types', href: '/docs/evaluation/concepts/output-types' }, + { title: 'Error localization', href: '/docs/evaluation/concepts/error-localization' }, + { title: 'Guardrails', href: '/docs/evaluation/concepts/guardrails' }, + { title: 'Composite evals', href: '/docs/evaluation/concepts/composite-evals' }, + { title: 'Feedback', href: '/docs/evaluation/concepts/feedback' }, + { title: 'Ground truth', href: '/docs/evaluation/concepts/ground-truth' }, ] }, { - title: 'Features', + title: 'Guides', + items: [ + { title: 'Running Evaluations', href: '/docs/evaluation/guides/running-evaluations' }, + { + title: 'Explore playground', + items: [ + { title: 'Overview', href: '/docs/evaluation/guides/explore-playground' }, + { title: 'Test an eval', href: '/docs/evaluation/guides/explore-playground/test-an-eval' }, + { title: 'Usage & analytics', href: '/docs/evaluation/guides/explore-playground/usage-analytics' }, + { title: 'Eval logs', href: '/docs/evaluation/guides/explore-playground/eval-logs' }, + ] + }, + { title: 'Create a custom eval', href: '/docs/evaluation/guides/custom-evals' }, + { title: 'Build a composite evals', href: '/docs/evaluation/guides/composite-evals' }, + { title: 'Set up guardrails', href: '/docs/evaluation/guides/guardrails' }, + { title: 'Add ground truth', href: '/docs/evaluation/guides/ground-truth' }, + { title: 'Collect feedback', href: '/docs/evaluation/guides/feedback' }, + { title: 'Using Connectors', href: '/docs/evaluation/guides/using-connectors' }, + { title: 'Use custom models', href: '/docs/evaluation/guides/custom-models' }, + { title: 'Evaluate in CI/CD', href: '/docs/evaluation/guides/cicd' }, + ] + }, + { + title: 'References', + items: [ + { title: 'Built-in evals', href: '/docs/evaluation/builtin' }, + { title: 'Output types & scoring', href: '/docs/evaluation/reference/output-types' }, + { title: 'SDK & API', href: '/docs/evaluation/reference/sdk-api' }, + ] + }, + { + title: 'Troubleshooting', items: [ - { title: 'Built-in Evals', href: '/docs/evaluation/builtin' }, - { title: 'Evaluate via Platform & SDK', href: '/docs/evaluation/features/evaluate' }, - { title: 'Create Custom Evals', href: '/docs/evaluation/features/custom' }, - { title: 'Use Custom Models', href: '/docs/evaluation/features/custom-models' }, - { title: 'Future AGI Models', href: '/docs/evaluation/features/futureagi-models' }, - { title: 'Evaluate CI/CD Pipeline', href: '/docs/evaluation/features/cicd' }, + { title: 'Evaluation FAQ & fixes', href: '/docs/evaluation/troubleshooting' }, ] }, ] diff --git a/src/lib/redirects.ts b/src/lib/redirects.ts index 1e9cd2ed..6980ab33 100644 --- a/src/lib/redirects.ts +++ b/src/lib/redirects.ts @@ -173,12 +173,12 @@ export const redirectMap: Record = { '/future-agi/get-started/evaluation/builtin-evals/toxicity': '/docs/evaluation/builtin/toxicity', '/future-agi/get-started/evaluation/builtin-evals/translation-accuracy': '/docs/evaluation/builtin/translation-accuracy', '/future-agi/get-started/evaluation/builtin-evals/valid-links': '/docs/evaluation/builtin', - '/future-agi/get-started/evaluation/create-custom-evals': '/docs/evaluation/features/custom', - '/future-agi/get-started/evaluation/evaluate-ci-cd-pipeline': '/docs/evaluation/features/cicd', - '/future-agi/get-started/evaluation/evaluate-patterns': '/docs/evaluation/features/evaluate', - '/future-agi/get-started/evaluation/future-agi-models': '/docs/evaluation/features/futureagi-models', - '/future-agi/get-started/evaluation/running-your-first-eval': '/docs/evaluation/features/evaluate', - '/future-agi/get-started/evaluation/use-custom-models': '/docs/evaluation/features/custom-models', + '/future-agi/get-started/evaluation/create-custom-evals': '/docs/evaluation/guides/custom-evals', + '/future-agi/get-started/evaluation/evaluate-ci-cd-pipeline': '/docs/evaluation/guides/cicd', + '/future-agi/get-started/evaluation/evaluate-patterns': '/docs/evaluation/guides/running-evaluations', + '/future-agi/get-started/evaluation/future-agi-models': '/docs/evaluation/concepts/evaluator-models', + '/future-agi/get-started/evaluation/running-your-first-eval': '/docs/evaluation/guides/running-evaluations', + '/future-agi/get-started/evaluation/use-custom-models': '/docs/evaluation/guides/custom-models', '/future-agi/get-started/knowledge-base/concept': '/docs/knowledge-base/concepts/concept', '/future-agi/get-started/knowledge-base/how-to/create-kb-using-sdk': '/docs/knowledge-base/features/sdk', '/future-agi/get-started/knowledge-base/how-to/create-kb-using-ui': '/docs/knowledge-base/features/ui', diff --git a/src/pages/docs/cookbook/decrease-hallucination.mdx b/src/pages/docs/cookbook/decrease-hallucination.mdx index d4677500..3727b475 100644 --- a/src/pages/docs/cookbook/decrease-hallucination.mdx +++ b/src/pages/docs/cookbook/decrease-hallucination.mdx @@ -592,7 +592,7 @@ The winner configuration was CharacterTextSplitter_mmr_map_rerank, which combine - **Can I create custom evaluations tailored to my RAG use case in Future AGI?** - Yes. The Deterministic Eval template in Future AGI supports custom evaluations (***Click [here](/docs/evaluation/features/custom) to learn more about deterministic eval***). This lets you apply stringent criteria to your RAG outputs minimising variability. + Yes. The Deterministic Eval template in Future AGI supports custom evaluations (***Click [here](/docs/evaluation/guides/custom-evals) to learn more about deterministic eval***). This lets you apply stringent criteria to your RAG outputs minimising variability. ## Ready to Reduce Hallucinations in Your RAG Applications? diff --git a/src/pages/docs/cookbook/evaluation/eval-correction-loop.mdx b/src/pages/docs/cookbook/evaluation/eval-correction-loop.mdx index 4141f59e..96e679cf 100644 --- a/src/pages/docs/cookbook/evaluation/eval-correction-loop.mdx +++ b/src/pages/docs/cookbook/evaluation/eval-correction-loop.mdx @@ -250,10 +250,10 @@ You ran a built-in eval, found rows where it disagreed with human judgment, enco ## Explore further - + Full reference for the custom eval template API - + Pick the right judge model: turing_small, turing_flash, turing_large diff --git a/src/pages/docs/dataset/features/experiments.mdx b/src/pages/docs/dataset/features/experiments.mdx index 5b026a59..cd91bea7 100644 --- a/src/pages/docs/dataset/features/experiments.mdx +++ b/src/pages/docs/dataset/features/experiments.mdx @@ -77,7 +77,7 @@ Experiment creation is a guided three-step flow: **Basic Info → Configuration Models you've added through Custom Models show up in the model picker for prompt configurations across all experiment types. - See [Custom Models](/docs/evaluation/features/custom-models) for how to register a custom or self-hosted model. + See [Custom Models](/docs/evaluation/guides/custom-models) for how to register a custom or self-hosted model. @@ -90,7 +90,7 @@ Experiment creation is a guided three-step flow: **Basic Info → Configuration **Compare against baseline (optional)**: pick a column from the dataset to compare model outputs against (typically a ground-truth or existing run-prompt column). Skip it if you don't have a reference output yet; you can still run the experiment, attach evals that don't need a baseline, and add a base column later by editing the experiment. - **Add evaluations**: click **Add Evaluation** and pick from the [built-in eval](/docs/evaluation/builtin) catalog or [create a custom eval](/docs/evaluation/features/custom). Add as many as you need. Every eval runs on every configuration so the results are directly comparable. + **Add evaluations**: click **Add Evaluation** and pick from the [built-in eval](/docs/evaluation/builtin) catalog or [create a custom eval](/docs/evaluation/guides/custom-evals). Add as many as you need. Every eval runs on every configuration so the results are directly comparable. ![Choosing Evals](/screenshot/product/dataset/how-to/experiments-in-dataset/7.png) For each eval, map its inputs (e.g. `output`, `input`, `expected`) to the model output or to dataset columns. Mapping is required before the experiment can run. diff --git a/src/pages/docs/dataset/features/run-prompt.mdx b/src/pages/docs/dataset/features/run-prompt.mdx index 7a09a37e..7583abf3 100644 --- a/src/pages/docs/dataset/features/run-prompt.mdx +++ b/src/pages/docs/dataset/features/run-prompt.mdx @@ -36,28 +36,28 @@ Run Prompt lets you add a new column to your dataset that is filled by a model ( Choose **LLM** for text generation (chat). Use for Q&A, summarization, or any text-in, text-out task. Select a chat model from the list; ensure the provider has an API key configured. ![LLM](/screenshot/product/dataset/how-to/run-prompt-in-dataset/3.png) - Click [here](/docs/evaluation/features/custom-models) to learn how to create custom models. + Click [here](/docs/evaluation/guides/custom-models) to learn how to create custom models. Choose **Text-to-Speech** to generate audio from text. The prompt output column will store audio (e.g. URLs). You can configure voice and format for supported TTS models. ![Text-to-Speech](/screenshot/product/dataset/how-to/run-prompt-in-dataset/4.png) - Click [here](/docs/evaluation/features/custom-models) to learn how to create custom models. + Click [here](/docs/evaluation/guides/custom-models) to learn how to create custom models. Choose **Speech-to-Text** to transcribe audio into text. Use when a column contains audio; the model output will be text in the new column. ![Speech-to-Text](/screenshot/product/dataset/how-to/run-prompt-in-dataset/5.png) - Click [here](/docs/evaluation/features/custom-models) to learn how to create custom models. + Click [here](/docs/evaluation/guides/custom-models) to learn how to create custom models. Choose **Image Generation** to create images from text (or image + text) prompts. The prompt output column will store image URLs. Select an image-generation model and ensure the provider has an API key configured. ![Image Generation](/screenshot/product/dataset/how-to/run-prompt-in-dataset/6.png) - Click [here](/docs/evaluation/features/custom-models) to learn how to create custom models. + Click [here](/docs/evaluation/guides/custom-models) to learn how to create custom models. diff --git a/src/pages/docs/evaluation/concepts/composite-evals.mdx b/src/pages/docs/evaluation/concepts/composite-evals.mdx new file mode 100644 index 00000000..0ec52136 --- /dev/null +++ b/src/pages/docs/evaluation/concepts/composite-evals.mdx @@ -0,0 +1,109 @@ +--- +title: "Composite evals" +description: "Roll several eval child templates into one score" +--- + +## What are they + +A **composite eval** rolls several child [templates](/docs/evaluation/concepts/eval-templates) into a single score, so one check can stand for several at once. Every template is either `single` or `composite`. A composite template holds an ordered list of child templates. When it runs, the parent scores each child on the same input, then combines the child scores into one parent score with an [aggregation function](#aggregation-functions). + +Children share one [output type](/docs/evaluation/concepts/output-types), either pass/fail, score, or choices, so their scores are comparable before they are combined. A composite made of graded children produces a graded score; one made of pass/fail children produces a pass/fail-style score. + +Reach for a composite when "good" is several checks at once, for example faithful *and* on-tone *and* complete. Instead of reading three separate results, you get one score that already reflects all three, and one threshold to gate on. + +## Aggregation functions + +The parent combines child scores with one of five functions. Every child score is normalized to a 0 to 1 range first, so they can be compared on the same scale. + +The examples below all score the same response with three graded children: Groundedness 0.9 with weight 2.0, Tone 0.7 and Completeness 0.8 with weight 1.0. + +### Weighted average + +Each child score times its weight, divided by the total weight. This is the default. Use it when some children matter more than others. + +weight 2.0"] --> AGG["Weighted average"] + C2["Tone 0.7
weight 1.0"] --> AGG + C3["Completeness 0.8
weight 1.0"] --> AGG + AGG --> P["Composite score 0.825"] +`} /> + +On the example: (0.9 × 2.0 + 0.7 + 0.8) / 4.0 = 0.825, and one threshold on that single number gates all three checks at once. + +### Average + +The plain mean of the child scores; weights are ignored. On the example: (0.9 + 0.7 + 0.8) / 3 = 0.8. + + AGG["Average"] + C2["Tone 0.7"] --> AGG + C3["Completeness 0.8"] --> AGG + AGG --> P["Composite score 0.8"] +`} /> + +### Minimum + +The lowest child score, a strict gate where every child has to do well. Use it when any one failure should sink the whole score. On the example: 0.7, the Tone score. + + AGG["Minimum"] + C2["Tone 0.7"] --> AGG + C3["Completeness 0.8"] --> AGG + AGG --> P["Composite score 0.7"] +`} /> + +### Maximum + +The highest child score, when clearing the bar on any one check is enough. On the example: 0.9, the Groundedness score. + + AGG["Maximum"] + C2["Tone 0.7"] --> AGG + C3["Completeness 0.8"] --> AGG + AGG --> P["Composite score 0.9"] +`} /> + +### Pass rate + +The fraction of children that individually meet their own pass threshold. Use it when you care how many checks passed rather than by how much. On the example, with each child's threshold at 0.8: two of the three pass, so 0.67. + + AGG["Pass rate"] + C2["Tone 0.7 fails"] --> AGG + C3["Completeness 0.8 passes"] --> AGG + AGG --> P["Composite score 0.67"] +`} /> + + +## Weights and pinned versions + +Two settings on each child keep a composite stable as it grows. + +- **Weight** sets how much a child counts in a weighted average, from 0.0 to 10.0, defaulting to 1.0. A child you care about more gets a higher weight, and a child at 0.0 drops out of the weighted score +- **Pinned version** locks a child to a specific [template version](/docs/evaluation/concepts/eval-templates), so the composite keeps scoring the same way even after that child template is edited later + +Weights only change the weighted average function; the other four ignore them. Pinning matters most for a composite you rely on in CI, where a silent change to a child would quietly move the gate. + +## Keep exploring + + + + Run a composite on any surface + + + Write a child template of your own + + + 156 templates you can drop in as children + + diff --git a/src/pages/docs/evaluation/concepts/error-localization.mdx b/src/pages/docs/evaluation/concepts/error-localization.mdx new file mode 100644 index 00000000..a220ccbb --- /dev/null +++ b/src/pages/docs/evaluation/concepts/error-localization.mdx @@ -0,0 +1,41 @@ +--- +title: "Error localization" +description: "Pinpoint which input field caused an evaluation to fail" +--- + +## Which input caused the failure + +When an evaluation fails, **error localization** narrows down the cause by analyzing which input field led to the failure, using the [evaluator model](/docs/evaluation/concepts/evaluator-models)'s explanation and the [eval template](/docs/evaluation/concepts/eval-templates)'s criteria. Instead of knowing only that an eval failed, you learn which input, whether the prompt, context, or query, was responsible, so you can see whether the problem is in your data, your retrieval, or your instructions. This works the same whether you're testing in the Playground or reviewing a live trace in [Observe](/docs/observe). You turn it on with the Error Localization checkbox when you configure an eval, on the template or a single run; see [Running Evaluations](/docs/evaluation/guides/running-evaluations) for the flow. + +## How it works + +1. **Evaluation fails**: error localization runs when a result comes back failed, pass/fail evals by their own verdict, scored evals on a low score. +2. **Analysis**: an error-localizing agent receives the input data, the eval criteria, the eval result, and the evaluator model's explanation, then works out which input drove the failure. +3. **Result**: you get back an `error_analysis` block describing the problem and a `selected_input_key` naming the input field identified as the issue. +4. **Review**: the flagged field shows up in the trace detail eval view in Observe, and in the Playground's run tests. + +## When it runs + +Error localization runs on a failed result, as long as the eval isn't a code-type or composite eval. It's skipped when: + +- Evaluation passed, nothing to localize +- Code-type eval, not supported +- Composite eval, not supported yet + +## Where results appear + +- **Trace detail eval view**: looking at a span's eval result in Observe shows the flagged input and analysis inline +- **Develop's dataset row drawer**: open a row on a dataset run to see the same breakdown +- **Playground**: the Dataset, Trace, and Simulation test modes render it while you're testing a template +- **SDK**: the result object returns `error_analysis` and `selected_input_key` directly + +## Keep exploring + + + + Run an eval and see error localization results + + + Where the Error Localization checkbox lives on a template + + diff --git a/src/pages/docs/evaluation/concepts/eval-templates.mdx b/src/pages/docs/evaluation/concepts/eval-templates.mdx index 4201fec8..5cbb9a09 100644 --- a/src/pages/docs/evaluation/concepts/eval-templates.mdx +++ b/src/pages/docs/evaluation/concepts/eval-templates.mdx @@ -1,15 +1,32 @@ --- -title: "Eval Templates: Built-in and Custom Evaluation Templates" -description: "Explains what evaluation templates are, the difference between built-in and custom templates, and how output types determine what an eval returns." +title: "Eval templates & versions" +description: "The reusable definition of what an eval measures, and the versions that keep a run stable" --- -## About +## What a template holds -An eval template is the definition of what to measure. It contains the criteria the judge model will apply to each response and specifies what kind of result to return. You create a template once and reuse it across any dataset, simulation, experiment, or SDK call. +A **template** is where you define what an eval checks. You write it once and reuse it by name across a [dataset](/docs/dataset), a [simulation](/docs/simulation), a live trace, or a check that runs on every pull request, and it is either built by Future AGI (built-in) or written by you (custom). A template holds: -Templates are the reusable unit of evaluation logic. Whether you're checking for toxicity, verifying that a response stays grounded in a source document, or enforcing a company-specific rule, the logic lives in the template. +- **Criteria** the [evaluator model](/docs/evaluation/concepts/evaluator-models) applies, written as a rule the model can follow +- **Required inputs**: the keys the template needs, like `output` (the response) and `context` (the retrieved source the response should stay grounded in, not your whole knowledge base) +- **Output type**: whether the result is pass or fail, a score, or a category, covered in [Output types](/docs/evaluation/concepts/output-types) +- **Pass threshold**: for a score or a category, the line that turns the raw value into a pass or fail +- **Reason** (optional): a plain-language explanation of the verdict, when the eval produces one ---- +## Templates and configs + +A template is the definition, written once. An **eval config** is that definition pointed at one place your data lives, and you can create as many as you want: one mapped to a dataset's columns, one attached to a project's live traces, one in a CI job. + +the definition: criteria, inputs, output type"] --> C1["Config
mapped to dataset columns"] + T --> C2["Config
mapped to span attributes"] + T --> C3["Config
mapped to a CI job's outputs"] + style T fill:#2f2f2f,stroke:#ffffff,stroke-width:2px +`} /> + +Each config runs at its module's level, and a config is what the platform loosely calls an "Eval". When someone says an eval ran, a config ran. ## Built-in vs custom templates @@ -17,55 +34,41 @@ Templates are the reusable unit of evaluation logic. Whether you're checking for |---|---|---| | **Who writes the criteria** | Future AGI | You | | **How to access** | Select from the template list in the UI or pass the name to the SDK | Create via UI or API, then use by name | -| **Covers** | 70+ categories: quality, safety, factuality, RAG, bias, format, audio, image | Any domain-specific, business, or regulatory rule you define | +| **Covers** | 156 templates across 14 groups: quality, safety, factuality, RAG, bias, format, audio, image | Any domain-specific, business, or regulatory rule you define | | **Required inputs** | Defined per template (e.g. `input`, `output`, `context`) | You define the required keys in the template config | -See [Built-in evals](/docs/evaluation/builtin) for the full list of available templates. +[Built-in evals](/docs/evaluation/builtin) lists every template; [Create a custom eval](/docs/evaluation/guides/custom-evals) shows how to write your own. -See [Create custom evals](/docs/evaluation/features/custom) for how to write your own. +## Required inputs and mapping ---- - -## Output types +A template declares the input keys it needs, and at run time a config maps your real data to those keys. A groundedness template, for example, needs `output` and `context`; you point each one at the right column or field. -Every template returns one of three output types: - -| Type | Description | Example | -|---|---|---| -| **Pass/Fail** | Binary result: 1.0 for pass, 0.0 for fail | Toxicity check: passed or failed | -| **Score (percentage)** | Numeric value between 0 and 100 | Groundedness: 87 out of 100 | -| **Deterministic choices** | Categorical result from a defined set of options | Tone classification: `formal`, `informal`, `neutral` | +Custom templates define their own keys with `{{variable}}` placeholders in the rule prompt. The names you write become the inputs you must supply: -Every result also includes a **reason**: a plain-language explanation of why the judge assigned that result. This makes it possible to understand failures without reviewing each response manually. - ---- - -## Required keys and input mapping - -Templates declare the input keys they expect. For example, a groundedness template might require `output` (the model response) and `context` (the source document). When you run an eval, you map your actual data to these keys. +```text +Rate whether {{output}} is fully supported by {{context}}. +``` -**In the UI**: When you add a template to a dataset or simulation, the platform shows a mapping form. You select which column corresponds to each required key. +Here `output` and `context` become the required inputs for that template. -**In the SDK**: Pass a dict where the keys match what the template expects: +## Versions -```python -result = evaluator.evaluate( - template=Groundedness(), - input={ - "output": "The Eiffel Tower is in Paris.", - "context": "The Eiffel Tower is a wrought-iron lattice tower in Paris, France.", - }, -) -``` +Every time you change a template, Future AGI saves the previous one as a numbered version, so nothing you already ran gets overwritten. One version is the **default**: the one a new run uses. Each version is a frozen copy of the exact criteria and threshold, so you can **pin** a run to a specific version: re-run the eval a month later and it scores the same way, and an eval that gates your pull requests keeps behaving the same even after you edit the template. -Built-in templates have fixed required keys documented in the template reference. Custom templates let you define any keys using `{{variable_name}}` placeholders in the rule prompt: the key names you use in the prompt become the required keys you must supply at run time. +## Single or composite ---- +A template is **single** (it runs on its own) or **[composite](/docs/evaluation/concepts/composite-evals)** (it aggregates several child templates into one score). Reach for a composite when "good" means several checks at once, like faithful and on-tone and complete. -## Next steps +## Keep exploring -- [Built-in evals](/docs/evaluation/builtin): Full list of available templates with required keys and output types. -- [Create custom evals](/docs/evaluation/features/custom): Write your own criteria and rule prompts. -- [Eval types](/docs/evaluation/concepts/eval-types): LLM as Judge, Deterministic, Statistical Metric, and LLM as Ranker. -- [Judge models](/docs/evaluation/concepts/judge-models): How the model applies a template to produce a result. -- [Eval results](/docs/evaluation/concepts/eval-results): What the output of an eval run looks like. + + + Pass or fail, score, and category, and how a threshold decides + + + The models that apply a template's criteria + + + 156 ready templates, each with its inputs and output type + + diff --git a/src/pages/docs/evaluation/concepts/eval-types.mdx b/src/pages/docs/evaluation/concepts/eval-types.mdx index c00c3a98..8e414882 100644 --- a/src/pages/docs/evaluation/concepts/eval-types.mdx +++ b/src/pages/docs/evaluation/concepts/eval-types.mdx @@ -1,122 +1,66 @@ --- -title: "Eval Types: Four Evaluation Methods in Future AGI" -description: "The four evaluation methods in Future AGI: LLM as Judge, Deterministic, Statistical Metric, and LLM as Ranker, and how modality affects which ones apply." +title: "Eval types" +description: "The three eval types: Agent Evaluator, LLM-as-Judge, and Code Eval, and when to reach for each" --- -## About +## Three ways to reach a verdict -Every eval template in Future AGI uses one of four evaluation methods to produce a result. The method determines how the eval computes its output, whether a judge model is required, and what kind of result to expect. Choosing the right type for your use case gives you the right balance of accuracy, speed, and cost. +Every eval is authored as one of three types. The type decides how the eval reaches its verdict, what it can reason over, and whether an [evaluator model](/docs/evaluation/concepts/evaluator-models) is involved. It's the first choice you make when you create an eval. ---- - -## LLM as Judge - -The judge model reads the response, applies the template's criteria, and reasons about whether it passes. This is the most flexible type: it handles subjective, context-dependent, and nuanced quality checks that cannot be expressed as a fixed rule. - -**Requires a judge model.** Configure one in [Future AGI models](/docs/evaluation/features/futureagi-models) or [custom models](/docs/evaluation/features/custom-models). - -**Returns**: a result (pass/fail, score, or category) and a plain-language **reason** explaining the judgment. - -**Examples**: Groundedness, Toxicity, Task Completion, Tone, Detect Hallucination, Instruction Adherence, PII Detection, Context Adherence, and all custom evals. - -**Best for:** -- Quality checks that require understanding meaning or intent -- Safety and policy enforcement -- RAG pipeline evaluation (context adherence, relevance, chunk attribution) -- Custom business or regulatory rules written in plain language - ---- - -## Deterministic / Rule-based - -Computed directly from the text using code or string logic. No model is called and no API key is required. Given the same input, always returns the same output. - -**Does not require a judge model.** Runs locally; works without an API key via the standalone `evaluate()` function. - -**Returns**: pass/fail only. No reason field. - -**Examples**: Is JSON, Is Email, Contains Valid Link, No Invalid Links, One Line. - -**Best for:** -- Format validation (valid JSON, email address, URL presence) -- High-volume pipelines where speed and zero API cost matter -- Offline or air-gapped environments -- First-pass filtering before running LLM-based evals - ---- - -## Statistical Metric - -Computes a numeric score using an algorithm applied to the output and a reference value. Covers overlap metrics, edit distance, semantic similarity, and information retrieval metrics. No judge model is needed for most: embedding-based metrics call an embedding model, not a generative one. - -**Returns**: a numeric score (e.g. 0–1 or 0–100). No reason field. +### Agent Evaluator -**Examples:** +A reasoning evaluator that runs multiple turns and can use tools and knowledge bases to reach a judgment. Reach for it when a single prompt cannot capture the check, like scoring a multi-step agent transcript or a judgment that needs to look something up. It is the most capable type, and the newest. -| Metric | What it measures | -|---|---| -| BLEU, ROUGE | N-gram overlap between output and reference | -| Levenshtein Similarity | Character edit distance between output and reference | -| Numeric Similarity | Numerical difference between output and reference | -| Embedding Similarity | Semantic vector similarity between output and reference | -| Semantic List Contains | Whether output contains phrases semantically similar to a reference list | -| Recall@K, Precision@K, NDCG@K, MRR, Hit Rate | Retrieval quality for RAG pipelines | -| FID Score | Distribution similarity between sets of real and generated images | -| CLIP Score | Alignment between an image and its text description | +### LLM-as-Judge -**Best for:** -- Benchmarking against a ground-truth reference answer -- RAG retrieval quality (recall, precision, ranking) -- Image generation quality -- Reproducible, model-free scoring +An evaluator model reads the response, applies the [template](/docs/evaluation/concepts/eval-templates)'s criteria in a single pass, and returns a result plus a reason. It is the workhorse for subjective, context-dependent quality: safety, tone, faithfulness, instruction adherence, and any custom rule you can write in plain language. ---- - -## LLM as Ranker +### Code Eval -A variant of LLM as Judge where instead of scoring a single response, the model ranks a set of retrieved context chunks based on relevance to a query. Used specifically for evaluating retrieval ordering in RAG pipelines. +Deterministic logic runs in a sandbox (Python or JavaScript) and computes the result directly from the text. Given the same input it always returns the same output and calls no model. This is the type behind format validation (valid JSON, email, URL), overlap and similarity metrics (BLEU, ROUGE, edit distance, embedding similarity), and retrieval metrics (recall@k, precision@k, NDCG, MRR). -**Requires a judge model.** +## One check, three ways -**Returns**: a ranked score per context item. +Take one check: is this answer grounded in the source document? -**Examples**: Eval Ranking. +- A **Code Eval** can only measure overlap: an embedding-similarity score between the answer and the source. Fast and repeatable, but a paraphrased hallucination can slip through +- An **LLM-as-Judge** reads both and judges faithfulness in one pass, returning a verdict and the reason a claim isn't supported +- An **Agent Evaluator** goes further: it can search the knowledge base behind the answer and verify it claim by claim -**Best for:** -- Evaluating whether a retrieval system surfaces the most relevant chunks at the top -- Diagnosing retrieval ordering issues in RAG pipelines +Same check, three depths. The deeper you go, the more the verdict costs in time and determinism. ---- +## Which type to reach for -## Modality +The three types are a ladder: a Code Eval computes the verdict, an LLM-as-Judge reads and judges it, an Agent Evaluator investigates before judging. Each step up buys reasoning power and costs speed and determinism, so climb only as high as the check requires. -In addition to the four types above, evals also vary by the kind of input they accept: +|"Yes"| CE["Code Eval"] + Q1 -->|"No"| Q2["One read of the response enough?"] + Q2 -->|"Yes"| LJ["LLM-as-Judge"] + Q2 -->|"No"| AE["Agent Evaluator"] +`} /> -| Modality | What it evaluates | Example evals | +| Type | Evaluator model | Returns a reason | |---|---|---| -| **Text** | Any text input or output | Most built-in evals | -| **Image** | Images passed as inputs | CLIP Score, FID Score, Caption Hallucination, Image Instruction Adherence, Synthetic Image Evaluator, OCR Evaluation | -| **Audio** | Audio files or speech | Audio Quality, Audio Transcription, TTS Accuracy | -| **Conversation** | Multi-turn conversation histories | Customer Agent evals (Loop Detection, Context Retention, Query Handling, etc.) | - -Multimodal evals (image, audio, conversation) require a judge model that supports the relevant modality. Use `turing_large` or `turing_small` for image and audio inputs. - ---- - -## Quick reference - -| Type | Judge model required | Returns reason | No API key possible | -|---|---|---|---| -| LLM as Judge | Yes | Yes | No | -| Deterministic | No | No | Yes | -| Statistical Metric | No (most) | No | Yes (most) | -| LLM as Ranker | Yes | No | No | - ---- - -## Next steps - -- [Built-in evals](/docs/evaluation/builtin): Full list with evaluation method and required inputs for each template. -- [Create custom evals](/docs/evaluation/features/custom): Custom evals always use LLM as Judge. -- [Judge models](/docs/evaluation/concepts/judge-models): Choose the right model for LLM as Judge and LLM as Ranker evals. -- [Eval groups](/docs/evaluation): Combine different eval types and run them together in one pass. +| Agent Evaluator | Yes | Yes | +| LLM-as-Judge | Yes | Yes | +| Code Eval | No | No | + +## Keep exploring + + + + The models Agent Evaluator and LLM-as-Judge use + + + Where the criteria and output type live + + + 156 evaluators, each tagged with its type + + + Write your own, usually as LLM-as-Judge + + diff --git a/src/pages/docs/evaluation/concepts/evaluator-models.mdx b/src/pages/docs/evaluation/concepts/evaluator-models.mdx new file mode 100644 index 00000000..35e69186 --- /dev/null +++ b/src/pages/docs/evaluation/concepts/evaluator-models.mdx @@ -0,0 +1,49 @@ +--- +title: "Evaluator models" +description: "The model that reads a response and applies a template's criteria to score it" +--- + +## What an evaluator model does + +An **evaluator model** reads a response and applies an eval [template's](/docs/evaluation/concepts/eval-templates) criteria to produce a result, and an optional reason. It receives the text to evaluate, the template's rule, and the required inputs, then returns a verdict. An evaluator model only reads and scores; it never generates or edits your AI's output. [Agent Evaluator and LLM-as-Judge](/docs/evaluation/concepts/eval-types) evals use an evaluator model; Code Evals do not. + + J["Evaluator model
reads and scores"] + J --> OUT["Result + optional reason"] +`} /> + +## Future AGI evaluator models + +Future AGI ships proprietary models built for evaluation, not for general-purpose chat: + +| Model | Code | Inputs | Best for | Latency | +|---|---|---|---|---| +| TURING_LARGE | `turing_large` | Text, image, audio | Max accuracy, multimodal evals | Higher | +| TURING_SMALL | `turing_small` | Text, image | High fidelity at lower cost | Medium | +| TURING_FLASH | `turing_flash` | Text, image | Fast, high-accuracy evals | Low | +| PROTECT | `protect` | Text, audio | Safety, guardrails, custom rules | Low | +| PROTECT_FLASH | `protect_flash` | Text | First-pass binary filtering | Ultra-low | + +## Bring your own LLM + +You can also use your own model as the evaluator, from OpenAI, Bedrock, SageMaker, Vertex AI, Azure, or a custom endpoint. Reach for a custom model when you need a domain-tuned model, must keep inference in a region, or already pay for a model you want to reuse. See [Use custom models](/docs/evaluation/guides/custom-models). + +## Modality + +An evaluator model can only score inputs it can read. Image evals run on any of the Turing models; audio evals need `turing_large` or `protect`. A text-only model cannot score an image or an audio input, so pick a multimodal one when the eval carries one. + +## Keep exploring + + + + Where the Protect models do their real-time work + + + Bring your own LLM as the evaluator + + + What the evaluator model produces after scoring + + diff --git a/src/pages/docs/evaluation/concepts/feedback.mdx b/src/pages/docs/evaluation/concepts/feedback.mdx new file mode 100644 index 00000000..fe998b1e --- /dev/null +++ b/src/pages/docs/evaluation/concepts/feedback.mdx @@ -0,0 +1,74 @@ +--- +title: "Feedback" +description: "Record corrections and human signals on eval results to improve your evaluator over time" +--- + +## What feedback is + +**Feedback** is a human or system signal recorded on an [evaluation](/docs/evaluation) result, marking whether you agree with the judgment. When an eval returns a wrong result, you submit feedback with the correct verdict and an optional explanation. That feedback is stored and used to improve future runs: it steers the evaluator model with examples of what you called right and wrong, and it powers the correction loop that teaches custom evals your domain's definition of quality. + +Feedback turns scattered disagreements into systematic improvement, so evaluators converge on what your team means by correct. + +## How feedback is captured + +Each feedback entry records: + +| Field | What it holds | +|---|---| +| **Value** | Your verdict: `passed` or `failed`, what the eval should have returned | +| **Explanation** | Why you are correcting the eval, optional, up to 5000 characters | +| **Improvement suggestion** | How the eval could be better, optional, up to 5000 characters | +| **Source** | Where the eval ran: `dataset`, `trace`, `experiment`, `eval_playground`, `prompt`, `observe`, or `sdk` | +| **Source ID** | The row, trace, or item the feedback refers to | + +Every entry is tied to the [eval template](/docs/evaluation/concepts/eval-templates), the person who submitted it, and the evaluated row, so you can pull up every correction for a given eval. On a dataset eval you also pick what happens next when you submit: retune the eval with your correction, re-run just that row, or re-run the whole dataset. + +## How feedback improves the next run + +Corrections don't sit in a log. When you submit feedback on a dataset eval, the platform stores the corrected row against the eval template. Every later run of that eval retrieves the most similar past corrections and hands them to the evaluator model as few-shot examples, so it sees what you called right and wrong before it scores. + +a result"] --> B["Correction stored
against the template"] + B --> C["Next run retrieves
similar corrections"] + C --> D["Evaluator model scores
with your examples"] +`} /> + +The template itself never changes: the criteria and version stay put. The steering happens at run time, each time the eval runs. + +## The correction loop + +The correction loop is the pattern that turns feedback into a better eval: run an eval, find rows where it disagrees with your judgment, record the corrections, rewrite the eval rules to include those corrections as examples, then re-run and confirm agreement climbs. + +1. **Establish a baseline**: run a built-in eval on a batch of rows and compare its results to your manual verdicts. +2. **Find disagreements**: a disagreement is any row where the eval and your team reach different verdicts. These rows teach the evaluator something new. +3. **Encode corrections**: write a [custom eval](/docs/evaluation/guides/custom-evals) whose rule prompt states your domain rules and includes a few disagreement rows as few-shot examples for the evaluator model. +4. **Re-score and measure**: run the new eval on the same batch and measure agreement against your verdicts. +5. **Iterate**: pull a fresh batch, add new examples, and increment the version. Most evals converge in a few rounds. + +See the [eval correction loop cookbook](/docs/cookbook/evaluation/eval-correction-loop) for a worked end-to-end example. + +## Sources + +Feedback can come from any surface where you evaluate, so you can close the loop wherever the eval ran: + +| Source | Comes from | Use case | +|---|---|---| +| **Dataset** | Dataset column results | Labeling rows where bulk evals got it wrong | +| **Traces** | Live traces in Observe | Marking production responses that were misscored | +| **Experiment** | Experiment results | Marking experiment outcomes that did not match expectations | +| **Eval playground** | The eval tester | Quick feedback on an eval without running it on data | +| **Prompt workbench** | Prompt run results | Correcting scores while iterating on a prompt | +| **SDK** | Programmatic collection | Collecting corrections at scale in scripts or notebooks | + +## Keep exploring + + + + Encode your corrections as rules the evaluator model follows + + + A worked example of turning corrections into a better eval + + diff --git a/src/pages/docs/evaluation/concepts/ground-truth.mdx b/src/pages/docs/evaluation/concepts/ground-truth.mdx new file mode 100644 index 00000000..6d5fd976 --- /dev/null +++ b/src/pages/docs/evaluation/concepts/ground-truth.mdx @@ -0,0 +1,81 @@ +--- +title: "Ground truth" +description: "The known-correct answers an eval compares your AI's output against" +--- + +## What ground truth is + +**Ground truth** is a collection of expected or reference values that reference-based evals compare model outputs against. Instead of measuring subjective qualities like tone or safety, a reference-based eval checks whether the generated output matches the known-correct answer. You upload ground truth as a table of rows, attach it to specific [eval templates](/docs/evaluation/concepts/eval-templates), and those templates compare each model output to its reference at eval time. + +inputs + correct outputs"] --> M["Mapped to a template
variables + roles"] + M --> C["Compare
output vs the reference"] + M --> F["Few-shot
similar rows shown to the evaluator"] + C --> S["Score"] + F --> S +`} /> + +Read it left to right: one upload, mapped once, working two jobs. The reference gives comparison evals their answer key, and the same rows calibrate the evaluator on judged runs. + +## When to use ground truth + +Ground truth is the right choice when you have a clear, correct answer to compare against. Common cases: + +- A question answering system where the correct answer is known in advance +- A code generation tool where you know what the correct code should produce +- A data extraction pipeline where the expected fields and values are labeled +- A classification task where the correct label is on file +- Any task where correct means matching a specific reference output + +If you do not have known-correct answers yet, or correctness is subjective like tone or helpfulness, let an [evaluator model](/docs/evaluation/concepts/evaluator-models) judge the response directly instead. + +## Evals that use ground truth + +Several built-in templates compare against a reference value: + +- [Ground Truth Match](/docs/evaluation/builtin): pass or fail on whether the output matches the reference value semantically +- [Answer Similarity](/docs/evaluation/builtin): a similarity score between the output and a reference answer +- [Fuzzy Match](/docs/evaluation/builtin): approximate string matching with configurable tolerance +- Retrieval metrics such as [Recall@K](/docs/evaluation/builtin) and [Mean Average Precision](/docs/evaluation/builtin): measure whether a retrieved set contains the expected items + +Each template documents the input keys it expects. When you attach ground truth to a template, you map your columns to those keys. + +## How ground truth data is structured + +You upload ground truth as a CSV, XLS, XLSX, or JSON file; the platform stores it as rows with named columns. This is its own upload attached to a template, not a [dataset](/docs/dataset) in the Datasets product. At minimum it needs: + +- **Input columns**: the data the model receives, such as the question column for a question-answering eval +- **Output column**: the correct answer the eval compares against. Every row must have one +- **Optional explanation column**: the reasoning behind the answer, shown to the evaluator alongside it + +## Mapping ground truth to eval inputs + +When you attach ground truth to an eval template, you configure two mappings: + +- **Variable mapping** connects each eval template input to a ground truth column. If the template needs a `question` input, you map it to your `prompt` column. At eval time the platform uses this mapping to pull the right column value for each row, and it is also what similarity retrieval matches on +- **Role mapping** labels the semantic parts: which column is the correct **output** (required), and which column carries the **explanation** behind it (optional) + +## Retrieval and few-shot injection + +Ground truth can also calibrate the evaluator by injecting similar examples into its prompt, called **few-shot grounding**. When enabled, the platform: + +1. Embeds your ground truth rows so similar examples can be found +2. At eval runtime, finds the rows most similar to the current input (three by default) +3. Includes those rows in the evaluator prompt as reference calibration + +This helps the evaluator apply your correctness criteria without you rewriting the prompt. + +The platform embeds ground truth asynchronously after you save a mapping, and retrieval starts on the next eval run once embedding completes. If you change the variable mapping, the embeddings are marked stale and re-embedded automatically. + +## Keep exploring + + + + The reference-based templates, with the inputs each one expects + + + Attach ground truth and run evals against it on any surface + + diff --git a/src/pages/docs/evaluation/concepts/guardrails.mdx b/src/pages/docs/evaluation/concepts/guardrails.mdx new file mode 100644 index 00000000..18b07a4c --- /dev/null +++ b/src/pages/docs/evaluation/concepts/guardrails.mdx @@ -0,0 +1,44 @@ +--- +title: "Guardrails" +description: "Safety evals that block bad content before it reaches your user" +--- + +## What a guardrail is + +A **guardrail** is a safety valve: anything that stands between your AI and your user and can stop bad content from being delivered. On Future AGI, guardrails are a group of evals put to that job, user-facing safety checks like toxicity, sexism, and personal-information detection, run inline so the verdict becomes an action (deliver or block) instead of a stored score. + +response"] --> G["Guardrail
a safety eval, run inline"] + G --> P["Passes
delivered to your user"] + G --> B["Fails
blocked before delivery"] + style G fill:#2f2f2f,stroke:#ffffff,stroke-width:2px +`} /> + +What makes an eval a guardrail isn't the template, it's the job: deciding in real time whether content ships. + +## Two ways to run guardrails + +### Through the gateway + +[Agent Command Center's gateway](/docs/command-center/features/guardrails) guards traffic at the platform level. It ships built-in checks (rule-based and AI-powered), integrates third-party guardrailing systems, and offers Future AGI's [Protect](/docs/protect) models natively. You group checks into policies, choose enforce or monitor mode, and scope them globally, per project, or per key. Nothing in your application code changes; the gateway sits on the request path. + +### Through the Protect SDK + +Protect is the in-code path: a small set of guardrailing evals built for millisecond latency, available only through the [SDK's guardrails module](/docs/sdk/evals/guardrails-module). Nothing is on by default; you enable the checks you want at the point where your code produces output, so the screen runs inside your application without adding noticeable turnaround. + +## Guardrails and standard evals + +The same eval can hold both jobs. Toxicity is available as a guardrail (inline, blocking) and as a standard eval on the platform, where [evaluator models](/docs/evaluation/concepts/evaluator-models) like the Turing family give you deeper analysis over a dataset or your traces. Scoring yesterday's responses for toxicity is evaluation; refusing to deliver today's toxic response is a guardrail. If you don't need live blocking, run the same checks as standard evals and skip the gateway entirely. + +## Keep exploring + + + + Policies, built-in checks, and third-party integrations + + + The SDK safety evals built for real-time blocking + + diff --git a/src/pages/docs/evaluation/concepts/judge-models.mdx b/src/pages/docs/evaluation/concepts/judge-models.mdx deleted file mode 100644 index 60eacd86..00000000 --- a/src/pages/docs/evaluation/concepts/judge-models.mdx +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: "Judge Models: Choosing the Right Evaluation Scorer" -description: "Explains what a judge model is, how it scores AI responses, and how to choose the right judge model for your specific evaluation use case." ---- - -## About - -A judge model is the model that reads each response and applies the eval template criteria to produce a result. When you run an evaluation, the judge receives the text to evaluate, the template's rule prompt, and the required inputs, then returns a result and a reason. - -The judge model determines how accurately and how quickly each response gets scored. Choosing the right one lets you balance precision and performance for your specific workload. - ---- - -## How a judge scores a response - -1. The platform constructs a prompt from the eval template criteria and the row's input values. -2. The judge model receives this prompt and reads the response being evaluated. -3. The judge returns a result (pass/fail, score, or category) and a reason explaining the judgment. -4. The platform stores the result and reason for that row. - -The judge model does not generate or modify your AI's responses. It only reads and scores them. - ---- - -## Available judge models - -Future AGI provides a set of proprietary models built specifically for evaluation: - -| Model | Code | Best for | Latency | -|---|---|---|---| -| TURING_LARGE | `turing_large` | Max accuracy, multimodal evals (text, image, audio) | Higher | -| TURING_SMALL | `turing_small` | High fidelity at lower cost (text, image) | Medium | -| TURING_FLASH | `turing_flash` | Fast, high-accuracy evals (text, image) | Low | -| PROTECT | `protect` | Safety, guardrails, user-defined rules (text, audio) | Low | -| PROTECT_FLASH | `protect_flash` | First-pass binary filtering (text only) | Ultra-low | - -See [Future AGI models](/docs/evaluation/features/futureagi-models) for full details on each model. - -You can also bring your own model using the [custom models](/docs/evaluation/features/custom-models) integration. This is useful when you need a domain-specific fine-tuned model, want to keep inference in a specific cloud region, or already pay for a model you want to use as the judge. - ---- - -## How to choose a judge - -| Situation | Recommended model | -|---|---| -| Maximum accuracy matters more than speed | `turing_large` | -| High quality at reasonable cost | `turing_small` | -| Large-scale runs where speed is important | `turing_flash` | -| Safety and guardrail checks | `protect` or `protect_flash` | -| Evaluating images or audio | `turing_large` or `turing_small` | -| Domain-specific or compliance requirements | [Custom model](/docs/evaluation/features/custom-models) | - ---- - -## Next steps - -- [Future AGI models](/docs/evaluation/features/futureagi-models): Full reference for built-in judge models. -- [Use custom models](/docs/evaluation/features/custom-models): Bring your own model as the judge. -- [Eval templates](/docs/evaluation/concepts/eval-templates): The criteria the judge applies. -- [Eval results](/docs/evaluation/concepts/eval-results): What the judge produces after scoring. diff --git a/src/pages/docs/evaluation/concepts/eval-results.mdx b/src/pages/docs/evaluation/concepts/output-types.mdx similarity index 76% rename from src/pages/docs/evaluation/concepts/eval-results.mdx rename to src/pages/docs/evaluation/concepts/output-types.mdx index d11d1840..b8543af6 100644 --- a/src/pages/docs/evaluation/concepts/eval-results.mdx +++ b/src/pages/docs/evaluation/concepts/output-types.mdx @@ -1,11 +1,11 @@ --- -title: "Eval Results: Reading and Storing Evaluation Outputs" -description: "Understand what evaluation results contain, how to read them, and how results are stored and aggregated across runs in Future AGI." +title: "Output types" +description: "What an eval returns: pass/fail, scores, or choices, and where results land" --- ## About -Every evaluation run produces a result for each row or call that was scored. A result tells you whether the response passed the criteria, how it scored, and why the judge made that decision. Results are stored alongside your data so you can review them, compare across runs, and track quality over time. +Every evaluation run produces a result for each row or call that was scored. A result tells you whether the response passed the criteria, how it scored, and why the evaluator model made that decision. Results are stored alongside your data so you can review them, compare across runs, and track quality over time. --- @@ -16,7 +16,7 @@ Each individual result has three parts: | Field | Description | |---|---| | **Output** | The result value: `1.0` (pass), `0.0` (fail), a score between 0 and 100, or a category label depending on the template's output type | -| **Reason** | A plain-language explanation from the judge describing why it assigned that result | +| **Reason** | A plain-language explanation from the evaluator model describing why it assigned that result | | **Eval ID** | A unique identifier for the eval run, used to retrieve async results | The reason field is especially useful for diagnosing failures. Instead of reviewing each response manually, you can read the reason to understand exactly what caused a pass or fail judgment. @@ -60,7 +60,7 @@ These aggregates appear in the evaluation summary view and are tracked per eval ## Next steps -- [Evaluate via Platform & SDK](/docs/evaluation/features/evaluate): Run an eval and see results. +- [Evaluate via Platform & SDK](/docs/evaluation/guides/running-evaluations): Run an eval and see results. - [Eval templates](/docs/evaluation/concepts/eval-templates): How templates define what output type a result uses. -- [Judge models](/docs/evaluation/concepts/judge-models): How the judge produces the result and reason. -- [CI/CD pipeline](/docs/evaluation/features/cicd): Track results by version across deploys. +- [Evaluator models](/docs/evaluation/concepts/evaluator-models): How the evaluator model produces the result and reason. +- [CI/CD pipeline](/docs/evaluation/guides/cicd): Track results by version across deploys. diff --git a/src/pages/docs/evaluation/concepts/understanding-evaluation.mdx b/src/pages/docs/evaluation/concepts/understanding-evaluation.mdx index c09f74ce..d2c08d3a 100644 --- a/src/pages/docs/evaluation/concepts/understanding-evaluation.mdx +++ b/src/pages/docs/evaluation/concepts/understanding-evaluation.mdx @@ -1,58 +1,54 @@ --- -title: "Understanding AI Evaluation in Future AGI Platform" -description: "Covers how evaluation works in Future AGI: templates, judge models, eval types, results, and where evaluations run in the platform and SDK." +title: "Understanding Evaluation" +description: "The four objects behind every eval, and the loop they sit in" --- -## About +## The four objects -Evaluation in Future AGI is a systematic way to measure whether your AI is producing the right outputs. You define what "right" means once, using an eval template, and the platform scores every response automatically against that definition, returning a result and a reason for each one. +You define what "good" means once, and the platform scores every response against that definition. Four objects carry the whole product. Learn these and the rest is detail: -Every eval run has three components working together: a **template** that defines the criteria, a **judge model** that applies the criteria to each response, and a **result** that records the outcome. You supply the data; the platform handles the scoring. +- A **[template](/docs/evaluation/concepts/eval-templates)** defines *what* to measure: the criteria, the expected output type, and a pass threshold. Templates are reusable and versioned, and are either built by Future AGI or written by you +- A **config** defines *how* to measure for one run: which [evaluator model](/docs/evaluation/concepts/evaluator-models), how your data maps to the template's inputs, and the run settings +- A **run** is one execution: a template plus a config plus one unit of data, from a single row to millions +- A **score** is the outcome: a value (pass or fail, a number, or a category), an optional reason, and the metadata to trace it back ---- - -## How it works - -1. **Choose a template**: Select a built-in template (e.g. Toxicity, Groundedness, Tone) or create a custom one with your own rule prompt. Templates define what to measure and what output type to expect (pass/fail, score, or a category). - -2. **Map your data**: Tell the eval which columns or input keys contain the text to evaluate (e.g. which column is the model response, which is the reference context). - -3. **Pick a judge model**: Choose a Future AGI model (e.g. `turing_flash`) or bring your own via a custom model integration. The judge reads each row and applies the template criteria. - -4. **Run**: The platform processes every row in parallel. Each row gets a result (pass/fail or a score) and a reason explaining the judgment. +what to measure"] --> C["Config
pointed at your data"] + C --> R["Run
one execution"] + R --> S["Score
the outcome"] +`} /> -5. **Review**: Results appear as new columns in your dataset, or as aggregated summaries and KPIs across runs. +Read it left to right: a template is the definition, a config points it at data, a run executes, and a score comes out. Everything else in Evaluations elaborates on one of these four. And because the same definition runs everywhere your work lives, a score means the same thing on a dataset, a simulation, a live trace, or a pull request. ---- - -## Where evals run - -Evals are not limited to datasets. The same templates work across every surface in Future AGI: +## The quality loop -| Surface | What you evaluate | -|---|---| -| **Dataset** | Score every row in a dataset against one or more templates | -| **Simulation** | Evaluate agent responses in a run test against your criteria | -| **Experiments** | Compare prompt or model versions using the same eval criteria | -| **CI/CD pipeline** | Run evals automatically on every code change and track results by version | -| **SDK** | Call `evaluator.evaluate()` from any script or application | +Evaluation is the middle of a loop, not a standalone tool. You [observe](/docs/observe) what your agent did, evaluate its quality, [optimize](/docs/optimization) the prompt or model, and enforce a bar before anything ships. Traces become eval inputs, scores drive optimization, and thresholds become production gates. -Using the same template across surfaces keeps results directly comparable without redefining criteria each time. +## How an eval reaches a verdict ---- - -## Key concepts +Every eval is authored as one of three [types](/docs/evaluation/concepts/eval-types): -- **[Eval templates](/docs/evaluation/concepts/eval-templates)**: The definition of what to measure. Built-in or custom. -- **[Judge models](/docs/evaluation/concepts/judge-models)**: The model that applies the template criteria and produces the result. -- **[Eval results](/docs/evaluation/concepts/eval-results)**: The output of a run: result value, reason, and aggregates. -- **[Eval groups](/docs/evaluation)**: Named collections of templates you run together as a single unit. +- **Agent Evaluator** reasons over multiple turns and can use tools to reach a judgment +- **LLM-as-Judge** has an evaluator model read the response and apply the criteria in one pass +- **Code Eval** computes the result in a sandbox, with no model and no API key ---- +The type decides how the verdict is reached and whether an evaluator model is involved. -## Next steps +## Keep exploring -- [Evaluate via Platform & SDK](/docs/evaluation/features/evaluate): Run your first eval. -- [Built-in evals](/docs/evaluation/builtin): 70+ templates across quality, safety, factuality, RAG, and more. -- [Create custom evals](/docs/evaluation/features/custom): Define your own criteria and rule prompts. -- [Eval groups](/docs/evaluation): Bundle multiple evals and run them in one pass. + + + Agent Evaluator, LLM-as-Judge, and Code Eval in depth + + + Criteria, inputs, output types, and version snapshots + + + The models that read and score a response + + + The value, optional reason, and aggregates + + diff --git a/src/pages/docs/evaluation/features/evaluate.mdx b/src/pages/docs/evaluation/features/evaluate.mdx deleted file mode 100644 index 57a3ffe4..00000000 --- a/src/pages/docs/evaluation/features/evaluate.mdx +++ /dev/null @@ -1,170 +0,0 @@ ---- -title: "Evaluate via Platform and SDK in Future AGI" -description: "Run evaluations using the Future AGI platform UI or the Python SDK. Choose individual templates or batch runs for scalable model assessment." ---- - -## About - -Evaluation is how you measure whether your AI is actually doing what you want it to do. - -You give it an input (a prompt, a response, a conversation) and an eval scores it. The score tells you if the output was accurate, safe, on-topic, well-structured, or whatever quality you care about. Every evaluation returns a **result** (e.g. Passed / Failed, or a numeric score), and a **reason** explaining why. - -You can run evaluations two ways: - -- **Platform UI**: point-and-click on a dataset. No code required. -- **Python SDK**: call `evaluator.evaluate()` from your code, scripts, or CI pipeline. - -Both support the same [built-in eval templates](/docs/evaluation/builtin) (e.g. Toxicity, Groundedness, Tone) and any custom evals you've defined. - ---- - -## When to use - -- **Catch regressions before they ship**: Run evals in CI so a bad prompt change or model update gets flagged before it reaches production. -- **Score outputs at scale**: Attach evals to a dataset and every row gets a score automatically, without reviewing each one manually. -- **Enforce safety and compliance**: Check every response for toxicity, PII, bias, or data privacy issues as part of your standard pipeline. -- **Compare models or prompts**: Evaluate the same inputs across different models or prompt variations to see which performs better on your criteria. -- **Monitor quality over time**: Run the same evals repeatedly to track whether your AI's output quality is improving or degrading. - ---- - -## How to - -Choose **UI** or **SDK** below; each tab has the process in steps. - - - - - - You need a dataset to run evals from the UI. If you don’t have one, add a dataset first. See [Dataset overview](/docs/dataset). - ![Select a dataset](/screenshot/product/evaluation/evaluate/1.png) - - - Open your dataset, then click **Evaluate** in the top-right. The evaluation configuration panel opens. - ![Open the evaluation panel](/screenshot/product/evaluation/evaluate/2.png) - - - Click **Add Evaluation**. Choose a **built-in template** (e.g. Tone) or click **Create your own eval**. For a built-in template: click it, give it a **name**, and under **config** select the **dataset column(s)** to use as input (and output if required). - ![Add and run an eval](/screenshot/product/evaluation/evaluate/3.png) - ![Add and run an eval](/screenshot/product/evaluation/evaluate/4.png) - - - Optionally enable **Error Localization** to pinpoint which part of a row caused a failure. Select a **model** if the template requires one. Click **Add & Run** to score every row in the dataset. - ![Add and run an eval](/screenshot/product/evaluation/evaluate/5.png) - - - From the Add Evaluation flow, click **Create your own eval** to define a custom template (name, model, rule prompt, output type, and optional settings). After you save it, the new eval appears in the evaluation list and you can add it to your dataset as in the step above. For full details on creating and configuring custom evals, see [Create custom evals](/docs/evaluation/features/custom). - - - - - - Some evals can run without an API key using the standalone `evaluate()` function, including local metrics like `contains`, `faithfulness`, and LLM-as-judge. See the SDK reference for details. - - - - Install the package **ai-evaluation** and create an `Evaluator` with your Future AGI API key and secret. Prefer setting **FI_API_KEY** and **FI_SECRET_KEY** in the environment instead of passing them in code. See [Accessing API keys](/docs/admin-settings#accessing-api-keys). - - ```bash - pip install ai-evaluation - ``` - - ```python - from fi.evals import Evaluator - - evaluator = Evaluator( - fi_api_key="your_api_key", - fi_secret_key="your_secret_key", - ) - ``` - - - Call `evaluate` with the eval template **name** (e.g. `tone`), **inputs** (dict with the keys the template expects, e.g. `"input"`), and **model_name**. Many built-in (system) templates require a model. - - ```python - result = evaluator.evaluate( - eval_templates="tone", - inputs={ - "input": "Dear Sir, I hope this email finds you well. I look forward to any insights or advice you might have whenever you have a free moment" - }, - model_name="turing_flash", - ) - print(result.eval_results[0].output) - print(result.eval_results[0].reason) - ``` - - - For long-running or large runs, set `is_async=True`. The call returns immediately with an **eval_id**; the evaluation runs in the background. - - ```python - result = evaluator.evaluate( - eval_templates="tone", - inputs={"input": "Your text here"}, - model_name="turing_flash", - is_async=True, - ) - eval_id = result.eval_results[0].eval_id - ``` - - - Use `get_eval_result(eval_id)` to fetch the result when the evaluation has finished. The same method works for both sync and async runs (e.g. to re-fetch a result). - - ```python - result = evaluator.get_eval_result(eval_id) - print(result.eval_results[0].output) - print(result.eval_results[0].reason) - ``` - - - To use a template you created in the UI, pass its **name** as `eval_templates` and supply the **inputs** dict with the keys your template’s required_keys expect (e.g. `"input"`, `"output"`). Use the same template name you see in the evaluation list. - - ```python - result = evaluator.evaluate( - eval_templates="name-of-your-eval", - inputs={ - "input": "your_input_text", - "output": "your_output_text" - }, - model_name="model_name" - ) - - print(result.eval_results[0].output) - print(result.eval_results[0].reason) - ``` - - - - - For **system (built-in)** eval templates, **model_name** is required and must be one of the models listed for that template. The backend validates required input keys from the template’s config. - - - - - -For more eval templates and Future AGI models, see [Built-in evals](/docs/evaluation/builtin) and [Future AGI models](/docs/evaluation/features/futureagi-models). - - ---- - -## Next Steps - - - - Define your own eval rules and criteria. - - - Run multiple evals together as a group. - - - Bring your own model for evaluations. - - - Built-in models available for evals. - - - Run evals automatically in your pipeline. - - - How evaluation fits into the platform. - - diff --git a/src/pages/docs/evaluation/features/futureagi-models.mdx b/src/pages/docs/evaluation/features/futureagi-models.mdx deleted file mode 100644 index 7dce3c35..00000000 --- a/src/pages/docs/evaluation/features/futureagi-models.mdx +++ /dev/null @@ -1,95 +0,0 @@ ---- -title: "Use Future AGI Models for AI Evaluation and Scoring" -description: "Future AGI's proprietary judge models are trained on diverse datasets to perform accurate evaluations and score AI outputs." ---- - -## About - -When you run an evaluation, the model you choose determines how accurately and how fast each response gets scored. Future AGI provides a set of proprietary models built and optimized specifically for evaluation, not general-purpose chat or generation. - -Each model is designed for a different need. Some prioritize accuracy across complex multimodal inputs. Others are built for speed, making them suitable for real-time guardrailing or high-volume pipelines. Choosing the right model lets you balance quality and performance for your specific workload. - -All models are available in the platform UI and the SDK, and work with both built-in and custom eval templates. - ---- - -## Available models - -- **TURING_LARGE** `turing_large`: Flagship evaluation model that delivers best-in-class accuracy across multimodal inputs (text, images, audio). Recommended when maximal precision outweighs latency constraints. - -- **TURING_SMALL** `turing_small`: Compact variant that preserves high evaluation fidelity while lowering computational cost. Supports text and image evaluations. - -- **TURING_FLASH** `turing_flash`: Latency-optimised version of TURING, providing high-accuracy assessments for text and image inputs with fast response times. - -- **PROTECT** `protect`: Real-time guardrailing model for safety, policy compliance, and content-risk detection. Offers very low latency on text and audio streams and permits user-defined rule sets. - -- **PROTECT_FLASH** `protect_flash`: Ultra-fast binary guardrail for text content. Designed for first-pass filtering where millisecond-level turnaround is critical. - ---- -### Quick comparison - -| Model | Code | Inputs | Best for | Latency | -| --- | --- | --- | --- | --- | -| TURING_LARGE | `turing_large` | Text, image, audio | Max accuracy, multimodal evals | Higher | -| TURING_SMALL | `turing_small` | Text, image | High fidelity, lower cost | Medium | -| TURING_FLASH | `turing_flash` | Text, image | Fast, high-accuracy evals | Low | -| PROTECT | `protect` | Text, audio | Safety, guardrails, user-defined rules | Low | -| PROTECT_FLASH | `protect_flash` | Text | First-pass binary filtering | Ultra-low | - ---- - -## How to - - - - - - When adding or configuring an evaluation on a dataset or run test, choose **Use Future AGI Models** and pick a model from the dropdown. - ![Use Future AGI Models in the UI](/screenshot/product/evaluation/future-agi-models/1.png) - - - - - - - Pass `model_name` in your `evaluator.evaluate()` call. Use the model code from the table above (e.g. `turing_flash`, `turing_large`, `protect`). - - ```python - from fi.evals import Evaluator - - evaluator = Evaluator(fi_api_key="...", fi_secret_key="...") - result = evaluator.evaluate( - eval_templates="tone", - inputs={"input": "Your text to evaluate."}, - model_name="turing_small", # or turing_flash, turing_large, protect, protect_flash - ) - ``` - - - - - ---- - -## Next Steps - - - - Run evals from the UI or SDK. - - - Define your own eval rules and choose a model to run them. - - - Run multiple evals together as a group. - - - Bring your own model for evaluations. - - - Run evals automatically in your pipeline. - - - How evaluation fits into the platform. - - diff --git a/src/pages/docs/evaluation/features/cicd.mdx b/src/pages/docs/evaluation/guides/cicd.mdx similarity index 98% rename from src/pages/docs/evaluation/features/cicd.mdx rename to src/pages/docs/evaluation/guides/cicd.mdx index 456dd99d..3c6f14ce 100644 --- a/src/pages/docs/evaluation/features/cicd.mdx +++ b/src/pages/docs/evaluation/guides/cicd.mdx @@ -1,5 +1,5 @@ --- -title: "Evaluate via CI/CD Pipeline: Automated Quality Checks" +title: "Evaluate in CI/CD" description: "Run Future AGI evaluations in your CI/CD pipeline to assess model performance on every pull request and keep quality checks consistent before deployment." --- @@ -401,16 +401,16 @@ The workflow posts a comment on your PR with the current version identifier and ## Next Steps - + Run a single eval from the UI or SDK. - + Define eval templates to use in your pipeline. Run multiple evals together as a group. - - Bring your own model for evaluations. + + Bring your own LLM for evaluations diff --git a/src/pages/docs/evaluation/features/custom.mdx b/src/pages/docs/evaluation/guides/custom-evals.mdx similarity index 56% rename from src/pages/docs/evaluation/features/custom.mdx rename to src/pages/docs/evaluation/guides/custom-evals.mdx index 66c648b6..7f04a6f5 100644 --- a/src/pages/docs/evaluation/features/custom.mdx +++ b/src/pages/docs/evaluation/guides/custom-evals.mdx @@ -1,40 +1,39 @@ --- -title: "Create Custom Evaluation Templates in Future AGI" -description: "Define custom evaluation criteria and rules tailored to your use case, extending beyond the built-in templates available in Future AGI." +title: "Create a custom eval" +description: "Write your own eval when no built-in template fits" --- -## About +Every AI product has its own definition of a good response. When no [built-in template](/docs/evaluation/builtin) fits, a custom eval lets you encode that definition yourself: write the criteria once, in plain language, and Future AGI scores every response against it. -Every AI product has its own definition of a good response. Custom evals let you encode those rules and run them at scale, you write the criteria once, in plain language, and Future AGI scores every response against it automatically, returning a result and a reason for each one. - -Once created, a custom eval works exactly like any built-in template: use it on a dataset, in a simulation, or call it from the SDK. - ---- +Most custom evals are [LLM-as-Judge](/docs/evaluation/concepts/eval-types), where an [evaluator model](/docs/evaluation/concepts/evaluator-models) applies your rule in one pass and returns a reason with each verdict. Use a Code Eval when the rule is objective and computable, or an Agent Evaluator when one pass won't do. Once created, a custom eval works exactly like a built-in one: use it on a dataset, in a simulation, or from the SDK, by its name. ## When to use -- **Domain-specific validation**: Assess content against industry or regulatory standards that aren’t in the default templates. -- **Business rule compliance**: Enforce your organization’s guidelines (tone, format, disclosures) in a repeatable eval. -- **Complex or weighted scoring**: Implement multi-criteria or custom scoring logic via your rule prompt. -- **Custom output formats**: Validate specific response structures or formats (e.g. JSON shape, required fields) with a tailored eval. - ---- +- Assess content against industry or regulatory standards that the default templates do not cover +- Enforce your organization's own guidelines for tone, format, or required disclosures +- Apply multi-criteria or weighted scoring through the rule prompt +- Validate a specific response structure, such as a JSON shape or a set of required fields ## How to -You can create custom evals from the **UI** or via the **SDK** (by calling the REST API from your code). After the template is saved, run it from the UI or from the evaluation SDK using the template name. +You can create a custom eval from the **UI** or via the **SDK** (by calling the REST API from your code). After the template is saved, run it from the UI or from the evaluation SDK using the template name. - Open your **dataset**, click **Evaluate** in the top-right, then **Add Evaluation**. Select **Create your own eval** to start the custom-eval flow. + Open your **dataset**, click **Evaluate** in the top-right, then **Add Evaluation**. Select **Create your own evals** to start the custom-eval flow. ![Open evaluation creation](/screenshot/product/evaluation/create-custom-evals/1.png) **Name**: unique name for the eval (lowercase letters, numbers, hyphens, and underscores only; cannot start or end with `-` or `_`). Used when you add the eval to a dataset or call it from the SDK. - **Model**: choose **Use Future AGI Models** (e.g. turing_large, turing_flash, turing_small, protect, protect_flash) or **Use other LLMs** (your own or external providers). For model details, see [Future AGI models](/docs/evaluation/features/futureagi-models) and [Use custom models](/docs/evaluation/features/custom-models). + **Model**: pick who applies your rule. This choice is also what decides the eval's type: + + - **Use Future AGI Agents**: a Future AGI model applies the rule (turing_large, turing_flash, turing_small, protect, protect_flash) + - **Use other LLMs**: a model you bring applies the rule; set one up in [Use custom models](/docs/evaluation/guides/custom-models) + - **Function based**: a Code Eval built on an existing function eval. The result is computed by code, so the rule prompt below doesn't apply + ![Configure basic settings](/screenshot/product/evaluation/create-custom-evals/2.png) @@ -42,14 +41,19 @@ You can create custom evals from the **UI** or via the **SDK** (by calling the R ![Define evaluation rules](/screenshot/product/evaluation/create-custom-evals/3.png) - **Pass/Fail**: binary result (e.g. 1.0 pass, 0.0 fail). **Percentage (score)**: numeric score between 0 and 100. **Deterministic choices**: categorical result; provide a dict of allowed choices. + Choose what the eval returns: + + - **Pass/Fail**: binary result, `1.0` pass and `0.0` fail + - **Percentage**: a score between 0 and 100 + - **Deterministic choices**: a category label; you define the set of allowed choices + ![Configure output type](/screenshot/product/evaluation/create-custom-evals/4.png) - - **Tags**: for filtering and organization. - - **Description**: shown in the evaluation list. - - **Check Internet**: allow the eval to fetch up-to-date information when needed. - - **Required keys**: list the input variable names the eval expects (e.g. `input`, `output`, `user_query`, `chatbot_response`). + - **Tags**: for filtering and organization + - **Description**: shown in the evaluation list + - **Check Internet**: allow the eval to fetch up-to-date information when needed + - **Required keys**: list the input variable names the eval expects (e.g. `input`, `output`, `user_query`, `chatbot_response`) ![Add optional settings and save](/screenshot/product/evaluation/create-custom-evals/5.png) @@ -57,7 +61,7 @@ You can create custom evals from the **UI** or via the **SDK** (by calling the R Click **Create Evaluation**. The new template appears in your evaluation list and can be added to any dataset or called via the SDK using the name you gave it. - In your dataset, click **Evaluate** → **Add Evaluation**, select the custom eval you created, map the **columns** to the rule-prompt variables, then click **Add & Run**. See [Running your first eval](/docs/evaluation) for the full UI flow. + In your dataset, click **Evaluate** then **Add Evaluation**, select the custom eval you created, map the **columns** to the rule-prompt variables, then click **Add & Run**. See [Running evaluations on Future AGI](/docs/evaluation/guides/running-evaluations) for the full run flow. @@ -93,7 +97,7 @@ You can create custom evals from the **UI** or via the **SDK** (by calling the R "tags": ["customer-service"], }, ) - print(response.json()) # {"eval_template_id": "..."} + print(response.json()) # {"status": true, "result": {"eval_template_id": "..."}} ``` @@ -123,27 +127,16 @@ You can create custom evals from the **UI** or via the **SDK** (by calling the R ---- - -## Next Steps +## Dive deeper - - - Run evals from the UI or SDK. - - - Add your custom eval to a group and run it with others. - - - Bring your own model for evaluations. - - - Built-in models available for evals. + + + Run your custom eval on any surface - - Run evals automatically in your pipeline. + + Bring your own LLM as the evaluator - - How evaluation fits into the platform. + + Run your custom eval on every pull request diff --git a/src/pages/docs/evaluation/features/custom-models.mdx b/src/pages/docs/evaluation/guides/custom-models.mdx similarity index 91% rename from src/pages/docs/evaluation/features/custom-models.mdx rename to src/pages/docs/evaluation/guides/custom-models.mdx index d9d8a4a5..b362c630 100644 --- a/src/pages/docs/evaluation/features/custom-models.mdx +++ b/src/pages/docs/evaluation/guides/custom-models.mdx @@ -1,11 +1,11 @@ --- -title: "Use Custom Models for AI Evaluation in Future AGI" +title: "Use custom models" description: "Use your own or third-party models for evaluations in Future AGI via supported providers or a custom API endpoint with full configuration control." --- ## About -Evaluations need a model to act as the judge: to read each response and decide whether it passes, fails, or scores within a range. Custom models let you bring your own judge instead of using Future AGI's built-in models. +Evaluations need a model to act as the evaluator: to read each response and decide whether it passes, fails, or scores within a range. Custom models let you bring your own LLM as the evaluator instead of using Future AGI's built-in models. This matters when you have a model that knows your domain better, when you need inference to stay within a specific cloud provider or region, or when you want to track evaluation costs against a model you already pay for. @@ -17,13 +17,13 @@ Two ways to connect: - **Custom endpoint**: Connect any model behind an HTTP API, including self-hosted, fine-tuned, or proxy deployments. -Learn how to define eval rules that use your model: [Create custom evals](/docs/evaluation/features/custom). +Learn how to define eval rules that use your model: [Create custom evals](/docs/evaluation/guides/custom-evals). --- ## When to use -- **Control cost and compliance**: Bring your own model and set token costs so evaluation spend is tracked. Keep inference in your chosen region or provider for compliance. +- **Control cost and compliance**: Bring your own LLM and set token costs so evaluation spend is tracked. Keep inference in your chosen region or provider for compliance. - **Evaluate with a fine-tuned or internal model**: Run evals with a model tuned on your domain or hosted in-house by connecting it via the custom endpoint option. - **Unify evals across providers**: Add multiple models and use the same eval templates against each to compare quality or cost. - **Proxy or third-party APIs**: Connect any API-compatible endpoint when it is not one of the built-in providers. @@ -120,19 +120,19 @@ Fields you may see when adding a model (from a provider or custom). **Applies to ## Next Steps - + Run a single eval from the UI or SDK. - + Define eval rules and select your custom model. Run multiple evals together as a group. - + Built-in models available for evals. - + Run evals automatically in your pipeline. diff --git a/src/pages/docs/evaluation/guides/running-evaluations.mdx b/src/pages/docs/evaluation/guides/running-evaluations.mdx new file mode 100644 index 00000000..b949adcf --- /dev/null +++ b/src/pages/docs/evaluation/guides/running-evaluations.mdx @@ -0,0 +1,54 @@ +--- +title: "Running Evaluations" +description: "Where to run evals on Future AGI, and the guide for each surface" +--- + +Evaluations have no separate quickstart. An eval scores data, so you run it wherever your data already lives: a dataset, your live traces, a simulation, the playground, the SDK, or a CI job. The template you run is the same on every surface. The surface only changes what you feed it and where the score lands. + +This page is the entry point. Pick your surface below and follow its guide. + +## Where evals run + + D["Dataset"] + T --> TR["Traces"] + T --> S["Simulation"] + T --> P["Playground"] + T --> K["SDK"] + T --> C["CI/CD"] + D --> SC["Comparable scores"] + TR --> SC + S --> SC + P --> SC + K --> SC + C --> SC +`} /> + +In evaluation, **offline** and **online** describe the data, not your connection: an offline run scores data you've already stored (dataset rows, experiment outputs), an online run scores live production traffic as it arrives. + +| Surface | Where you run it | Guide | +|---|---|---| +| Dataset and experiments | Offline, over every row of a dataset | [Run experiments](/docs/dataset/features/experiments) | +| Traces | Online, on live spans, traces, and sessions | [Set up evals in Observe](/docs/observe/guides/setup-evals) | +| Simulation | Over simulated conversations | [Run a simulation](/docs/simulation/features/run-simulation) | +| Playground | On one sample input, to see the verdict and reason before you commit | [Explore playground](/docs/evaluation/guides/explore-playground) | +| SDK | Programmatic runs, with local Code Evals that need no API key | [Evaluation SDK](/docs/sdk/evals) | +| CI/CD | On every pull request, gating the merge on eval scores | [Evaluate in CI/CD](/docs/evaluation/guides/cicd) | + +The online how-to for scoring live traces lives in Observe; this hub only points to it. Everything else runs from its own module, with the same templates and the same scores. + +## Dive deeper + + + + Write your own when no built-in fits + + + 156 templates across 14 groups + + + Pick an evaluator model for your evals + + diff --git a/src/pages/docs/evaluation/guides/using-connectors.mdx b/src/pages/docs/evaluation/guides/using-connectors.mdx new file mode 100644 index 00000000..584ec58d --- /dev/null +++ b/src/pages/docs/evaluation/guides/using-connectors.mdx @@ -0,0 +1,54 @@ +--- +title: "Using Connectors" +description: "Give an Agent Evaluator external tools it can call while it judges" +--- + +A **connector** is an external tool you attach to an eval so the evaluator can use it while it judges. On its own, an [evaluator model](/docs/evaluation/concepts/evaluator-models) reasons only over the text you hand it. With tools, an [Agent Evaluator](/docs/evaluation/concepts/eval-types) can act mid-judgment: when the verdict needs a fact the response does not carry, it calls a tool, reads what comes back, and only then decides. + + AE["Agent Evaluator"] + AE -->|"needs an outside fact"| T["Tools
web search, KB lookup, your MCP connector"] + T -->|"result"| AE + AE --> V["Verdict + reason"] + style T fill:#2f2f2f,stroke:#ffffff,stroke-width:2px +`} /> + +## What the evaluator can call + +- **Built-in tools**, no setup beyond a toggle: web search when you turn on **Use Internet**, lookups in any knowledge base you attach, and walking the trace when the eval runs on one +- **Your connectors**: any tool you expose over the Model Context Protocol (MCP), which the evaluator calls the same way it calls a built-in + + +Only Agent Evaluators use tools. A single-pass LLM-as-Judge or a Code Eval scores what it's given, so the tool options don't appear for those types. + + +## When to reach for a connector + +Attach one when the verdict depends on something only your systems know: checking a claim against your own database, confirming an ID actually exists, verifying a rule that lives in an internal service. For public facts, the built-in web search usually covers it. + +## Attach a connector to an eval + + + + Connectors are registered once for your organization and shared across evals (and with Falcon AI). If your MCP server isn't registered yet, follow [MCP Connectors](/docs/falcon-ai/features/mcp-connectors) to add it. + + + Create or open an Agent Evaluator eval, and in the model bar under the instructions, open the **+** menu and choose **Connectors**. Select the connectors this eval should be able to call. The same menu holds the **Use Internet** toggle and knowledge bases. + + + Save the eval and run it anywhere evals run. The evaluator decides on each run whether a tool call is worth making, and the result comes back as a verdict and reason like any other eval. + + + +## Dive deeper + + + + Bring your own model as the evaluator + + + Turn eval scores into a merge gate + + diff --git a/src/pages/docs/evaluation/index.mdx b/src/pages/docs/evaluation/index.mdx index dd526cb4..71875e71 100644 --- a/src/pages/docs/evaluation/index.mdx +++ b/src/pages/docs/evaluation/index.mdx @@ -1,45 +1,88 @@ --- -title: "Future AGI Evaluation: Measure Prompt and Agent Quality" -description: "Measure and compare the quality of prompts and agents across datasets, simulations, and experiments using built-in or custom eval templates." +title: "Overview" +description: "Measure how well your AI achieves the user's goal" --- -## About +Evaluation is Future AGI's quality measurement layer: it scores every response your AI produces against a definition of "good" that you control. And it's audio native, so voice agents get scored as directly as text ones. -Evaluation is Future AGI's quality measurement layer. It gives you a consistent, repeatable way to measure whether your prompts and agents are behaving correctly and whether changes you make improve things or introduce regressions. +## What is evaluation? -There are two building blocks: **eval templates** define what to measure (task completion, tone, hallucination, safety, factual accuracy, or a custom rule you write yourself), and **eval configs** define how to measure (the judge model, input mapping, and run settings). Combine them with your data and you get a score, a pass/fail result, and an optional explanation per row or call, plus aggregated summaries, KPIs, and trend data across runs. +An LLM's output is free text, and whether it is *right* is a judgment. Evaluation turns that judgment into a measurement. You define what "good" means once, as an eval, and the platform applies it to every response automatically, the same way every time. -Evaluations run across every surface in Future AGI: datasets, simulations, experiments, playground, replay sessions, and CI/CD pipelines. You can also run them programmatically via the SDK. Using the same templates and configs across contexts keeps results directly comparable without redefining your quality criteria each time. +Each eval scores one goal on one metric, for example: -Future AGI ships 70+ built-in templates covering quality, safety, factuality, RAG retrieval, format, bias, audio, and image evaluation. You can also create custom templates and bundle any combination into **eval groups** to apply multiple evals in a single run. +- **Task completion**: did the response do what was asked +- **Factual accuracy**: are its claims true to the source +- **Safety**: is it free of toxicity, prompt injection, and data leaks +- **Tone**: does it speak the way your product should -## How Evaluation Connects to Other Features +Every run returns a score (pass/fail, a number, or a category) and, when an evaluator model is involved, a plain-language reason. Because the definition is fixed, the same bar is applied to every response: scores stay comparable across a dataset, a live trace, or a pull request, and a threshold on that score becomes a decision you can automate instead of a vibe check. -- **Datasets**: Run evals across dataset rows and store scores as new columns. [Learn more](/docs/dataset) -- **Simulation**: Score simulated agent conversations for quality, context retention, and escalation. [Learn more](/docs/simulation) -- **Optimization**: Feed eval results into prompt optimization to improve quality automatically. [Learn more](/docs/optimization) -- **CI/CD**: Gate pull requests on eval scores to catch regressions before they ship. [Learn more](/docs/evaluation/features/cicd) -- **Error Feed**: Eval-powered scoring for every traced agent execution. [Learn more](/docs/error-feed) +## The quality loop -## Getting Started +Evaluation is the middle of a loop, not a standalone tool: + +what the agent did"] --> E["Evaluate
score it against your bar"] + E --> OPT["Optimize
the prompt or model"] + OPT --> G["Enforce
gate the release"] + G --> O + style E fill:#2f2f2f,stroke:#ffffff,stroke-width:2px +`} /> + +- [Observe](/docs/observe) records what your agent did, and its traces become eval inputs +- Evaluate scores each response and shows the result back on the trace +- [Optimization](/docs/optimization) consumes the scores to improve the prompt or model +- A threshold turns the score into a merge gate in [CI/CD](/docs/evaluation/guides/cicd) before anything ships + +## Start here - - Run the first eval from the UI or SDK in minutes. + + Every surface you can run an eval on, and how to get going - 70+ templates: quality, safety, factuality, RAG, and more. + 156 ready evaluators across quality, safety, RAG, format, and more - - Define your own eval rules and output types. + + +## Understand the model + +Four objects carry the whole product: + +- A **[template](/docs/evaluation/concepts/eval-templates)** defines what to measure +- A **config** points it at your data +- A **run** executes +- A **[score](/docs/evaluation/concepts/output-types)** comes out + +A few short pages give you the whole mental model: + + + + The template, config, run, and score model, and the quality loop + + + Agent Evaluator, LLM-as-Judge, and Code Eval, and when to reach for each - - Bundle multiple evals and run them together. + + What decides the score, and how to choose one - - Pick the right evaluation model for your task. + + The value, optional reason, and aggregates you get back + + + +## How it connects + +Beyond the loop, the same evals run offline: [datasets](/docs/dataset) score rows in bulk, and [simulations](/docs/simulation) score synthetic conversations before anything reaches production. + + + + Attach an eval to a project's traces and score new data as it arrives - - Run evals automatically on every pull request. + + Turn an eval threshold into a merge gate that blocks regressions diff --git a/src/pages/docs/faq.mdx b/src/pages/docs/faq.mdx index 108cf801..e1ccef7b 100644 --- a/src/pages/docs/faq.mdx +++ b/src/pages/docs/faq.mdx @@ -33,7 +33,7 @@ Future AGI has 70+ built-in evaluation templates covering quality, safety, factu **How do I run my first evaluation?** -See [Evaluate via Platform & SDK](/docs/evaluation/features/evaluate) for step-by-step instructions using the UI or Python SDK. +See [Evaluate via Platform & SDK](/docs/evaluation/guides/running-evaluations) for step-by-step instructions using the UI or Python SDK. **How do I evaluate RAG applications?** @@ -205,7 +205,7 @@ Check that `FI_API_KEY` and `FI_SECRET_KEY` are set correctly. Verify the instru **Evaluations are failing with "model_name required".** -Some built-in evaluations require a judge model. Pass `model_name="turing_flash"` (or another judge model) in your evaluate call. See [Judge Models](/docs/evaluation/concepts/judge-models). +Some built-in evaluations require an evaluator model. Pass `model_name="turing_flash"` (or another evaluator model) in your evaluate call. See [Evaluator Models](/docs/evaluation/concepts/evaluator-models). **I can't find my API keys.** diff --git a/src/pages/docs/simulation/features/prompt-simulation.mdx b/src/pages/docs/simulation/features/prompt-simulation.mdx index 6a617ba0..a602dd37 100644 --- a/src/pages/docs/simulation/features/prompt-simulation.mdx +++ b/src/pages/docs/simulation/features/prompt-simulation.mdx @@ -159,7 +159,7 @@ Before you start: have a **prompt template** with at least one saved **prompt ve Use AI-powered suggestions to improve your prompt based on simulation results. - + Build evaluations tailored to your specific use case.