Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/components/Sidebar.astro
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@ const isApiTab = activeTab?.tab === 'API';

function inferApiMethod(title: string): { method: string; css: string } | null {
const t = title.toLowerCase();
if (/\b(list|get|retrieve|health|find|export|progress|analytics|agreement)\b/.test(t)) {
if (/\b(list|get|retrieve|health|find|export|progress|analytics|agreement|compare|stats|summary|voices|tts)\b/.test(t)) {
return { method: 'GET', css: 'api-method-get' };
}
if (/\b(create|add|generate|execute|submit|assign|bulk|complete|skip|release|pause|unpause|check|upload)\b/.test(t)) {
if (/\b(create|add|generate|execute|submit|assign|bulk|complete|skip|release|pause|unpause|check|upload|start|duplicate|fetch|run|rerun|cancel|clone|merge)\b/.test(t)) {
return { method: 'POST', css: 'api-method-post' };
}
if (/\b(delete|remove)\b/.test(t)) {
Expand Down
54 changes: 29 additions & 25 deletions src/components/docs/ApiCollapsible.astro
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*
* Used inside API reference pages to collapse nested object properties.
* Renders as a pill/button that toggles visibility of child content.
* Supports nesting (collapsible inside collapsible).
*
* Usage:
* <ApiCollapsible title="Show 3 properties">
Expand All @@ -15,15 +16,14 @@ interface Props {
}

const { title = 'Show properties' } = Astro.props;
const cid = `apic-${Math.random().toString(36).slice(2, 8)}`;
---

<div class="api-collapsible" id={cid}>
<button type="button" class="api-collapsible-trigger" data-apic-trigger>
<span class="api-collapsible-icon" data-apic-icon>+</span>
<span class="api-collapsible-label" data-apic-label>{title}</span>
<div class="api-collapsible">
<button type="button" class="api-collapsible-trigger">
<span class="api-collapsible-icon">+</span>
<span class="api-collapsible-label">{title}</span>
</button>
<div class="api-collapsible-content" data-apic-content>
<div class="api-collapsible-content">
<slot />
</div>
</div>
Expand All @@ -34,21 +34,33 @@ function initApiCollapsibles() {
if (el._apicBound) return;
el._apicBound = true;

var trigger = el.querySelector('[data-apic-trigger]');
var icon = el.querySelector('[data-apic-icon]');
var label = el.querySelector('[data-apic-label]');
// Use direct children only — not nested collapsible triggers
var children = el.children;
var trigger = null;
var content = null;
for (var c = 0; c < children.length; c++) {
if (children[c].classList.contains('api-collapsible-trigger')) trigger = children[c];
if (children[c].classList.contains('api-collapsible-content')) content = children[c];
}
if (!trigger || !content) return;

var icon = trigger.querySelector('.api-collapsible-icon');
var label = trigger.querySelector('.api-collapsible-label');
var isOpen = false;
var showTitle = label ? label.textContent : 'Show properties';
var hideTitle = showTitle.replace(/^Show\b/, 'Hide');

if (trigger) {
trigger.addEventListener('click', function() {
isOpen = !isOpen;
el.classList.toggle('api-collapsible-open', isOpen);
if (icon) icon.textContent = isOpen ? '\u00d7' : '+';
if (label) label.textContent = isOpen ? hideTitle : showTitle;
});
}
trigger.addEventListener('click', function(e) {
e.stopPropagation();
isOpen = !isOpen;
if (isOpen) {
content.style.display = 'block';
} else {
content.style.display = 'none';
}
if (icon) icon.textContent = isOpen ? '\u00d7' : '+';
if (label) label.textContent = isOpen ? hideTitle : showTitle;
});
});
}
initApiCollapsibles();
Expand Down Expand Up @@ -98,10 +110,6 @@ document.addEventListener('astro:page-load', initApiCollapsibles);
flex-shrink: 0;
}

.api-collapsible-open .api-collapsible-icon {
color: #a1a1aa;
}

.api-collapsible-label {
font-size: 13px;
font-weight: 500;
Expand All @@ -111,8 +119,4 @@ document.addEventListener('astro:page-load', initApiCollapsibles);
display: none;
padding-top: 12px;
}

.api-collapsible-open .api-collapsible-content {
display: block;
}
</style>
19 changes: 18 additions & 1 deletion src/lib/api-navigation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -431,8 +431,25 @@ export const apiNavigation: ApiNavGroup[] = [
{
"title": "Datasets",
"items": [
{ "title": "Get Dataset", "href": "/docs/api/datasets/get-dataset", "method": "GET" },
{ "title": "List Datasets", "href": "/docs/api/datasets/list-datasets", "method": "GET" },
{ "title": "Create Dataset", "href": "/docs/api/datasets/create-dataset", "method": "POST" },
{ "title": "Upload Dataset from File", "href": "/docs/api/datasets/upload-dataset", "method": "POST" }
{ "title": "Create Empty Dataset", "href": "/docs/api/datasets/create-empty-dataset", "method": "POST" },
{ "title": "Upload Dataset from File", "href": "/docs/api/datasets/upload-dataset", "method": "POST" },
{ "title": "Create from HuggingFace", "href": "/docs/api/datasets/create-dataset-from-huggingface", "method": "POST" },
{ "title": "Clone Dataset", "href": "/docs/api/datasets/clone-dataset", "method": "POST" },
{ "title": "Duplicate Dataset", "href": "/docs/api/datasets/duplicate-dataset", "method": "POST" },
{ "title": "Add as New Dataset", "href": "/docs/api/datasets/add-as-new", "method": "POST" },
{ "title": "Update Dataset", "href": "/docs/api/datasets/update-dataset", "method": "POST" },
{ "title": "Merge Dataset", "href": "/docs/api/datasets/merge-dataset", "method": "POST" },
{ "title": "Delete Dataset", "href": "/docs/api/datasets/delete-dataset", "method": "DELETE" },
{ "title": "Add Rows from File", "href": "/docs/api/datasets/add-rows-from-file", "method": "POST" },
{ "title": "Add Empty Rows", "href": "/docs/api/datasets/add-empty-rows", "method": "POST" },
{ "title": "Add Rows from Existing", "href": "/docs/api/datasets/add-rows-from-existing", "method": "POST" },
{ "title": "Add Rows from HuggingFace", "href": "/docs/api/datasets/add-rows-from-huggingface", "method": "POST" },
{ "title": "Duplicate Rows", "href": "/docs/api/datasets/duplicate-rows", "method": "POST" },
{ "title": "Delete Rows", "href": "/docs/api/datasets/delete-rows", "method": "DELETE" },
{ "title": "Update Cell Value", "href": "/docs/api/datasets/update-cell-value", "method": "PUT" }
]
},
{
Expand Down
65 changes: 65 additions & 0 deletions src/lib/navigation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -988,6 +988,20 @@ export const tabNavigation: NavTab[] = [
{ title: 'Get Eval Log Details', href: '/docs/api/eval-logs-metrics/getevallogdetails' },
]
},
{
title: 'Dataset Evals',
items: [
{ title: 'Get Eval Template Names', href: '/docs/api/dataset-evals/get-eval-template-names' },
{ title: 'Create Custom Eval Template', href: '/docs/api/dataset-evals/create-custom-eval-template' },
{ title: 'List Dataset Evals', href: '/docs/api/dataset-evals/list-dataset-evals' },
{ title: 'Get Eval Structure', href: '/docs/api/dataset-evals/get-eval-structure' },
{ title: 'Add Dataset Eval', href: '/docs/api/dataset-evals/add-dataset-eval' },
{ title: 'Start Evals Process', href: '/docs/api/dataset-evals/start-evals-process' },
{ title: 'Delete Dataset Eval', href: '/docs/api/dataset-evals/delete-dataset-eval' },
{ title: 'Edit and Run Eval', href: '/docs/api/dataset-evals/edit-and-run-eval' },
{ title: 'Get Eval Metrics', href: '/docs/api/dataset-evals/get-eval-metrics' },
]
},
{
title: 'Scenarios',
items: [
Expand Down Expand Up @@ -1091,8 +1105,59 @@ export const tabNavigation: NavTab[] = [
{
title: 'Datasets',
items: [
{ title: 'Get Dataset', href: '/docs/api/datasets/get-dataset' },
{ title: 'List Datasets', href: '/docs/api/datasets/list-datasets' },
{ title: 'Create Dataset', href: '/docs/api/datasets/create-dataset' },
{ title: 'Create Empty Dataset', href: '/docs/api/datasets/create-empty-dataset' },
{ title: 'Upload Dataset from File', href: '/docs/api/datasets/upload-dataset' },
{ title: 'Create from HuggingFace', href: '/docs/api/datasets/create-dataset-from-huggingface' },
{ title: 'Clone Dataset', href: '/docs/api/datasets/clone-dataset' },
{ title: 'Duplicate Dataset', href: '/docs/api/datasets/duplicate-dataset' },
{ title: 'Add as New Dataset', href: '/docs/api/datasets/add-as-new' },
{ title: 'Update Dataset', href: '/docs/api/datasets/update-dataset' },
{ title: 'Merge Dataset', href: '/docs/api/datasets/merge-dataset' },
{ title: 'Delete Dataset', href: '/docs/api/datasets/delete-dataset' },
{ title: 'Add Rows from File', href: '/docs/api/datasets/add-rows-from-file' },
{ title: 'Add Empty Rows', href: '/docs/api/datasets/add-empty-rows' },
{ title: 'Add Rows from Existing', href: '/docs/api/datasets/add-rows-from-existing' },
{ title: 'Add Rows from HuggingFace', href: '/docs/api/datasets/add-rows-from-huggingface' },
{ title: 'Duplicate Rows', href: '/docs/api/datasets/duplicate-rows' },
{ title: 'Delete Rows', href: '/docs/api/datasets/delete-rows' },
{ title: 'Update Cell Value', href: '/docs/api/datasets/update-cell-value' },
]
},
{
title: 'Dataset Columns',
items: [
{ title: 'Get Column Details', href: '/docs/api/datasets/columns/get-column-details' },
{ title: 'Get Column Config', href: '/docs/api/datasets/columns/get-column-config' },
{ title: 'Add Static Column', href: '/docs/api/datasets/columns/add-static-column' },
{ title: 'Add Multiple Static Columns', href: '/docs/api/datasets/columns/add-multiple-static-columns' },
{ title: 'Add Columns', href: '/docs/api/datasets/columns/add-columns' },
{ title: 'Update Column Name', href: '/docs/api/datasets/columns/update-column-name' },
{ title: 'Update Column Type', href: '/docs/api/datasets/columns/update-column-type' },
{ title: 'Delete Column', href: '/docs/api/datasets/columns/delete-column' },
]
},
{
title: 'Dataset Run Prompt',
items: [
{ title: 'Add Run Prompt Column', href: '/docs/api/datasets/run-prompt/add-run-prompt-column' },
{ title: 'Edit Run Prompt Column', href: '/docs/api/datasets/run-prompt/edit-run-prompt-column' },
{ title: 'Get Run Prompt Config', href: '/docs/api/datasets/run-prompt/retrieve-run-prompt-column-config' },
{ title: 'Get Run Prompt Options', href: '/docs/api/datasets/run-prompt/retrieve-run-prompt-options' },
{ title: 'Get Model Voices', href: '/docs/api/datasets/run-prompt/get-model-voices' },
{ title: 'TTS Voices', href: '/docs/api/datasets/run-prompt/tts-voices' },
{ title: 'Get Column Values', href: '/docs/api/datasets/run-prompt/get-column-values' },
]
},
{
title: 'Dataset Analytics',
items: [
{ title: 'Run Prompt Stats', href: '/docs/api/datasets/analytics/run-prompt-stats' },
{ title: 'Eval Stats', href: '/docs/api/datasets/analytics/eval-stats' },
{ title: 'Annotation Summary', href: '/docs/api/datasets/analytics/annotation-summary' },
{ title: 'Explanation Summary', href: '/docs/api/datasets/analytics/explanation-summary' },
]
},
{
Expand Down
134 changes: 134 additions & 0 deletions src/pages/docs/api/dataset-evals/add-dataset-eval.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
---
title: "Add Dataset Eval"
description: "Add an evaluation to a dataset by selecting a template and configuring the key mapping."
---

<ApiPlayground
method="POST"
endpoint="/model-hub/develops/{dataset_id}/add_user_eval/"
baseUrl="https://api.futureagi.com"
parameters={[
{"name": "dataset_id", "in": "path", "required": true, "description": "UUID of the dataset.", "type": "string"}
]}
requestBody={{
name: "Hallucination Check",
template_id: "d4e5f6a7-b8c9-0123-def0-123456789abc",
config: {
config: {},
params: {},
mapping: {
input: "question_column",
output: "response_column"
},
reason_column: true
},
model: "turing_large",
run: false,
save_as_template: false
}}
responseExample={{
data: "success",
success: true
}}
responseStatus={200}
responseStatusText="OK"
/>

<ApiSection title="Authentication">
<ParamField name="X-Api-Key" type="string" required>
Your Future AGI API key used to authenticate requests. You can find and manage your API keys in the [Dashboard](https://app.futureagi.com) under Settings.
</ParamField>
<ParamField name="X-Secret-Key" type="string" required>
Your Future AGI secret key, used alongside the API key for request authentication. This is generated when you create an API key in the [Dashboard](https://app.futureagi.com).
</ParamField>
</ApiSection>

<ApiSection title="Path parameters">
<ParamField path="dataset_id" type="string" required>
The universally unique identifier (UUID) of the dataset to which the evaluation will be added. The dataset must exist within your organization and be accessible with your current API credentials.
</ParamField>
</ApiSection>

<ApiSection title="Request body">
<ParamField body="name" type="string" required>
The human-readable name for the evaluation. Maximum 50 characters. This name is displayed in the dataset's eval sidebar and must be unique within the dataset for your organization.
</ParamField>

<ParamField body="template_id" type="string" required>
The universally unique identifier (UUID) of the eval template to use for this evaluation. Maximum 500 characters. The template defines the evaluation criteria, required variable keys, and output type. It must be accessible by your organization (either a built-in template or one you created).
</ParamField>

<ParamField body="config" type="object" required>
The configuration object that controls how the evaluation is executed against dataset rows. This includes template-specific settings, runtime parameters, and the variable-to-column mapping.

<ApiCollapsible title="Show 4 properties">
<ResponseField name="config" type="object">
Template-specific configuration parameters that customize the evaluation behavior. The available options depend on the eval template being used.
</ResponseField>

<ResponseField name="params" type="object">
Runtime parameters passed to the evaluation engine during processing. These may include thresholds, weights, or other execution-time settings.
</ResponseField>

<ResponseField name="mapping" type="object">
A key-value mapping that connects the eval template's variable keys to actual dataset column names. All required keys defined by the template must be present in this mapping. For example, `{"input": "question_column", "output": "response_column"}` maps the template's `input` variable to the `question_column` in the dataset.
</ResponseField>

<ResponseField name="reason_column" type="boolean">
Whether to create an additional reason column alongside the eval result column. When `true`, the evaluation engine generates an explanation for each evaluation outcome, stored in a separate column.
</ResponseField>
</ApiCollapsible>
</ParamField>

<ParamField body="kb_id" type="string" optional>
The universally unique identifier (UUID) of a knowledge base to associate with this evaluation. The knowledge base must exist within your organization and provides reference data that the evaluation can use to assess responses.
</ParamField>

<ParamField body="error_localizer" type="boolean" optional>
Whether to enable error localization for this evaluation. When `true`, the evaluation engine identifies and highlights the specific portions of the output that caused evaluation failures. Default is `false`.
</ParamField>

<ParamField body="model" type="string" optional>
The model to use for running the evaluation. Maximum 100 characters. This determines which LLM-based evaluation engine processes each row. Default is `turing_large`.
</ParamField>

<ParamField body="run" type="boolean" optional>
Whether to immediately start running the evaluation after adding it to the dataset. When `true`, the eval column is created and all rows are queued for processing. When `false`, the evaluation is configured but not executed until explicitly triggered. Default is `false`.
</ParamField>

<ParamField body="save_as_template" type="boolean" optional>
Whether to save this evaluation configuration as a new reusable eval template. When `true`, a new template is created with the provided `name`, which must be unique across all templates in your organization. Default is `false`.
</ParamField>
</ApiSection>

<ApiSection title="Response" status={200} statusText="OK">
<ResponseField name="data" type="string">
A confirmation message indicating the evaluation was successfully added to the dataset. Typically returns `success`.
</ResponseField>

<ResponseField name="success" type="boolean">
A boolean flag indicating whether the request completed successfully. Returns `true` when the evaluation is added without errors.
</ResponseField>
</ApiSection>

<ApiSection title="Errors">
<ParamField name="400" type="Bad Request">
The request was malformed or contained invalid parameters. This can occur when: the dataset does not belong to your organization; an eval with the same name already exists in the dataset; required mapping keys from the template are missing; the eval name does not meet naming requirements; the specified template does not exist or is not accessible; or `save_as_template` is `true` but the template name already exists.
</ParamField>

<ParamField name="401" type="Unauthorized">
Authentication credentials were not provided or are invalid. Verify that both `X-Api-Key` and `X-Secret-Key` headers are present and contain valid keys.
</ParamField>

<ParamField name="404" type="Not Found">
The specified dataset was not found. Verify that the `dataset_id` in the path is correct and that your API credentials have access to the target dataset.
</ParamField>

<ParamField name="429" type="Too Many Requests">
Resource limit reached. Your organization has exceeded the evaluation addition quota. Contact support or upgrade your plan to increase limits.
</ParamField>

<ParamField name="500" type="Internal Server Error">
An unexpected error occurred on the server while adding the evaluation. If this persists, contact Future AGI support with details of your request.
</ParamField>
</ApiSection>
Loading