Skip to content

refine instruction testing #458

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
iceljc wants to merge 1 commit into
base: main
Choose a base branch
from
Open

refine instruction testing #458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/lib/common/modals/StateModal.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@
<div class="min-w-0" style="flex: 0.4;">
{#if idx === 0}
<label for={`stm-key-${idx}`} class="stm-label">
{`Key ${validateKey ? '*' : ''}`}
{`Name ${validateKey ? '*' : ''}`}
</label>
{/if}
<input
Expand Down
8 changes: 5 additions & 3 deletions src/routes/page/instruction/instruction-components/instruction-agent.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* @type {{
* agents?: import('$agentTypes').AgentModel[],
* disabled?: boolean,
* onSelectAgent?: (detail: { agent: import('$agentTypes').AgentModel | null, template: any }) => void
* onSelectAgent?: (detail: { agent: import('$agentTypes').AgentModel | null, template: any, llmConfig: any }) => void
* }}
*/
let {
Expand Down Expand Up @@ -45,7 +45,8 @@
name: x.name,
label: x.name,
value: x.name,
content: x.content
content: x.content,
llm_config: x.llm_config || null
})) || [];

fireSelectAgent();
Expand All @@ -62,7 +63,8 @@
function fireSelectAgent() {
onSelectAgent?.({
agent: selectedAgent || null,
template: selectedTemplate || null
template: selectedTemplate || null,
llmConfig: selectedTemplate?.llm_config || selectedAgent?.llm_config || null
});
}
</script>
Expand Down
95 changes: 93 additions & 2 deletions src/routes/page/instruction/instruction-components/instruction-llm.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,35 @@
<script>
import Select from '$lib/common/dropdowns/Select.svelte';
import { INTEGER_REGEX } from '$lib/helpers/constants';
import { ReasoningEffortLevel } from '$lib/helpers/enums';

/**
* @type {{
* llmConfigs?: import('$commonTypes').LlmConfig[],
* disabled?: boolean,
* selectedProvider?: import('$commonTypes').LlmConfig | null | undefined,
* selectedModel?: string | null,
* onSelectLlm?: (detail: { provider: import('$commonTypes').LlmConfig | null, model: string | null }) => void
* selectedReasoningEffortLevel?: string | null,
* selectedMaxOutputTokens?: number | null,
* onSelectLlm?: (detail: { provider: import('$commonTypes').LlmConfig | null, model: string | null, reasoning_effort_level: string | null, max_output_tokens: number | null }) => void
* }}
*/
let {
llmConfigs = [],
disabled = false,
selectedProvider = $bindable(null),
selectedModel = $bindable(null),
selectedReasoningEffortLevel = $bindable(null),
selectedMaxOutputTokens = $bindable(null),
onSelectLlm
} = $props();

/** @type {import('$commonTypes').LabelValuePair[]} */
const defaultReasonLevelOptions = Object.entries(ReasoningEffortLevel).map(([k, v]) => ({
value: v,
label: v
}));

/** @type {import('$commonTypes').LabelValuePair[]} */
let providerOptions = $derived(
llmConfigs?.map(x => ({
Expand All @@ -34,12 +46,32 @@
}))?.sort((a, b) => a.label.localeCompare(b.label)) || []
);

let selectedModelSetting = $derived(
selectedProvider?.models?.find(x => x.name === selectedModel) || null
);

let isReasoningModel = $derived(selectedModelSetting?.reasoning != null);

/** @type {import('$commonTypes').LabelValuePair[]} */
let reasoningLevelOptions = $derived.by(() => {
if (!isReasoningModel) {
return defaultReasonLevelOptions;
}
const customOptions = selectedModelSetting?.reasoning?.parameters?.EffortLevel?.options;
if (customOptions?.length > 0) {
// @ts-ignore
return customOptions.map(x => ({ value: x, label: x }));
}
return defaultReasonLevelOptions;
});

/** @param {any} e */
function selectProvider(e) {
// @ts-ignore
const selectedValues = e.detail.selecteds?.map(x => x.value) || [];
selectedProvider = selectedValues.length > 0 ? llmConfigs?.find(x => x.provider === selectedValues[0]) || null : null;
selectedModel = null;
selectedReasoningEffortLevel = null;
fireSelectLlm();
}

Expand All @@ -48,13 +80,39 @@
// @ts-ignore
const selectedValues = e.detail.selecteds?.map(x => x.value) || [];
selectedModel = selectedValues.length > 0 ? modelOptions.find(x => x.value === selectedValues[0])?.value : null;
selectedReasoningEffortLevel = null;
fireSelectLlm();
}

/** @param {any} e */
function selectReasoningEffortLevel(e) {
// @ts-ignore
const selectedValues = e.detail.selecteds?.map(x => x.value) || [];
selectedReasoningEffortLevel = selectedValues.length > 0 ? selectedValues[0] : null;
fireSelectLlm();
}

/** @param {any} e */
function changeMaxOutputTokens(e) {
const value = Number(e.target.value) || 0;
selectedMaxOutputTokens = value > 0 ? value : null;
fireSelectLlm();
}

/** @param {any} e */
function validateIntegerInput(e) {
const reg = new RegExp(INTEGER_REGEX, 'g');
if (e.key !== 'Backspace' && !reg.test(e.key)) {
e.preventDefault();
}
}

function fireSelectLlm() {
onSelectLlm?.({
provider: selectedProvider || null,
model: selectedModel
model: selectedModel,
reasoning_effort_level: selectedReasoningEffortLevel || null,
max_output_tokens: selectedMaxOutputTokens || null
});
}
</script>
Expand Down Expand Up @@ -92,4 +150,37 @@
onselect={e => selectModel(e)}
/>
</div>

<div>
<label for="max-output-tokens" class="mb-1.5 inline-flex items-center gap-1.5 text-xs font-semibold uppercase tracking-wider text-primary">
<i class="mdi mdi-counter text-sm leading-none"></i>
Max output tokens
</label>
<input
id="max-output-tokens"
type="number"
class="form-control block w-full"
disabled={disabled}
value={selectedMaxOutputTokens || ''}
onkeydown={e => validateIntegerInput(e)}
onchange={e => changeMaxOutputTokens(e)}
/>
</div>

{#if isReasoningModel}
<div>
<label for="reasoning-effort-select" class="mb-1.5 inline-flex items-center gap-1.5 text-xs font-semibold uppercase tracking-wider text-primary">
<i class="mdi mdi-brain text-sm leading-none"></i>
Reasoning level
</label>
<Select
tag={'reasoning-effort-select'}
placeholder={'Select a level'}
disabled={disabled}
selectedValues={selectedReasoningEffortLevel ? [selectedReasoningEffortLevel] : []}
options={reasoningLevelOptions}
onselect={e => selectReasoningEffortLevel(e)}
/>
</div>
{/if}
</div>
2 changes: 1 addition & 1 deletion src/routes/page/instruction/instruction-components/instruction-state.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@

<div class="flex flex-col py-3">
<div class="grid grid-cols-[1fr_1fr_auto] gap-2 px-1 pb-1">
<div class="text-xs font-semibold uppercase tracking-wider text-primary">{'Key'}</div>
<div class="text-xs font-semibold uppercase tracking-wider text-primary">{'Name'}</div>
<div class="text-xs font-semibold uppercase tracking-wider text-primary">{'Value'}</div>
<div class="w-5"></div>
</div>
Expand Down
50 changes: 38 additions & 12 deletions src/routes/page/instruction/testing/+page.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@
/** @type {string | null} */
let selectedModel = $state(null);

/** @type {string | null} */
let selectedReasoningEffortLevel = $state(null);

/** @type {number | null} */
let selectedMaxOutputTokens = $state(null);

/** @type {string | null} */
let selectedTemplate = $state(null);

Expand Down Expand Up @@ -102,7 +108,14 @@
isThinking = true;
requestDone = false;
elapsedTime = '';
const formattedStates = formatKeyValues(states);
const clonedStates = [
{ key: 'reasoning_effort_level', value: selectedReasoningEffortLevel ? selectedReasoningEffortLevel : 'disable' }
];
if (selectedMaxOutputTokens && selectedMaxOutputTokens > 0) {
clonedStates.push({ key: 'max_tokens', value: selectedMaxOutputTokens.toString() });
}

const formattedStates = formatKeyValues(states, clonedStates);
const formatedArgs = formatKeyValues(args);
const start = new Date();
const agentId = selectedAgent?.id || '';
Expand All @@ -113,10 +126,10 @@
model: selectedModel,
template: selectedTemplate,
states: formattedStates,
codeOptions: {
codeOptions: selectedCodeScript?.name ? {
script_name: selectedCodeScript?.name,
arguments: formatedArgs
}
} : null
}).then(res => {
result = res?.text || '';
}).finally(() => {
Expand All @@ -127,14 +140,20 @@
});
}

/** @param {import('$commonTypes').KeyValuePair[]} kvs */
function formatKeyValues(kvs) {
return kvs?.filter(x => !!util.trim(x.key) && !!util.trim(x.value)).map(x => {
/**
* @param {import('$commonTypes').KeyValuePair[]} kvs
* @param {import('$commonTypes').KeyValuePair[]} [defaults]
*/
function formatKeyValues(kvs, defaults) {
const primary = kvs || [];
const primaryKeys = new Set(primary.map(x => util.trim(x.key)).filter(k => !!k));
const merged = [...(defaults || []).filter(x => !primaryKeys.has(util.trim(x.key))), ...primary];
return merged.filter(x => !!util.trim(x.key) && !!util.trim(x.value)).map(x => {
return {
key: util.trim(x.key),
value: util.trim(x.value)
};
}) || [];
});
}

/** @param {KeyboardEvent} e */
Expand All @@ -156,7 +175,7 @@
elapsedTime = ''
}

/** @param {{ agent: import('$agentTypes').AgentModel | null, template: any }} detail */
/** @param {{ agent: import('$agentTypes').AgentModel | null, template: any, llmConfig: any }} detail */
function onAgentSelected(detail) {
selectedAgent = detail.agent || null;
let localText = selectedAgent?.instruction;
Expand All @@ -168,10 +187,13 @@
}

instruction = localText || '';
const providerName = selectedAgent?.llm_config?.provider || null;
const modelName = selectedAgent?.llm_config?.model || null;
const llmConfig = detail?.llmConfig || null;
const providerName = llmConfig?.provider || null;
const modelName = llmConfig?.model || null;
selectedProvider = llmConfigs?.find(x => x.provider === providerName) || null;
selectedModel = modelName;
selectedReasoningEffortLevel = llmConfig?.reasoning_effort_level || null;
selectedMaxOutputTokens = llmConfig?.max_output_tokens || null;

if (selectedAgent?.id) {
initAgentCodeScripts(selectedAgent.id);
Expand All @@ -180,10 +202,12 @@
}
}

/** @param {{ provider: import('$commonTypes').LlmConfig | null, model: string | null }} detail */
/** @param {{ provider: import('$commonTypes').LlmConfig | null, model: string | null, reasoning_effort_level: string | null, max_output_tokens: number | null }} detail */
function onLlmSelected(detail) {
selectedProvider = detail.provider || null;
selectedModel = detail.model || '';
selectedReasoningEffortLevel = detail.reasoning_effort_level || null;
selectedMaxOutputTokens = detail.max_output_tokens || null;
}

/** @param {string} agentId */
Expand Down Expand Up @@ -246,7 +270,7 @@
</span>
{/if}
</div>
<div>{text?.length || 0}/{formatNumber(maxLength)}</div>
<div>{formatNumber(text?.length || 0)}/{formatNumber(maxLength)}</div>
</div>

<div class="mt-3 flex justify-end">
Expand Down Expand Up @@ -378,6 +402,8 @@
disabled={isThinking}
bind:selectedProvider={selectedProvider}
bind:selectedModel={selectedModel}
bind:selectedReasoningEffortLevel={selectedReasoningEffortLevel}
bind:selectedMaxOutputTokens={selectedMaxOutputTokens}
onSelectLlm={detail => onLlmSelected(detail)}
/>
</div>
Expand Down
Loading