From 0f8fa8394be658279a455d268194cd2d5a3ee700 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Thu, 28 May 2026 22:03:13 -0700 Subject: [PATCH 01/36] some fixes --- .../datalake/workspace_manager.py | 11 +- src/app/dfSlice.tsx | 28 ++ src/views/DataFormulator.tsx | 34 ++- src/views/DataLoadingChat.tsx | 263 +++++++++--------- src/views/DataThread.tsx | 16 +- src/views/UnifiedDataUploadDialog.tsx | 140 ++++------ src/views/dataLoadingSuggestions.ts | 82 ++++-- tests/backend/data/test_workspace_manager.py | 64 ++++- 8 files changed, 369 insertions(+), 269 deletions(-) diff --git a/py-src/data_formulator/datalake/workspace_manager.py b/py-src/data_formulator/datalake/workspace_manager.py index 679452ca..23e37176 100644 --- a/py-src/data_formulator/datalake/workspace_manager.py +++ b/py-src/data_formulator/datalake/workspace_manager.py @@ -169,6 +169,10 @@ def list_workspaces(self) -> list[dict]: workspace. If a workspace directory lacks this file (legacy), it is auto-repaired via :meth:`_ensure_meta`. + Every workspace directory is listed, including empty + "Untitled Session" entries from data-loading chats. Users + manage (rename/delete) these themselves via the sidebar. + Returns list of {"id": str, "display_name": str, "updated_at": str}. """ workspaces = [] @@ -184,13 +188,16 @@ def list_workspaces(self) -> list[dict]: except Exception: continue + tc = meta.get("tableCount") + cc = meta.get("chartCount") + workspaces.append({ "id": child.name, "display_name": meta.get("displayName", child.name), "created_at": meta.get("createdAt") or meta.get("updatedAt"), "updated_at": meta.get("updatedAt"), - "table_count": meta.get("tableCount"), - "chart_count": meta.get("chartCount"), + "table_count": tc, + "chart_count": cc, }) workspaces.sort(key=lambda w: w.get("updated_at") or "", reverse=True) diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index a3fe5add..89b075ab 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -210,6 +210,17 @@ export interface DataFormulatorState { * Transient — not persisted. */ dataLoadingChatResetCounter: number; + /** + * Pending submission queued for the data-loading chat. Set by any + * surface that wants to hand a prompt off to the chat (the menu + * agent input box, suggestion auto-run, external dialog callers). + * `DataLoadingChat` consumes it on render: it clears the slot and + * sends the carried payload as a fresh user message. Using a single + * redux slot (instead of props + a reset counter) eliminates the + * cross-tick race where the parent's pre-clear would otherwise + * cancel the auto-send for the new prompt. Transient — not persisted. + */ + dataLoadingChatPending: { text: string; images: string[]; attachments: string[] } | null; /** * Pending hand-off from the Data Agent to a peer agent. Set by the * Data Agent's `delegate` action card; consumed by `DataFormulator` @@ -299,6 +310,7 @@ const initialState: DataFormulatorState = { dataLoadingChatMessages: [], dataLoadingChatInProgress: false, dataLoadingChatResetCounter: 0, + dataLoadingChatPending: null, agentHandoffRequest: null, generatedReports: [], @@ -720,6 +732,7 @@ export const dataFormulatorSlice = createSlice({ state.dataLoadingChatMessages = []; state.dataLoadingChatInProgress = false; state.dataLoadingChatResetCounter = (state.dataLoadingChatResetCounter ?? 0) + 1; + state.dataLoadingChatPending = null; state.generatedReports = []; @@ -837,6 +850,7 @@ export const dataFormulatorSlice = createSlice({ config: { ...initialState.config, ...(saved.config || {}) }, dataCleanBlocks: saved.dataCleanBlocks || [], dataLoadingChatMessages: saved.dataLoadingChatMessages || [], + dataLoadingChatPending: null, generatedReports: saved.generatedReports || [], // Reset transient fields @@ -1665,6 +1679,20 @@ export const dataFormulatorSlice = createSlice({ state.dataLoadingChatMessages = []; state.dataLoadingChatInProgress = false; state.dataLoadingChatResetCounter = (state.dataLoadingChatResetCounter ?? 0) + 1; + // Note: `dataLoadingChatPending` is intentionally left + // alone. Callers that want "fresh slate + auto-send the + // new prompt" dispatch `clearChatMessages` followed by + // `setDataLoadingChatPending` in the same tick — clearing + // pending here would race with that ordering. + }, + setDataLoadingChatPending: ( + state, + action: PayloadAction<{ text: string; images: string[]; attachments: string[] }>, + ) => { + state.dataLoadingChatPending = action.payload; + }, + clearDataLoadingChatPending: (state) => { + state.dataLoadingChatPending = null; }, confirmTableLoad: (state, action: PayloadAction<{messageId: string, tableName: string}>) => { const msg = state.dataLoadingChatMessages.find(m => m.id === action.payload.messageId); diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index 2c21c15b..00e8086e 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -301,24 +301,37 @@ export const DataFormulatorFC = ({ }) => { // State for unified data upload dialog const [uploadDialogOpen, setUploadDialogOpen] = useState(false); const [uploadDialogInitialTab, setUploadDialogInitialTab] = useState('menu'); - const [uploadDialogInitialChatPrompt, setUploadDialogInitialChatPrompt] = useState(undefined); - const [uploadDialogInitialChatImages, setUploadDialogInitialChatImages] = useState(undefined); // Loading state for sessions (from Redux, shared with App.tsx) const sessionLoading = useSelector((state: DataFormulatorState) => state.sessionLoading); const sessionLoadingLabel = useSelector((state: DataFormulatorState) => state.sessionLoadingLabel); - const openUploadDialog = (tab: UploadTabType, initialChatPrompt?: string, initialChatImages?: string[]) => { + const openUploadDialog = (tab: UploadTabType) => { // If no workspace is active, generate an ID (backend creates folder lazily on first data op) if (!activeWorkspace) { dispatch(dfActions.setActiveWorkspace({ id: generateSessionId(), displayName: 'Untitled Session' })); } setUploadDialogInitialTab(tab); - setUploadDialogInitialChatPrompt(initialChatPrompt); - setUploadDialogInitialChatImages(initialChatImages); setUploadDialogOpen(true); }; + // Seed the Data Loading chat through the single redux `pending` slot, + // then navigate to the extract tab. This is the one channel that + // carries text, images, AND file attachments as first-class fields — + // replacing the older `initialChatPrompt/Images` props that silently + // dropped file attachments (they had no dedicated field and only + // survived if their name was baked into the prompt text). + const startDataLoadingChat = (text: string, images: string[] = [], attachments: string[] = []) => { + if (text.trim().length > 0 || images.length > 0 || attachments.length > 0) { + // Fresh query replaces any prior conversation. + if (dataLoadingChatMessages.length > 0) { + dispatch(dfActions.clearChatMessages()); + } + dispatch(dfActions.setDataLoadingChatPending({ text, images, attachments })); + } + openUploadDialog('extract'); + }; + // Honor cross-component requests to hand off to the Data Loading // chat seeded with a prompt (e.g. Data Agent's `delegate` card with // target='data_loading'). Hand-offs targeting other agents (e.g. @@ -326,7 +339,7 @@ export const DataFormulatorFC = ({ }) => { const agentHandoffRequest = useSelector((state: DataFormulatorState) => state.agentHandoffRequest); useEffect(() => { if (agentHandoffRequest && agentHandoffRequest.target === 'data_loading') { - openUploadDialog('extract', agentHandoffRequest.prompt, agentHandoffRequest.images); + startDataLoadingChat(agentHandoffRequest.prompt, agentHandoffRequest.images ?? [], []); dispatch(dfActions.clearAgentHandoffRequest()); } // openUploadDialog is stable enough for this purpose; we only react @@ -730,7 +743,7 @@ export const DataFormulatorFC = ({ }) => { openUploadDialog(`connector:${conn.id}` as UploadTabType); } }} - onStartChat={(prompt, images) => openUploadDialog('extract', prompt, images)} + onStartChat={(prompt, images, attachments) => startDataLoadingChat(prompt, images, attachments)} hasPriorConversation={dataLoadingChatMessages.length > 0} onResumeChat={() => openUploadDialog('extract')} serverConfig={serverConfig} @@ -933,16 +946,9 @@ export const DataFormulatorFC = ({ }) => { open={uploadDialogOpen} onClose={() => { setUploadDialogOpen(false); - // Clear one-shot seed values so the next dialog - // open (e.g. via the upload button) doesn't - // re-fire the agent with a stale prompt/image. - setUploadDialogInitialChatPrompt(undefined); - setUploadDialogInitialChatImages(undefined); refreshPageConnectors(); }} initialTab={uploadDialogInitialTab} - initialChatPrompt={uploadDialogInitialChatPrompt} - initialChatImages={uploadDialogInitialChatImages} onConnectorsChanged={handleConnectorsChanged} /> {/* Loading overlay for session loading */} diff --git a/src/views/DataLoadingChat.tsx b/src/views/DataLoadingChat.tsx index 379e29fb..9fe59000 100644 --- a/src/views/DataLoadingChat.tsx +++ b/src/views/DataLoadingChat.tsx @@ -60,7 +60,11 @@ const getUniqueTableName = (baseName: string, existingNames: Set): strin // Modern monospace font stack for code blocks const CODE_FONT = '"SF Mono", "Cascadia Code", "Fira Code", Menlo, Consolas, "Liberation Mono", monospace'; -const MarkdownContent: React.FC<{ content: string }> = ({ content }) => { +// Memoized so typing in the chat input (which re-renders the parent +// `DataLoadingChat` on every keystroke) doesn't re-parse every assistant +// message through react-markdown. `content` is a stable string per +// committed message, so the default shallow equality is sufficient. +const MarkdownContent = React.memo(({ content }: { content: string }) => { return ( = ({ content }) => { ); -}; +}); // --------------------------------------------------------------------------- // Inline table preview — compact notebook-style @@ -317,10 +321,16 @@ const CodeBlockView: React.FC<{ block: CodeExecution }> = ({ block }) => { // Single chat message bubble // --------------------------------------------------------------------------- -const ChatBubble: React.FC<{ +// Memoized so typing in the chat input doesn't re-render every prior +// bubble (each one renders MarkdownContent + potentially code blocks / +// table previews, which is expensive on long threads). The parent +// stabilises `existingNames` via useMemo so memo equality holds across +// keystrokes. +const ChatBubble = React.memo<{ message: ChatMessage; existingNames: Set; -}> = ({ message, existingNames }) => { + onTableLoaded?: () => void; +}>(({ message, existingNames, onTableLoaded }) => { const theme = useTheme(); const { t } = useTranslation(); const dispatch = useDispatch(); @@ -340,6 +350,9 @@ const ChatBubble: React.FC<{ if (table) { dispatch(loadTable({ table: { ...table, source: { type: 'extract' as const } } })); dispatch(dfActions.confirmTableLoad({ messageId: message.id, tableName: pending.name })); + // Loading data is a deliberate commit — return the + // user to the canvas (the dialog closes via this hook). + onTableLoaded?.(); } } } catch (err) { @@ -468,6 +481,11 @@ const ChatBubble: React.FC<{ })); } dispatch(dfActions.markLoadPlanConfirmed({ messageId: message.id })); + if (selected.length > 0) { + // Return the user to the canvas after a + // deliberate batch load. + onTableLoaded?.(); + } }} /> )} @@ -493,7 +511,7 @@ const ChatBubble: React.FC<{ ); -}; +}); // --------------------------------------------------------------------------- // Tool call label mapping @@ -517,7 +535,10 @@ interface ToolStep { label: string; } -const StreamingIndicator: React.FC<{ content: string; toolSteps: ToolStep[] }> = ({ content, toolSteps }) => { +// Memoized so an unrelated parent re-render (e.g. typing) doesn't +// reflow the shimmer animation. Props are state values that only change +// during an active stream. +const StreamingIndicator = React.memo<{ content: string; toolSteps: ToolStep[] }>(({ content, toolSteps }) => { const theme = useTheme(); return ( @@ -579,55 +600,56 @@ const StreamingIndicator: React.FC<{ content: string; toolSteps: ToolStep[] }> = )} ); -}; +}); // --------------------------------------------------------------------------- // Main chat component // --------------------------------------------------------------------------- -export interface DataLoadingChatProps { - /** - * Optional initial text to pre-fill the chat input when the component - * mounts (or when the value changes). Used by external entry points - * (e.g. landing page quick-chat box) that want to hand off a prompt - * to the agent. - */ - initialPrompt?: string; - /** - * Optional images (data URLs) to seed alongside `initialPrompt` — - * used when an external surface (e.g. landing-page agent box) has - * already collected pasted/attached images and is handing them off. - */ - initialImages?: string[]; - /** - * If true, automatically send the `initialPrompt` once on mount/change. - * Otherwise the prompt is only pre-filled and the user presses Enter. - */ - autoSendInitialPrompt?: boolean; +interface DataLoadingChatProps { + /** Called after a table is successfully loaded into the app. The + * upload dialog wires this to its close handler so loading data + * returns the user to the canvas. */ + onTableLoaded?: () => void; } -export const DataLoadingChat: React.FC = ({ - initialPrompt, - initialImages, - autoSendInitialPrompt, -}) => { +export const DataLoadingChat: React.FC = ({ onTableLoaded }) => { const theme = useTheme(); const { t } = useTranslation(); const dispatch = useDispatch(); + // Keep the latest callback in a ref so the stable `handleTableLoaded` + // identity below doesn't bust `ChatBubble`'s memoization even when the + // parent passes a fresh closure each render. + const onTableLoadedRef = useRef(onTableLoaded); + onTableLoadedRef.current = onTableLoaded; + const handleTableLoaded = useCallback(() => { + onTableLoadedRef.current?.(); + }, []); + const chatMessages = useSelector((state: DataFormulatorState) => state.dataLoadingChatMessages); const chatInProgress = useSelector((state: DataFormulatorState) => state.dataLoadingChatInProgress); - // External reset signal — bumped by `clearChatMessages` (manual reset - // button, new menu-level query, full session reset). When it changes - // we abort any in-flight stream, drop partial UI state, and re-seed - // from props if the parent provided a new prompt/images. Without - // this, an in-flight stream's eventual dispatches would leak into - // the freshly-cleared thread. + // External reset signal — bumped by `clearChatMessages` (manual + // reset button, fresh menu submission, full session reset). Used + // here only to abort an in-flight stream and invalidate any + // late-arriving dispatches from that stream via `sessionRef`. const chatResetCounter = useSelector((state: DataFormulatorState) => state.dataLoadingChatResetCounter ?? 0); + // Pending submission queued by an external surface (menu agent + // box, suggestion auto-run, external dialog caller). When set, we + // consume it in a useEffect: clear the slot first, then send the + // carried payload as a fresh user message via `sendMessage`. + // Single redux signal = no prop race. + const pendingSubmission = useSelector((state: DataFormulatorState) => state.dataLoadingChatPending); const existingTables = useSelector((state: DataFormulatorState) => state.tables); const activeModel = useSelector(dfSelectors.getActiveModel); const frontendRowLimit = useSelector((state: DataFormulatorState) => state.config?.frontendRowLimit ?? 2_000_000); - const existingNames = new Set(existingTables.map(tbl => tbl.id)); + // Stable reference across renders that don't actually change the + // table list — without this, every keystroke in the chat input + // would rebuild the Set and bust `ChatBubble`'s memo equality. + const existingNames = React.useMemo( + () => new Set(existingTables.map(tbl => tbl.id)), + [existingTables], + ); const [prompt, setPrompt] = useState(''); const [userImages, setUserImages] = useState([]); @@ -654,95 +676,44 @@ export const DataLoadingChat: React.FC = ({ // Auto-focus input useEffect(() => { inputRef.current?.focus(); }, []); - // ---- External initial prompt handling ------------------------------- - // Pre-fill the input (and optionally auto-send) when `initialPrompt` - // is provided. Used by external surfaces (e.g. landing-page quick chat - // box) to hand off text to the agent. Auto-send only fires for a - // fresh conversation — we never auto-resend on remount mid-chat. - const hasExistingMessages = chatMessages.length > 0; - const [pendingAutoSend, setPendingAutoSend] = useState(false); + // ---- Reset handling ------------------------------------------------- + // On external reset (counter bump from `clearChatMessages`): abort + // any in-flight stream, invalidate the current session token, and + // clear local input/streaming UI state. We deliberately do NOT + // re-seed anything here — a reset means "clean slate"; any new + // submission arrives separately via `pendingSubmission`. useEffect(() => { - // Detect external reset: abort, invalidate in-flight session, - // and clear all local UI state before re-seeding. Including - // `chatResetCounter` in the dep list also guarantees that an - // identical-prompt re-submission (same `initialPrompt` string) - // still triggers a fresh auto-send — otherwise the deps would - // be unchanged and the effect would skip. - const isReset = chatResetCounter !== lastResetRef.current; - if (isReset) { - lastResetRef.current = chatResetCounter; - sessionRef.current += 1; - abortControllerRef.current?.abort(); - abortControllerRef.current = null; - setStreamingContent(''); - setStreamingToolSteps([]); - setPrompt(''); - setUserImages([]); - setUserAttachments([]); - setPendingAutoSend(false); - } - - // Extract `[Uploaded: name]` mentions from the seeded prompt and - // surface them as chips. The mention template is locale-aware, - // so we build the regex from the current i18n value rather than - // hard-coding the English form. - const mentionTemplate = t('dataLoading.uploaded', { name: '__DF_NAME__' }); - const mentionPattern = mentionTemplate - .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - .replace('__DF_NAME__', '(.+?)'); - const mentionRegex = new RegExp(mentionPattern, 'g'); - let seededPrompt = initialPrompt || ''; - const extractedNames: string[] = []; - if (seededPrompt) { - let match: RegExpExecArray | null; - while ((match = mentionRegex.exec(seededPrompt)) !== null) { - extractedNames.push(match[1]); - } - if (extractedNames.length > 0) { - seededPrompt = seededPrompt - .replace(new RegExp(`\\n?${mentionPattern}`, 'g'), '') - .trim(); - } - } - - const hasText = seededPrompt.trim().length > 0; - const hasImages = !!initialImages && initialImages.length > 0; - const hasAttachments = extractedNames.length > 0; - // Skip re-seeding the input on a user-initiated reset — the - // reset is meant to restore a clean slate, not re-populate the - // input with the prompt the user just cleared. - if (!isReset) { - if (hasText) setPrompt(seededPrompt); - if (hasAttachments) setUserAttachments(extractedNames); - if (hasImages) { - // Always replace, never append. The prop is a "seed" — each - // change represents a fresh handoff from the parent, not an - // additive update. Appending caused the same image to stack - // up every time the parent re-rendered with a new array ref. - setUserImages([...initialImages!]); - } - } - // Auto-send only on a genuinely fresh open (no prior messages, - // and not a user-initiated reset). Resetting means the user wants - // a clean slate — re-running the seeded prompt against their will - // would defeat the purpose of the reset button. - if (autoSendInitialPrompt && !isReset && (hasText || hasImages || hasAttachments) && !hasExistingMessages) { - setPendingAutoSend(true); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [initialPrompt, initialImages, autoSendInitialPrompt, chatResetCounter]); + if (chatResetCounter === lastResetRef.current) return; + lastResetRef.current = chatResetCounter; + sessionRef.current += 1; + abortControllerRef.current?.abort(); + abortControllerRef.current = null; + setStreamingContent(''); + setStreamingToolSteps([]); + setPrompt(''); + setUserImages([]); + setUserAttachments([]); + }, [chatResetCounter]); const stopGeneration = () => { abortControllerRef.current?.abort(); }; // ---- Send message ---- - const sendMessage = useCallback(() => { - const text = prompt.trim(); - if (!text && userImages.length === 0 && userAttachments.length === 0) return; + // Accepts an optional explicit payload so callers (suggestion + // auto-run, pending-submission consume) can submit the exact + // values they just chose without waiting for React state to flush. + // Reading via the `prompt`/`userImages`/`userAttachments` closures + // alone would be racy with batching and could submit the previous + // round's values on a fresh handoff. + const sendMessage = useCallback((explicit?: { text: string; images: string[]; attachments: string[] }) => { + const text = (explicit?.text ?? prompt).trim(); + const imgs = explicit?.images ?? userImages; + const atts = explicit?.attachments ?? userAttachments; + if (!text && imgs.length === 0 && atts.length === 0) return; if (chatInProgress) return; - const imageAttachments: ChatAttachment[] = userImages.map((url, i) => ({ + const imageAttachments: ChatAttachment[] = imgs.map((url, i) => ({ type: 'image' as const, name: `image-${i + 1}`, url, })); - const fileAttachments: ChatAttachment[] = userAttachments.map(name => ({ + const fileAttachments: ChatAttachment[] = atts.map(name => ({ type: 'file' as const, name, })); const attachments: ChatAttachment[] = [...imageAttachments, ...fileAttachments]; @@ -751,7 +722,7 @@ export const DataLoadingChat: React.FC = ({ // chips (rendered from `attachments`). The agent payload below // re-injects `[Uploaded: name]` mentions so the backend still // sees the file references inline. - const displayText = text || (userImages.length > 0 ? t('dataLoading.defaultImageMessage') : ''); + const displayText = text || (imgs.length > 0 ? t('dataLoading.defaultImageMessage') : ''); const userMsg: ChatMessage = { id: `msg-${Date.now()}-user`, role: 'user', @@ -967,25 +938,48 @@ export const DataLoadingChat: React.FC = ({ } } })(); - }, [prompt, userImages, chatInProgress, chatMessages, activeModel, existingTables, dispatch, streamingContent, t]); + }, [prompt, userImages, userAttachments, chatInProgress, chatMessages, activeModel, existingTables, dispatch, streamingContent, t]); - // Auto-send the initial prompt once it has been applied to state. + // Consume a queued submission from any external surface (menu + // agent input, suggestion auto-run, or a cross-component handoff + // routed through `startDataLoadingChat`). Single redux signal, + // single consumer — no prop race. + // + // Idempotency note: under React.StrictMode (dev), effects are + // intentionally double-invoked on mount with the *same* closure, + // so the `clearDataLoadingChatPending` dispatch in the first run + // isn't visible to the second run. `lastConsumedRef` tracks the + // exact payload object we've already sent, so the second + // invocation short-circuits before calling `sendMessage` again. + const lastConsumedRef = useRef(null); useEffect(() => { - if (!pendingAutoSend) return; + if (!pendingSubmission) return; + if (pendingSubmission === lastConsumedRef.current) return; if (chatInProgress) return; - if (prompt.trim().length === 0 && userImages.length === 0) return; - setPendingAutoSend(false); - sendMessage(); - }, [pendingAutoSend, prompt, userImages, chatInProgress, sendMessage]); + lastConsumedRef.current = pendingSubmission; + const payload = pendingSubmission; + dispatch(dfActions.clearDataLoadingChatPending()); + sendMessage(payload); + }, [pendingSubmission, chatInProgress, sendMessage, dispatch]); // Reuse the shared sample-task list so this in-session panel stays in // sync with the upload-dialog entry point (`UnifiedDataUploadDialog`). + // Auto-run is wired through the redux pending slot so the click — + // even on a chat with prior history — atomically clears the thread + // and queues the new submission. const focusSuggestions = React.useMemo(() => buildDataLoadingSuggestions({ t, setInput: setPrompt, setImages: setUserImages, setAttachments: setUserAttachments, - }), [t]); + requestAutoSend: (payload) => { + if (chatMessages.length > 0) { + dispatch(dfActions.clearChatMessages()); + } + dispatch(dfActions.setDataLoadingChatPending(payload)); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + }), [t, dispatch]); const isEmpty = chatMessages.length === 0 && !streamingContent; @@ -1047,7 +1041,7 @@ export const DataLoadingChat: React.FC = ({ ) : ( <> {chatMessages.map((msg) => ( - + ))} {streamingContent !== '' && } {chatInProgress && !streamingContent && } @@ -1065,7 +1059,7 @@ export const DataLoadingChat: React.FC = ({ onChange={setPrompt} images={userImages} onImagesChange={setUserImages} - onSend={sendMessage} + onSend={() => sendMessage()} onStop={stopGeneration} inProgress={chatInProgress} placeholder={t('dataLoading.placeholder')} @@ -1076,8 +1070,13 @@ export const DataLoadingChat: React.FC = ({ formData.append('file', file); apiRequest(getUrls().SCRATCH_UPLOAD_URL, { method: 'POST', body: formData, - }).then(() => { - setUserAttachments(prev => [...prev, file.name]); + }).then(({ data }) => { + // The backend hash-suffixes the filename + // (e.g. `name_a1b2c3d4.xlsx`). Store the + // server-assigned name so the `[Uploaded:]` + // mention points to the real scratch file. + const scratchName = (data?.path || `scratch/${file.name}`).replace(/^scratch\//, ''); + setUserAttachments(prev => [...prev, scratchName]); }).catch(err => console.error('Upload failed:', err)); }} attachments={userAttachments} diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index ae0cc9f1..c69a71a5 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -1751,6 +1751,9 @@ let SingleThreadGroupView: FC<{ const TIMELINE_GAP = '4px'; // gap between timeline and card content const DOT_SIZE = 6; const CARD_PY = '6px'; // vertical padding for each timeline row + // Mirror the left timeline gutter on the right so cards sit visually + // centred in their column instead of hugging the right edge. + const CARD_CONTENT_PR = `${TIMELINE_WIDTH}px`; // CSS `border-style: dashed` stretches dashes to fit each element's // height, so stacked segments end up with mismatched dash lengths. A @@ -1907,7 +1910,7 @@ let SingleThreadGroupView: FC<{ {isLast && hasContinuationBelow && } {isLast && !hasContinuationBelow && } - + {item.element} @@ -1983,7 +1986,7 @@ let SingleThreadGroupView: FC<{ {isLast && hasContinuationBelow && } {isLast && !hasContinuationBelow && } - + {item.element} @@ -2006,7 +2009,7 @@ let SingleThreadGroupView: FC<{ {isLast && hasContinuationBelow && } {isLast && !hasContinuationBelow && } - + {item.element} @@ -2054,7 +2057,7 @@ let SingleThreadGroupView: FC<{ )} {isLast && !hasContinuationBelow && } - {item.element} @@ -3119,7 +3122,10 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { // benefit, since the segments would just stack in the same single column. const CARD_GAP = 12; // padding + spacing between cards in a column const PANEL_PADDING = 16; - const CARD_WIDTH = 220; + // 220 visual card width + 14px right gutter (CARD_CONTENT_PR) so cards + // keep their original size while gaining a right margin that balances + // the left timeline gutter. + const CARD_WIDTH = 234; const COLUMN_WIDTH = CARD_WIDTH + CARD_GAP; // n columns need: n*CARD_WIDTH + (n-1)*CARD_GAP + PANEL_PADDING // Solving for n: n <= (containerWidth - PANEL_PADDING + CARD_GAP) / COLUMN_WIDTH diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index bd7167f8..73d325e1 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -448,12 +448,14 @@ export interface DataLoadMenuProps { onSelectConnector?: (connector: ConnectorInstance) => void; /** * Called when the user submits a prompt from the top-level Data Loading - * Agent chat box. Implementations should open the agent chat surface - * with the prompt (and optional pasted/attached images) pre-filled — - * typically auto-sent. If not provided, the chat box falls back to - * `onSelectTab('extract')`. + * Agent chat box. Implementations should hand the payload off to the + * agent chat surface, which will auto-send it as a fresh user + * message. Attachments are file names (already uploaded to the + * session scratch space) — the chat surface re-injects them as + * `[Uploaded: name]` mentions when building the backend payload. + * If not provided, the chat box falls back to `onSelectTab('extract')`. */ - onStartChat?: (prompt: string, images?: string[]) => void; + onStartChat?: (prompt: string, images: string[], attachments: string[]) => void; /** * True when a prior data-loading agent conversation exists in * state. When set together with `onResumeChat`, the menu renders @@ -605,22 +607,17 @@ export const DataLoadMenu: React.FC = ({ const submitAgentChat = () => { const text = agentInput.trim(); if (text.length === 0 && agentImages.length === 0 && agentAttachments.length === 0) { - // Empty submission — just open the chat surface. - if (onStartChat) onStartChat('', []); + // Empty submission — just surface the chat. + if (onStartChat) onStartChat('', [], []); else onSelectTab('extract'); return; } - // Augment the outgoing prompt with `[Uploaded: name]` lines so the - // agent sees attachments as text references, without polluting - // the editable input the user sees. - const mentions = agentAttachments - .map(name => t('dataLoading.uploaded', { name })) - .join('\n'); - const finalText = mentions - ? (text ? `${text}\n${mentions}` : mentions) - : text; + // Pass payload pieces unchanged — the chat surface builds the + // backend mentions itself. We deliberately do NOT pre-inject + // `[Uploaded: name]` into `text` here, so the visible message + // bubble stays clean and the file chips render uniformly. if (onStartChat) { - onStartChat(finalText, agentImages); + onStartChat(text, agentImages, agentAttachments); } else { onSelectTab('extract'); } @@ -631,14 +628,26 @@ export const DataLoadMenu: React.FC = ({ // Suggestions surfaced as a focus-time dropdown — sourced from a shared // factory so the in-session `DataLoadingChat` panel renders the exact - // same list. See `dataLoadingSuggestions.ts`. + // same list. See `dataLoadingSuggestions.ts`. Auto-run is routed + // through `onStartChat` so the parent dialog can dispatch its + // `clearChatMessages` + `setDataLoadingChatPending` sequence + // atomically — same path as a manual submit. const agentChatSuggestions = useMemo(() => buildDataLoadingSuggestions({ t, setInput: setAgentInput, setImages: setAgentImages, setAttachments: setAgentAttachments, ensureActiveWorkspace, - }), [t]); + requestAutoSend: onStartChat + ? (payload) => { + onStartChat(payload.text, payload.images, payload.attachments); + setAgentInput(''); + setAgentImages([]); + setAgentAttachments([]); + } + : undefined, + // eslint-disable-next-line react-hooks/exhaustive-deps + }), [t, onStartChat]); const agentChatBox = ( = ({ formData.append('file', file); apiRequest(getUrls().SCRATCH_UPLOAD_URL, { method: 'POST', body: formData, - }).then(() => { - setAgentAttachments(prev => [...prev, file.name]); + }).then(({ data }) => { + // The backend hash-suffixes the filename; store the + // server-assigned name so the `[Uploaded:]` mention + // resolves to the real scratch file. + const scratchName = (data?.path || `scratch/${file.name}`).replace(/^scratch\//, ''); + setAgentAttachments(prev => [...prev, scratchName]); }).catch(err => console.error('Upload failed:', err)); }} attachments={agentAttachments} @@ -1112,14 +1125,6 @@ export interface UnifiedDataUploadDialogProps { open: boolean; onClose: () => void; initialTab?: UploadTabType; - /** - * Optional initial prompt to hand off to the Data Loading Agent. When - * non-empty and `initialTab === 'extract'`, the prompt is pre-filled - * and auto-sent in the chat panel. - */ - initialChatPrompt?: string; - /** Optional images (data URLs) to seed the chat alongside `initialChatPrompt`. */ - initialChatImages?: string[]; onConnectorsChanged?: () => void; } @@ -1127,8 +1132,6 @@ export const UnifiedDataUploadDialog: React.FC = ( open, onClose, initialTab = 'menu', - initialChatPrompt, - initialChatImages, onConnectorsChanged, }) => { const theme = useTheme(); @@ -1143,21 +1146,6 @@ export const UnifiedDataUploadDialog: React.FC = ( const existingNames = new Set(existingTables.map(t => t.id)); const [activeTab, setActiveTab] = useState(initialTab === 'menu' ? 'menu' : initialTab); - // Prompt to seed the agent chat with. Sourced from the `initialChatPrompt` - // prop when the dialog opens directly on 'extract', or set internally - // when the user submits the in-menu agent chat box. - const [seededChatPrompt, setSeededChatPrompt] = useState( - initialTab === 'extract' ? initialChatPrompt : undefined, - ); - const [seededChatImages, setSeededChatImages] = useState( - initialTab === 'extract' ? initialChatImages : undefined, - ); - const [autoSendSeededPrompt, setAutoSendSeededPrompt] = useState( - initialTab === 'extract' && ( - (!!initialChatPrompt && initialChatPrompt.trim().length > 0) - || (!!initialChatImages && initialChatImages.length > 0) - ), - ); const fileInputRef = useRef(null); const urlInputRef = useRef(null); @@ -1175,27 +1163,8 @@ export const UnifiedDataUploadDialog: React.FC = ( if (open) { setConnectorInstances([]); refreshConnectors(); - // Re-seed chat prompt/images from props each time the dialog opens. - if (initialTab === 'extract') { - setSeededChatPrompt(initialChatPrompt); - setSeededChatImages(initialChatImages); - const hasText = !!initialChatPrompt && initialChatPrompt.trim().length > 0; - const hasImages = !!initialChatImages && initialChatImages.length > 0; - setAutoSendSeededPrompt(hasText || hasImages); - // Opening the dialog with a fresh prompt/images means the - // user wants a new data-loading conversation; clear any - // stale messages from a previous session so the new query - // isn't appended to an unrelated thread. - if ((hasText || hasImages) && dataLoadingChatMessages.length > 0) { - dispatch(dfActions.clearChatMessages()); - } - } else { - setSeededChatPrompt(undefined); - setSeededChatImages(undefined); - setAutoSendSeededPrompt(false); - } } - }, [open, refreshConnectors, identityKey, initialTab, initialChatPrompt, initialChatImages]); + }, [open, refreshConnectors, identityKey]); // Storage is determined by backend config — no user toggle const isEphemeral = serverConfig.WORKSPACE_BACKEND === 'ephemeral'; @@ -1848,29 +1817,32 @@ export const UnifiedDataUploadDialog: React.FC = ( setActiveTab(`connector:${conn.id}` as UploadTabType); } }} - onStartChat={(prompt, images) => { + onStartChat={(prompt, images, attachments) => { const hasText = prompt.trim().length > 0; - const hasImages = !!images && images.length > 0; - // If a prior conversation exists, treat a - // new query from the menu as a fresh data - // reload and reset the chat. Without this - // the new prompt would be appended onto an - // unrelated thread, confusing the agent. - if ((hasText || hasImages) && dataLoadingChatMessages.length > 0) { - dispatch(dfActions.clearChatMessages()); + const hasImages = images.length > 0; + const hasAttachments = attachments.length > 0; + // Always surface the chat. If the user + // is starting a fresh query, clear any + // prior conversation and enqueue the new + // submission as a redux `pending` slot + // — `DataLoadingChat` consumes it on + // render and auto-sends. Doing both + // dispatches in the same tick keeps the + // handoff atomic; there's no prop race. + if (hasText || hasImages || hasAttachments) { + if (dataLoadingChatMessages.length > 0) { + dispatch(dfActions.clearChatMessages()); + } + dispatch(dfActions.setDataLoadingChatPending({ + text: prompt, images, attachments, + })); } - setSeededChatPrompt(prompt); - setSeededChatImages(images); - setAutoSendSeededPrompt(hasText || hasImages); setActiveTab('extract'); }} hasPriorConversation={dataLoadingChatMessages.length > 0} onResumeChat={() => { // Reopen the existing thread without // clearing messages or auto-sending. - setSeededChatPrompt(undefined); - setSeededChatImages(undefined); - setAutoSendSeededPrompt(false); setActiveTab('extract'); }} serverConfig={serverConfig} @@ -2403,11 +2375,7 @@ export const UnifiedDataUploadDialog: React.FC = ( {/* Extract Data Tab */} - + {/* Local Folder Tab */} diff --git a/src/views/dataLoadingSuggestions.ts b/src/views/dataLoadingSuggestions.ts index 8d91b92b..f37e04f0 100644 --- a/src/views/dataLoadingSuggestions.ts +++ b/src/views/dataLoadingSuggestions.ts @@ -22,6 +22,12 @@ export interface DataLoadingSuggestion { onClick: () => void; } +export interface SuggestionPayload { + text: string; + images: string[]; + attachments: string[]; +} + export interface BuildSuggestionsArgs { t: TFunction; setInput: (value: string) => void; @@ -29,12 +35,22 @@ export interface BuildSuggestionsArgs { setAttachments: (names: string[]) => void; /** Optional hook that workspaces use to make sure a session exists before uploading. */ ensureActiveWorkspace?: () => void; + /** + * Optional auto-run hook. When provided, suggestions submit the + * complete payload immediately (after any required async upload / + * data-URL prep) instead of just pre-filling the input. Callers + * typically wire this to a redux pending-submission dispatch so the + * payload survives the parent→child handoff without prop races. + * When absent, the suggestion behaves like a paste: it only fills + * the input fields via the `set*` callbacks. + */ + requestAutoSend?: (payload: SuggestionPayload) => void; } const EXCEL_SAMPLE_NAME = 'climate-gas-indicator.xlsx'; export function buildDataLoadingSuggestions( - { t, setInput, setImages, setAttachments, ensureActiveWorkspace }: BuildSuggestionsArgs, + { t, setInput, setImages, setAttachments, ensureActiveWorkspace, requestAutoSend }: BuildSuggestionsArgs, ): DataLoadingSuggestion[] { const kindAsk = t('upload.agentChatSuggestion.kind.ask', { defaultValue: 'ask' }); const kindFind = t('upload.agentChatSuggestion.kind.find', { defaultValue: 'find' }); @@ -61,37 +77,38 @@ export function buildDataLoadingSuggestions( const iconSx = { fontSize: 14 }; + // Common: fill the input fields AND (if auto-run is enabled) submit + // the payload. Centralising the dual behaviour keeps every + // suggestion below short and consistent. + const fillAndMaybeSend = (payload: SuggestionPayload) => { + setImages(payload.images); + setAttachments(payload.attachments); + setInput(payload.text); + requestAutoSend?.(payload); + }; + return [ { kind: kindAsk, label: askLabel, icon: React.createElement(QuestionAnswerOutlinedIcon, { sx: iconSx }), - onClick: () => { - setImages([]); - setAttachments([]); - setInput(askLabel); - }, + onClick: () => fillAndMaybeSend({ text: askLabel, images: [], attachments: [] }), }, { kind: kindFind, label: findLabel, icon: React.createElement(SearchIcon, { sx: iconSx }), - onClick: () => { - setImages([]); - setAttachments([]); - setInput(findLabel); - }, + onClick: () => fillAndMaybeSend({ text: findLabel, images: [], attachments: [] }), }, { kind: kindExtract, label: extractExcelLabel, icon: React.createElement(TableChartOutlinedIcon, { sx: iconSx }), onClick: () => { - // Surface the attachment chip synchronously so it is - // always present when the user hits send, even if the - // upload below is still mid-flight. The chip is what - // gets serialised into the outgoing `[Uploaded: name]` - // mention and ultimately the chat bubble. + // Surface the attachment chip / input synchronously so + // it is visible during the async upload. The auto-send + // (if enabled) waits until the upload completes so the + // backend can actually find the scratch file. setImages([]); setAttachments([EXCEL_SAMPLE_NAME]); setInput(extractExcelLabel); @@ -108,6 +125,18 @@ export function buildDataLoadingSuggestions( method: 'POST', body: formData, }); }) + .then(({ data }) => { + // The backend hash-suffixes the filename, so use the + // server-assigned name for the chip and the mention + // — otherwise the agent looks for a file that the + // upload renamed and reports it missing. + const scratchName = (data?.path || `scratch/${EXCEL_SAMPLE_NAME}`).replace(/^scratch\//, ''); + setAttachments([scratchName]); + requestAutoSend?.({ + text: extractExcelLabel, images: [], + attachments: [scratchName], + }); + }) .catch(err => console.error('Sample Excel upload failed:', err)); }, }, @@ -116,16 +145,21 @@ export function buildDataLoadingSuggestions( label: extractImageLabel, icon: React.createElement(ImageOutlinedIcon, { sx: iconSx }), onClick: () => { + // Image needs to be read into a data URL before we can + // surface it as a chip or send it. Defer auto-send until + // the FileReader resolves. fetch(exampleImageTable) .then(res => res.blob()) .then(blob => { const reader = new FileReader(); reader.onload = () => { - if (reader.result) { - setImages([reader.result as string]); - setAttachments([]); - setInput(extractImageLabel); - } + if (!reader.result) return; + const dataUrl = reader.result as string; + fillAndMaybeSend({ + text: extractImageLabel, + images: [dataUrl], + attachments: [], + }); }; reader.readAsDataURL(blob); }); @@ -135,11 +169,7 @@ export function buildDataLoadingSuggestions( kind: kindExtract, label: extractTextLabel, icon: React.createElement(DescriptionOutlinedIcon, { sx: iconSx }), - onClick: () => { - setImages([]); - setAttachments([]); - setInput(extractTextPrompt); - }, + onClick: () => fillAndMaybeSend({ text: extractTextPrompt, images: [], attachments: [] }), }, ]; } diff --git a/tests/backend/data/test_workspace_manager.py b/tests/backend/data/test_workspace_manager.py index e2a00eb7..d82766c6 100644 --- a/tests/backend/data/test_workspace_manager.py +++ b/tests/backend/data/test_workspace_manager.py @@ -376,6 +376,13 @@ def test_legacy_workspace_with_only_yaml_appears_in_list(self, manager): yaml.safe_dump({"version": "1.1", "tables": {}}), encoding="utf-8", ) + # Pretend the legacy workspace had session state with tables. + (ws_dir / "session_state.json").write_text( + json.dumps({"tables": [{"id": "t1"}]}), + encoding="utf-8", + ) + # Trigger meta repair with a non-empty table count. + manager.save_session_state("legacy_ws", {"tables": [{"id": "t1"}]}) ws_list = manager.list_workspaces() ids = [w["id"] for w in ws_list] @@ -385,16 +392,23 @@ def test_legacy_workspace_with_only_yaml_appears_in_list(self, manager): assert (ws_dir / WORKSPACE_META_FILENAME).exists() def test_legacy_workspace_with_only_session_state_appears_in_list(self, manager): - """A directory with only session_state.json should be auto-repaired.""" + """A directory with session_state.json (containing tables) is + auto-repaired and visible in list_workspaces. The displayName + is inferred from session_state.""" ws_dir = manager.root / "state_only" ws_dir.mkdir(parents=True) (ws_dir / "session_state.json").write_text( json.dumps({ - "tables": [], + "tables": [{"id": "t1", "name": "T1"}], "activeWorkspace": {"displayName": "My Old Session"}, }), encoding="utf-8", ) + # Re-save so meta is written with tableCount > 0. + manager.save_session_state("state_only", { + "tables": [{"id": "t1", "name": "T1"}], + "activeWorkspace": {"displayName": "My Old Session"}, + }) ws_list = manager.list_workspaces() ids = [w["id"] for w in ws_list] @@ -405,7 +419,9 @@ def test_legacy_workspace_with_only_session_state_appears_in_list(self, manager) assert entry["display_name"] == "My Old Session" def test_legacy_workspace_with_empty_dir_appears_in_list(self, manager): - """Even a bare directory (no metadata files at all) should be listed.""" + """A bare directory with no metadata at all is auto-repaired by + _ensure_meta (meta.json gets created with fallback displayName) + and appears in list_workspaces.""" ws_dir = manager.root / "bare" ws_dir.mkdir(parents=True) @@ -413,7 +429,7 @@ def test_legacy_workspace_with_empty_dir_appears_in_list(self, manager): ids = [w["id"] for w in ws_list] assert "bare" in ids - # workspace_meta.json auto-created with fallback displayName = dir name + # Auto-repair created the meta with a fallback displayName. meta = json.loads((ws_dir / WORKSPACE_META_FILENAME).read_text(encoding="utf-8")) assert meta["displayName"] == "bare" @@ -452,3 +468,43 @@ def test_move_legacy_workspace_auto_repairs_meta(self, tmp_path): # Destination should have workspace_meta.json dst_ws = dst.get_workspace_path("old_ws") assert (dst_ws / WORKSPACE_META_FILENAME).exists() + + +class TestEmptyWorkspaceVisibility: + """list_workspaces() lists every workspace directory, including + empty "Untitled Session" entries from abandoned data-loading + chats. Users manage (rename/delete) these themselves via the + sidebar — they are not hidden.""" + + def test_empty_workspace_is_visible(self, manager): + manager.create_workspace("ghost") + # No save_session_state — meta has no tableCount/chartCount. + + ws_list = manager.list_workspaces() + + assert any(w["id"] == "ghost" for w in ws_list) + assert manager.workspace_exists("ghost") + + def test_workspace_with_tables_is_visible(self, manager): + manager.create_workspace("real") + manager.save_session_state("real", { + "tables": [{"id": "t1", "name": "T1"}], + "activeWorkspace": {"id": "real", "displayName": "Real"}, + }) + + ws_list = manager.list_workspaces() + + assert any(w["id"] == "real" for w in ws_list) + + def test_zero_count_workspace_is_visible(self, manager): + """A workspace whose tables were all deleted (zero tables) still + appears in the list — the user decides whether to remove it.""" + manager.create_workspace("emptied") + manager.save_session_state("emptied", { + "tables": [], + "activeWorkspace": {"id": "emptied", "displayName": "Emptied"}, + }) + + ws_list = manager.list_workspaces() + + assert any(w["id"] == "emptied" for w in ws_list) From 18cf3603dfd7da2197f4ba131e80ecb2fab285f3 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Thu, 28 May 2026 22:21:14 -0700 Subject: [PATCH 02/36] small fixes --- src/views/DataFormulator.tsx | 4 +-- src/views/DataSourceSidebar.tsx | 64 ++++++++------------------------- 2 files changed, 17 insertions(+), 51 deletions(-) diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index 00e8086e..b340a477 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -571,7 +571,7 @@ export const DataFormulatorFC = ({ }) => { const fixedSplitPane = ( openUploadDialog((tab ?? 'add-connection') as UploadTabType)} + onOpenUploadDialog={(tab) => openUploadDialog((tab ?? 'menu') as UploadTabType)} connectorRefreshKey={connectorRefreshKey} /> { {tables.length > 0 ? fixedSplitPane : ( openUploadDialog((tab ?? 'add-connection') as UploadTabType)} + onOpenUploadDialog={(tab) => openUploadDialog((tab ?? 'menu') as UploadTabType)} connectorRefreshKey={connectorRefreshKey} /> {dataUploadRequestBox} diff --git a/src/views/DataSourceSidebar.tsx b/src/views/DataSourceSidebar.tsx index a8c4715d..7381a423 100644 --- a/src/views/DataSourceSidebar.tsx +++ b/src/views/DataSourceSidebar.tsx @@ -42,7 +42,6 @@ import { VirtualizedCatalogTree } from '../components/VirtualizedCatalogTree'; import StorageIcon from '@mui/icons-material/Storage'; import AddIcon from '@mui/icons-material/Add'; -import FileUploadOutlinedIcon from '@mui/icons-material/FileUploadOutlined'; import FolderOpenIcon from '@mui/icons-material/FolderOpen'; import FolderOutlinedIcon from '@mui/icons-material/FolderOutlined'; import UploadFileIcon from '@mui/icons-material/UploadFile'; @@ -51,9 +50,6 @@ import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import ChevronRightIcon from '@mui/icons-material/ChevronRight'; import RefreshIcon from '@mui/icons-material/Refresh'; -import ContentPasteOutlinedIcon from '@mui/icons-material/ContentPasteOutlined'; -import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; -import LinkOutlinedIcon from '@mui/icons-material/LinkOutlined'; import LinkOffOutlinedIcon from '@mui/icons-material/LinkOffOutlined'; import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import EditOutlinedIcon from '@mui/icons-material/EditOutlined'; @@ -159,7 +155,7 @@ export const DataSourceSidebar: React.FC<{ // built-in sample_datasets connector is shown there, giving users // something useful to explore immediately. The upgrade message only // appears when they try to add a new connector or link a folder. - const [initialTab, setInitialTab] = useState<'upload' | 'sources' | 'sessions' | 'knowledge'>('sources'); + const [initialTab, setInitialTab] = useState<'sources' | 'sessions' | 'knowledge'>('sources'); // External callers (e.g. SaveExperienceButton on success) can ask the // sidebar to open and switch to a specific tab. @@ -277,6 +273,18 @@ export const DataSourceSidebar: React.FC<{ pt: 1, gap: 0.5, }}> + {/* Primary action — adding data is the main task. Styled like + the view-switcher icons but kept in primary color as a + subtle cue; opens the upload dialog (landing menu). */} + + onOpenUploadDialog?.()} sx={{ + color: 'primary.main', + borderRadius: 1, + '&:hover': { bgcolor: 'action.hover' }, + }}> + + + { setInitialTab('sessions'); if (!isOpen) toggle(); else if (initialTab !== 'sessions') setInitialTab('sessions'); else toggle(); }} sx={{ color: isOpen && initialTab === 'sessions' ? 'primary.main' : 'text.secondary', @@ -295,15 +303,6 @@ export const DataSourceSidebar: React.FC<{ - - { setInitialTab('upload'); if (!isOpen) toggle(); else if (initialTab !== 'upload') setInitialTab('upload'); else toggle(); }} sx={{ - color: isOpen && initialTab === 'upload' ? 'primary.main' : 'text.secondary', - bgcolor: isOpen && initialTab === 'upload' ? 'action.selected' : 'transparent', - borderRadius: 1, - }}> - - - { setInitialTab('knowledge'); if (!isOpen) toggle(); else if (initialTab !== 'knowledge') setInitialTab('knowledge'); else toggle(); }} sx={{ color: isOpen && initialTab === 'knowledge' ? 'primary.main' : 'text.secondary', @@ -347,7 +346,7 @@ const DataSourceSidebarPanel: React.FC<{ panelWidth: number; onOpenUploadDialog?: (tab?: string) => void; onCollapse: () => void; - initialTab?: 'upload' | 'sources' | 'sessions' | 'knowledge'; + initialTab?: 'sources' | 'sessions' | 'knowledge'; connectorRefreshKey?: number; disableConnectors?: boolean; }> = ({ panelWidth, onOpenUploadDialog, onCollapse, initialTab = 'sources', connectorRefreshKey = 0, disableConnectors = false }) => { @@ -419,7 +418,7 @@ const DataSourceSidebarPanel: React.FC<{ const [searchingCatalog, setSearchingCatalog] = useState>({}); // Sidebar tab: 'sources' or 'sessions' or 'knowledge' - const [activeTab, setActiveTab] = useState<'upload' | 'sources' | 'sessions' | 'knowledge'>(initialTab); + const [activeTab, setActiveTab] = useState<'sources' | 'sessions' | 'knowledge'>(initialTab); // Sync tab when rail icon switches it useEffect(() => { @@ -1292,39 +1291,6 @@ const DataSourceSidebarPanel: React.FC<{ overflow: 'hidden', }}> - {/* ── Upload Data tab ── */} - {activeTab === 'upload' && ( - - - - {t('sidebar.uploadData', { defaultValue: 'Upload Data' })} - - - - - - - - - {[ - { icon: , label: t('upload.uploadFile', { defaultValue: 'Upload file' }), tab: 'upload' }, - { icon: , label: t('upload.pasteData', { defaultValue: 'Paste data' }), tab: 'paste' }, - { icon: , label: t('upload.extractData', { defaultValue: 'Data Assistant' }), tab: 'extract' }, - { icon: , label: t('upload.loadFromUrl', { defaultValue: 'Load from URL' }), tab: 'url' }, - ].map((item, i) => ( - onOpenUploadDialog?.(item.tab)} - sx={{ display: 'flex', alignItems: 'center', gap: 0.75, px: 1.5, py: 0.75, cursor: 'pointer', color: 'text.primary', '&:hover': { bgcolor: 'action.hover' }, userSelect: 'none' }} - > - {item.icon} - {item.label} - - ))} - - - )} - {/* ── Data Connectors tab ── Sample datasets remain available even when external connectors are disabled; the Add Connector / Link Folder From 4cb0a2f4f5e32bd5132a84b77cdcb3cc12f50f56 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Thu, 28 May 2026 22:50:07 -0700 Subject: [PATCH 03/36] cleanup --- src/views/ChartRecBox.tsx | 14 +++++++------- src/views/SimpleChartRecBox.tsx | 34 +++++---------------------------- 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/src/views/ChartRecBox.tsx b/src/views/ChartRecBox.tsx index 872dc3c1..9eb26085 100644 --- a/src/views/ChartRecBox.tsx +++ b/src/views/ChartRecBox.tsx @@ -292,10 +292,10 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde type={current ? undefined : 'button'} onClick={current ? undefined : () => dispatch(dfActions.setFocused({ type: 'table', tableId: table.id }))} sx={{ - display: 'inline-flex', alignItems: 'center', gap: current ? '6px' : '3px', + display: 'inline-flex', alignItems: 'center', gap: '3px', border: 'none', background: 'transparent', p: 0, fontFamily: theme.typography.fontFamily, - fontSize: current ? 16 : 11, lineHeight: 1.4, + fontSize: 11, lineHeight: 1.4, color: current ? 'primary.main' : 'text.secondary', fontWeight: current ? 600 : 400, cursor: current ? 'default' : 'pointer', @@ -304,7 +304,7 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde '&:hover': current ? undefined : { color: 'primary.main' }, }} > - + {table.displayId} ); @@ -682,10 +682,10 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde ); }; - // Center cluster auto-scales with chart count; neighbour - // clusters are halved and dimmed to read as context. - const centerN = Math.min(chartsForTable(currentTable.id).length, 8); - const centerScale = centerN <= 3 ? 1 : centerN <= 5 ? 0.82 : 0.66; + // All clusters render at the same scale; the current + // cluster is only distinguished by not being dimmed and by + // showing more thumbnails. + const centerScale = 0.5; const sideScale = 0.5; return ( diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index c96dec65..5e7bd63b 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -41,7 +41,7 @@ import StopIcon from '@mui/icons-material/Stop'; import AutoGraphIcon from '@mui/icons-material/AutoGraph'; import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; import { UnifiedDataUploadDialog } from './UnifiedDataUploadDialog'; -import { transition } from '../app/tokens'; +import { borderColor, transition } from '../app/tokens'; import { Theme } from '@mui/material/styles'; import { useTranslation } from 'react-i18next'; import { shouldAutoFocusGeneratedChart } from '../app/agentInteractionPolicy'; @@ -1380,12 +1380,6 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ }, [pendingClarification, dispatch, t]); const isReportMode = selectedAgent === 'report'; - const gradientBorder = isReportMode - ? `linear-gradient(135deg, ${alpha(theme.palette.warning.main, 0.6)}, ${alpha(theme.palette.warning.dark, 0.5)})` - : `linear-gradient(135deg, ${alpha(theme.palette.primary.main, 0.6)}, ${alpha(theme.palette.secondary.main, 0.55)})`; - const workingBorder = isReportMode - ? `linear-gradient(135deg, ${alpha(theme.palette.warning.main, 0.3)}, ${alpha(theme.palette.warning.dark, 0.25)})` - : `linear-gradient(135deg, ${alpha(theme.palette.primary.main, 0.3)}, ${alpha(theme.palette.secondary.main, 0.25)})`; // Landing / "no thread yet" highlight: when the user has loaded data // but hasn't started an exploration on the focused table (no real @@ -1419,12 +1413,9 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ mx: 1, mb: 1, mt: 0.5, px: 1.25, pt: 1, pb: 0.5, borderRadius: '12px', - // The 2-tone border is drawn by the `::before` gradient - // overlay below (works through border-radius + masks). We - // intentionally leave the Card's own border off so the two - // don't fight; focus state uses a shadow halo instead of a - // border-color shift. - border: 'none', + // Standard single-tone input style (matches AgentChatInput): a + // solid divider border that turns the accent color on focus. + border: `1px solid ${borderColor.divider}`, outline: 'none', position: 'relative', overflow: isChatFormulating ? 'hidden' : 'visible', @@ -1454,24 +1445,9 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ } : {}), '&:focus-within': { animation: 'none', + borderColor: isReportMode ? theme.palette.warning.main : theme.palette.primary.main, boxShadow: `0 0 0 2px ${alpha(isReportMode ? theme.palette.warning.main : theme.palette.primary.main, 0.15)}, 0 2px 10px rgba(32, 33, 36, 0.14)`, }, - // Gradient border via pseudo-element (works with border-radius) - '&::before': { - content: '""', - position: 'absolute', - inset: 0, - borderRadius: 'inherit', - padding: '1.5px', - background: isChatFormulating - ? workingBorder - : gradientBorder, - WebkitMask: 'linear-gradient(#fff 0 0) content-box, linear-gradient(#fff 0 0)', - WebkitMaskComposite: 'xor', - maskComposite: 'exclude', - pointerEvents: 'none', - zIndex: 3, - }, }} > {clarificationQuestions?.kind === 'clarification' && clarificationQuestions.questions && pendingClarification && !isChatFormulating && ( From c616338d67c4c12cb7290139ce046213abc44098 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 29 May 2026 00:06:50 -0700 Subject: [PATCH 04/36] some updates --- src/views/DataView.tsx | 92 +++++++++++++++++++++++++++++++-- src/views/VisualizationView.tsx | 20 ++++--- 2 files changed, 98 insertions(+), 14 deletions(-) diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index aa1263d0..f6c4a79f 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -2,11 +2,15 @@ // Licensed under the MIT License. import React, { FC, useEffect, useMemo, useCallback } from 'react'; +import ReactDOM from 'react-dom'; import _ from 'lodash'; -import { Typography, Box, Link, Breadcrumbs, useTheme, Fade } from '@mui/material'; +import { Typography, Box, Link, Breadcrumbs, useTheme, Fade, IconButton, Tooltip } from '@mui/material'; import { alpha } from '@mui/material/styles'; +import { useTranslation } from 'react-i18next'; +import OpenInFullIcon from '@mui/icons-material/OpenInFull'; +import CloseFullscreenIcon from '@mui/icons-material/CloseFullscreen'; import '../scss/DataView.scss'; @@ -16,11 +20,19 @@ import { useDispatch, useSelector } from 'react-redux'; import { Type } from '../data/types'; import { SelectableDataGrid } from './SelectableDataGrid'; import { formatCellValue, getColumnAlign } from './ViewUtils'; +import { borderColor } from '../app/tokens'; export interface FreeDataViewProps { + // When true, render a maximize/restore toggle that pops the table into a + // full-canvas overlay. Used wherever the grid is shown inline (under a + // chart, or as the focused-table preview). + maximizable?: boolean; } -export const FreeDataViewFC: FC = function DataView() { +export const FreeDataViewFC: FC = function DataView({ maximizable }) { + + const { t } = useTranslation(); + const [maximized, setMaximized] = React.useState(false); const dispatch = useDispatch(); @@ -32,6 +44,7 @@ export const FreeDataViewFC: FC = function DataView() { const focusedTableId = useMemo(() => { if (!focusedId) return undefined; if (focusedId.type === 'table') return focusedId.tableId; + if (focusedId.type !== 'chart') return undefined; const chartId = focusedId.chartId; const chart = allCharts.find(c => c.id === chartId); return chart?.tableRef; @@ -108,7 +121,7 @@ export const FreeDataViewFC: FC = function DataView() { ]; }, [targetTable, rowData, conceptShelfItems]); - return ( + const grid = ( @@ -124,4 +137,77 @@ export const FreeDataViewFC: FC = function DataView() { ); + + if (!maximizable) { + return grid; + } + + const toggleButton = ( + + setMaximized(m => !m)} + sx={{ + color: 'text.secondary', + '&:hover': { color: 'primary.main', backgroundColor: 'transparent' }, + }} + > + {maximized ? : } + + + ); + + // The toggle button sits just outside the table to the right (a slim panel), + // so it never overlaps the column headers and the card keeps its original look. + // In maximized mode the surrounding overlay already provides the card frame. + const cardSx = maximized ? { overflow: 'hidden' } : { + overflow: 'hidden', + borderRadius: '8px', + border: `1px solid ${borderColor.divider}`, + transition: 'box-shadow 0.2s ease', + '&:hover': { boxShadow: '0 0 8px rgba(25, 118, 210, 0.25)' }, + }; + const framed = ( + + + {grid} + + + {toggleButton} + + + ); + + if (maximized) { + const canvas = typeof document !== 'undefined' ? document.getElementById('vis-view-canvas') : null; + const overlay = ( + <> + {/* Transparent click-catcher — click outside to restore. Scoped to the visualization view. */} + setMaximized(false)} + sx={{ position: 'absolute', inset: 0, zIndex: 1299 }} + /> + {/* Table overlay filling the visualization view. */} + + {framed} + + + ); + return ( + <> + {/* Keep the inline slot occupied so surrounding layout doesn't jump. */} + + {canvas ? ReactDOM.createPortal(overlay, canvas) : overlay} + + ); + } + + return framed; } \ No newline at end of file diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index 96d91d8c..7b6d18b4 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -932,11 +932,12 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { return sum + Math.max(80, Math.min(280, contentLen * 10)) + 60; }, ROW_ID_COL_WIDTH); const SCROLLBAR_WIDTH = 17; - const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)); + // +34px gutter so the maximize button can sit just outside the table on the right. + const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)) + 34; return ( - - + + ); })()} @@ -1096,7 +1097,7 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { , [localScaleFactor, t]); - return + return {synthesisRunning ? = function VisualizationView } return ( - + @@ -1281,18 +1282,15 @@ export const VisualizationViewFC: FC = function VisualizationView return sum + Math.max(80, Math.min(280, contentLen * 10)) + 60; }, ROW_ID_COL_WIDTH); const SCROLLBAR_WIDTH = 17; - const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)); + // +34px gutter so the maximize button can sit just outside the table on the right. + const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)) + 34; return ( - + ); })()} From a59ec9e04fd3c0c447f3fbffbe2a0f0573b7dd04 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 29 May 2026 09:25:24 -0700 Subject: [PATCH 05/36] some cleanup --- src/views/DataFormulator.tsx | 22 ++++---- src/views/DataThread.tsx | 14 ++--- src/views/SimpleChartRecBox.tsx | 98 ++++++++++++++++++++++++++------- src/views/threadLayout.ts | 39 +++++++++++++ 4 files changed, 133 insertions(+), 40 deletions(-) create mode 100644 src/views/threadLayout.ts diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index b340a477..90547525 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -40,6 +40,7 @@ import { DndProvider } from 'react-dnd' import { HTML5Backend } from 'react-dnd-html5-backend' import { toolName } from '../app/App'; import { DataThread } from './DataThread'; +import { threadPaneWidth } from './threadLayout'; import dfLogo from '../assets/df-logo.png'; import exampleImageTable from "../assets/example-image-table.png"; @@ -443,12 +444,9 @@ export const DataFormulatorFC = ({ }) => { //boxShadow: '0 0 5px rgba(0,0,0,0.1)', } - // Discrete column snapping for DataThread - const CARD_WIDTH = 220; - const CARD_GAP = 12; - const COLUMN_WIDTH = CARD_WIDTH + CARD_GAP; - const PANE_PADDING = 48; - const columnSize = (n: number) => n * COLUMN_WIDTH + PANE_PADDING; + // Discrete column snapping for DataThread. + // Column geometry is defined once in ./threadLayout and shared with + // DataThread so the pane snap points line up with the rendered columns. const allotmentRef = useRef(null); const containerRef = useRef(null); @@ -459,13 +457,13 @@ export const DataFormulatorFC = ({ }) => { let bestCols = 1; let bestDist = Infinity; for (let n = 1; n <= 3; n++) { - const dist = Math.abs(raw - columnSize(n)); + const dist = Math.abs(raw - threadPaneWidth(n)); if (dist < bestDist) { bestDist = dist; bestCols = n; } } - const snapped = columnSize(bestCols); + const snapped = threadPaneWidth(bestCols); if (Math.abs(raw - snapped) > 2) { const totalWidth = sizes.reduce((a, b) => a + b, 0); allotmentRef.current.resize([snapped, totalWidth - snapped]); @@ -545,10 +543,10 @@ export const DataFormulatorFC = ({ }) => { let newSize: number | null = null; if (prev <= 1 && threadCount > 1) { // Case 1: was 1 thread, now 2+ → expand to 2 columns - newSize = columnSize(2); + newSize = threadPaneWidth(2); } else if (prev > 1 && threadCount <= 1) { // Case 2: was 2+ threads, now 1 → shrink to 1 column - newSize = columnSize(1); + newSize = threadPaneWidth(1); } // Case 3: was 2+ threads and still 2+ → don't change (respect user's manual setting) @@ -581,7 +579,9 @@ export const DataFormulatorFC = ({ }) => { position: 'relative'}}> {tables.length > 0 ? ( - + = function ({ sx }) { // only one column fits, splitting a long thread into segments adds visual // overhead (continuation headers + ghost parents) without any layout // benefit, since the segments would just stack in the same single column. - const CARD_GAP = 12; // padding + spacing between cards in a column - const PANEL_PADDING = 16; - // 220 visual card width + 14px right gutter (CARD_CONTENT_PR) so cards - // keep their original size while gaining a right margin that balances - // the left timeline gutter. - const CARD_WIDTH = 234; - const COLUMN_WIDTH = CARD_WIDTH + CARD_GAP; - // n columns need: n*CARD_WIDTH + (n-1)*CARD_GAP + PANEL_PADDING - // Solving for n: n <= (containerWidth - PANEL_PADDING + CARD_GAP) / COLUMN_WIDTH - const fittableColumns = Math.max(1, Math.min(3, Math.floor((containerWidth - PANEL_PADDING + CARD_GAP) / COLUMN_WIDTH))); + // Column geometry (CARD_WIDTH / CARD_GAP / PANEL_PADDING) is defined once + // in ./threadLayout and shared with DataFormulator's pane snapping. + const fittableColumns = fittableThreadColumns(containerWidth); // Adaptively split long derivation chains so the resulting segments fill // the available columns evenly. See `computeSplitExtraLeaves` for the diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index 5e7bd63b..23b30c7c 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -40,7 +40,7 @@ import StopIcon from '@mui/icons-material/Stop'; import AutoGraphIcon from '@mui/icons-material/AutoGraph'; import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; -import { UnifiedDataUploadDialog } from './UnifiedDataUploadDialog'; +import InsertDriveFileOutlinedIcon from '@mui/icons-material/InsertDriveFileOutlined'; import { borderColor, transition } from '../app/tokens'; import { Theme } from '@mui/material/styles'; import { useTranslation } from 'react-i18next'; @@ -151,7 +151,8 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ const [mentionHighlightIdx, setMentionHighlightIdx] = useState(0); const [selectedAgent, setSelectedAgent] = useState<'explore' | 'report'>('explore'); const [attachedImages, setAttachedImages] = useState([]); - const [uploadDialogOpen, setUploadDialogOpen] = useState(false); + const [attachedFiles, setAttachedFiles] = useState<{ name: string; content: string }[]>([]); + const fileInputRef = useRef(null); const agentAbortRef = useRef(null); const userChartFocusLockedRef = useRef(false); const lastAutoFocusedChartIdRef = useRef(null); @@ -296,6 +297,31 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ } }, []); + // Attach files as conversation context. Images become reference images + // (sent to the model as attachments); text-like files are read as text + // and folded into the agent prompt as context. + const handleAttachFiles = React.useCallback((fileList: FileList | null) => { + if (!fileList) return; + const MAX_TEXT_CHARS = 50000; + Array.from(fileList).forEach(file => { + if (file.type.startsWith('image/')) { + const reader = new FileReader(); + reader.onload = () => setAttachedImages(prev => [...prev, reader.result as string]); + reader.readAsDataURL(file); + } else { + const reader = new FileReader(); + reader.onload = () => { + let content = (reader.result as string) || ''; + if (content.length > MAX_TEXT_CHARS) { + content = content.slice(0, MAX_TEXT_CHARS) + '\n…[truncated]'; + } + setAttachedFiles(prev => [...prev, { name: file.name, content }]); + }; + reader.readAsText(file); + } + }); + }, []); + // Collect table IDs from root up to (and including) the focused table for agent action matching const threadTableIds = React.useMemo(() => { if (!focusedTableId) return new Set(); @@ -373,6 +399,14 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ }, displayPrompt?: string) => { if (!focusedTableId || (!clarificationContext && prompt.trim() === "")) return; + // Fold attached reference files into the prompt the agent sees, while + // keeping the timeline bubble (displayContent) clean for the user. + const fileContext = attachedFiles.length > 0 + ? '\n\n' + attachedFiles.map(f => `[Attached file: ${f.name}]\n${f.content}`).join('\n\n') + : ''; + const agentPrompt = prompt + fileContext; + const cleanDisplay = displayPrompt ?? (fileContext ? prompt : undefined); + const rootTables = tables.filter(t => t.derive === undefined || t.anchored); const currentTable = tables.find(t => t.id === focusedTableId); const priorityIds = (currentTable?.derive && !currentTable.anchored) @@ -404,8 +438,8 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // 'clarifying' status and pendingClarification storage. if (isResume && pendingClarification?.draftId) { dispatch(dfActions.appendDraftInteraction({ draftId: pendingClarification.draftId, entry: { - from: 'user', to: 'data-agent', role: 'prompt', content: prompt, - ...(displayPrompt ? { displayContent: displayPrompt } : {}), + from: 'user', to: 'data-agent', role: 'prompt', content: agentPrompt, + ...(cleanDisplay ? { displayContent: cleanDisplay } : {}), timestamp: Date.now() }})); dispatch(dfActions.updateDraftClarification({ draftId: pendingClarification.draftId, pendingClarification: null })); @@ -552,10 +586,10 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // backend appends it to the trajectory as a normal user message. // No special clarification payload needed. requestBody.trajectory = clarificationContext!.trajectory; - requestBody.user_question = prompt; + requestBody.user_question = agentPrompt; requestBody.completed_step_count = clarificationContext!.completedStepCount; } else { - requestBody.user_question = prompt; + requestBody.user_question = agentPrompt; if (focusedThread) requestBody.focused_thread = focusedThread; if (otherThreads) requestBody.other_threads = otherThreads; } @@ -603,13 +637,13 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ currentDraftParentTableId = existingDraft?.derive?.trigger?.tableId || null; currentDraftInteraction = [...(existingDraft?.derive?.trigger?.interaction || [])]; // The user reply was already appended above, add to local accumulator too - currentDraftInteraction.push({ from: 'user', to: 'data-agent', role: 'prompt', content: prompt, - ...(displayPrompt ? { displayContent: displayPrompt } : {}), + currentDraftInteraction.push({ from: 'user', to: 'data-agent', role: 'prompt', content: agentPrompt, + ...(cleanDisplay ? { displayContent: cleanDisplay } : {}), timestamp: Date.now() }); } else { const initialEntries: InteractionEntry[] = [ - { from: 'user', to: 'data-agent', role: 'prompt', content: prompt, - ...(displayPrompt ? { displayContent: displayPrompt } : {}), + { from: 'user', to: 'data-agent', role: 'prompt', content: agentPrompt, + ...(cleanDisplay ? { displayContent: cleanDisplay } : {}), timestamp: Date.now() } ]; createNextDraft(lastCreatedTableId || focusedTableId!, initialEntries); @@ -940,6 +974,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ clearTimeout(timeoutId); setChatPrompt(""); setAttachedImages([]); + setAttachedFiles([]); isCompleted = true; } @@ -988,6 +1023,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ clearTimeout(timeoutId); setChatPrompt(""); setAttachedImages([]); + setAttachedFiles([]); isCompleted = true; } @@ -1028,6 +1064,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ if (completionResult) { setChatPrompt(""); setAttachedImages([]); + setAttachedFiles([]); } }; @@ -1110,7 +1147,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ } } })(); - }, [focusedTableId, tables, draftNodes, activeModel, config, conceptShelfItems, dispatch, t]); + }, [focusedTableId, tables, draftNodes, activeModel, config, conceptShelfItems, dispatch, t, attachedImages, attachedFiles]); // ── Report generation via report agent ────────────────────────── @@ -1473,7 +1510,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ {/* @-mention table chips and image attachments. Skip the table-chip row entirely when there's only one root table — there's nothing else the user could @-mention, so the chip is noise. */} - {((primaryTableIds.length > 0 && rootTables.length > 1) || attachedImages.length > 0) && !isChatFormulating && ( + {((primaryTableIds.length > 0 && rootTables.length > 1) || attachedImages.length > 0 || attachedFiles.length > 0) && !isChatFormulating && ( {rootTables.length > 1 && primaryTableIds.map(id => { const tbl = tables.find(t => t.id === id); @@ -1517,6 +1554,27 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ }} /> ))} + {attachedFiles.map((file, idx) => ( + } + label={file.name} + onDelete={() => setAttachedFiles(prev => prev.filter((_, i) => i !== idx))} + sx={{ + height: 20, + fontSize: 10, + maxWidth: 160, + color: theme.palette.text.secondary, + backgroundColor: 'rgba(0,0,0,0.04)', + border: 'none', + borderRadius: '4px', + '& .MuiChip-label': { px: '4px', overflow: 'hidden', textOverflow: 'ellipsis' }, + '& .MuiChip-icon': { ml: '4px', mr: '-2px' }, + '& .MuiChip-deleteIcon': { fontSize: 12, color: theme.palette.text.disabled, mr: '2px' }, + }} + /> + ))} )} {/* @-mention dropdown */} @@ -1645,10 +1703,17 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ {/* Action buttons */} - + { handleAttachFiles(e.target.files); if (e.target) e.target.value = ''; }} + /> + { e.stopPropagation(); setUploadDialogOpen(true); }} + onClick={(e) => { e.stopPropagation(); fileInputRef.current?.click(); }} sx={{ p: 0.5, color: theme.palette.text.secondary, @@ -1762,11 +1827,6 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ {/* The input box */} {inputBox} - setUploadDialogOpen(false)} - initialTab="menu" - /> ); }; diff --git a/src/views/threadLayout.ts b/src/views/threadLayout.ts new file mode 100644 index 00000000..fa793f2c --- /dev/null +++ b/src/views/threadLayout.ts @@ -0,0 +1,39 @@ +// Single source of truth for DataThread column geometry. +// +// Both the DataThread panel (which renders the thread columns) and +// DataFormulator (which snaps the resizable Allotment pane to whole-column +// widths) must agree on these values, otherwise the pane snap points won't +// line up with the actual rendered columns. Keep all width/padding tuning +// here. + +/** Visual width of a single thread card / column (px). */ +export const CARD_WIDTH = 248; + +/** Horizontal gap between adjacent columns (px). */ +export const CARD_GAP = 8; + +/** Total horizontal padding inside the thread panel (left + right, px). */ +export const PANEL_PADDING = 32; + +/** Max number of columns the thread panel will ever lay out. */ +export const MAX_THREAD_COLUMNS = 3; + +/** + * Pixel width required to display exactly `n` columns: + * n cards + (n-1) gaps + panel padding. + */ +export const threadPaneWidth = (n: number): number => + n * CARD_WIDTH + Math.max(0, n - 1) * CARD_GAP + PANEL_PADDING; + +/** + * How many whole columns fit within `containerWidth`, clamped to + * [1, MAX_THREAD_COLUMNS]. Inverse of `threadPaneWidth`. + */ +export const fittableThreadColumns = (containerWidth: number): number => + Math.max( + 1, + Math.min( + MAX_THREAD_COLUMNS, + Math.floor((containerWidth - PANEL_PADDING + CARD_GAP) / (CARD_WIDTH + CARD_GAP)), + ), + ); From 3a23dc3039f6299845f444f42b34180ecd2cf525 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Fri, 29 May 2026 16:28:43 -0700 Subject: [PATCH 06/36] workflow design --- py-src/data_formulator/agent_config.py | 2 +- .../agents/agent_chart_insight.py | 4 +- .../agents/agent_data_loading_chat.py | 8 +- .../agents/agent_interactive_explore.py | 4 +- ...e_distill.py => agent_workflow_distill.py} | 208 ++++++++++----- py-src/data_formulator/agents/data_agent.py | 33 ++- py-src/data_formulator/app.py | 2 +- py-src/data_formulator/knowledge/store.py | 136 ++++++---- py-src/data_formulator/routes/knowledge.py | 121 +++++---- src/api/knowledgeApi.ts | 28 +- src/app/useKnowledgeStore.ts | 14 +- src/i18n/locales/en/common.json | 46 ++-- src/i18n/locales/zh/common.json | 48 ++-- src/views/DataFrameTable.tsx | 27 +- src/views/DataSourceSidebar.tsx | 2 +- src/views/DataThread.tsx | 11 - src/views/InteractionEntryCard.tsx | 5 + src/views/KnowledgePanel.tsx | 244 +++++++----------- src/views/SessionDistill.tsx | 56 ++-- src/views/SimpleChartRecBox.tsx | 46 +++- ...xperienceContext.ts => workflowContext.ts} | 4 +- .../test_agent_knowledge_integration.py | 8 +- ...ce_distill.py => test_workflow_distill.py} | 162 +++++++----- .../backend/knowledge/test_knowledge_store.py | 69 +++-- tests/backend/routes/test_knowledge_routes.py | 137 +++++----- 25 files changed, 794 insertions(+), 631 deletions(-) rename py-src/data_formulator/agents/{agent_experience_distill.py => agent_workflow_distill.py} (58%) rename src/views/{experienceContext.ts => workflowContext.ts} (98%) rename tests/backend/agents/{test_experience_distill.py => test_workflow_distill.py} (74%) diff --git a/py-src/data_formulator/agent_config.py b/py-src/data_formulator/agent_config.py index bec4c670..67dbbe31 100644 --- a/py-src/data_formulator/agent_config.py +++ b/py-src/data_formulator/agent_config.py @@ -56,7 +56,7 @@ # ── Light: single-turn extractors / classifiers / formatters ──────────── "data_load": "minimal", # one-shot type inference "data_clean": "minimal", # extract tables from text - "experience_distill": "minimal", # summarise an analysis context + "workflow_distill": "minimal", # summarise an analysis context "chart_insight": "minimal", # title + 1–3 takeaways from a chart "chart_restyle": "minimal", # apply style edits to a Vega-Lite spec "code_explanation": "minimal", # describe derived fields diff --git a/py-src/data_formulator/agents/agent_chart_insight.py b/py-src/data_formulator/agents/agent_chart_insight.py index a3ae8aba..c280efc2 100644 --- a/py-src/data_formulator/agents/agent_chart_insight.py +++ b/py-src/data_formulator/agents/agent_chart_insight.py @@ -64,7 +64,7 @@ def run(self, chart_image_base64, chart_type, field_names, input_tables=None, n= search_query = " ".join([chart_type] + field_names[:5]).strip() if search_query: relevant = self._knowledge_store.search( - search_query, categories=["experiences"], max_results=3, + search_query, categories=["workflows"], max_results=3, ) if relevant: kb_parts = ["Relevant analysis knowledge:"] @@ -72,7 +72,7 @@ def run(self, chart_image_base64, chart_type, field_names, input_tables=None, n= kb_parts.append(f"- {item['title']}: {item['snippet'][:200]}") context_parts.append("\n".join(kb_parts)) except Exception: - logger.warning("Failed to search knowledge experiences", exc_info=True) + logger.warning("Failed to search knowledge workflows", exc_info=True) context = "\n".join(context_parts) diff --git a/py-src/data_formulator/agents/agent_data_loading_chat.py b/py-src/data_formulator/agents/agent_data_loading_chat.py index 61d3a0e6..55f2640a 100644 --- a/py-src/data_formulator/agents/agent_data_loading_chat.py +++ b/py-src/data_formulator/agents/agent_data_loading_chat.py @@ -1292,7 +1292,7 @@ def _build_system_prompt(self, last_user_text: str = ""): """Build the system prompt with current workspace context. *last_user_text* is used to search the knowledge store for - experiences relevant to the user's current request. Falls back + workflows relevant to the user's current request. Falls back to a generic query when empty. """ table_names = "none" @@ -1324,7 +1324,7 @@ def _build_system_prompt(self, last_user_text: str = ""): if self._knowledge_store: prompt += self._knowledge_store.format_rules_block() - # Inject relevant experiences from knowledge store + # Inject relevant workflows from knowledge store if self._knowledge_store: try: search_query = ( @@ -1334,7 +1334,7 @@ def _build_system_prompt(self, last_user_text: str = ""): ) relevant = self._knowledge_store.search( search_query, - categories=["experiences"], + categories=["workflows"], max_results=3, ) if relevant: @@ -1343,7 +1343,7 @@ def _build_system_prompt(self, last_user_text: str = ""): knowledge_block += f"\n### {item['title']}\n{item['snippet']}\n" prompt += "\n\n" + knowledge_block except Exception: - logger.warning("Failed to search knowledge experiences", exc_info=True) + logger.warning("Failed to search knowledge workflows", exc_info=True) if self.language_instruction: prompt += "\n\n" + self.language_instruction diff --git a/py-src/data_formulator/agents/agent_interactive_explore.py b/py-src/data_formulator/agents/agent_interactive_explore.py index 67847ec2..0f5f90fb 100644 --- a/py-src/data_formulator/agents/agent_interactive_explore.py +++ b/py-src/data_formulator/agents/agent_interactive_explore.py @@ -162,7 +162,7 @@ def run(self, input_tables, start_question=None, if start_question: context += f"\n\n[START QUESTION]\n\n{start_question}" - # ── Inject relevant experiences from knowledge store ────────── + # ── Inject relevant workflows from knowledge store ────────── if self._knowledge_store: try: query = start_question or "" @@ -170,7 +170,7 @@ def run(self, input_tables, start_question=None, search_query = " ".join([query] + table_names[:5]).strip() if search_query: relevant = self._knowledge_store.search( - search_query, categories=["experiences"], max_results=3, + search_query, categories=["workflows"], max_results=3, ) if relevant: knowledge_block = "[RELEVANT KNOWLEDGE]\n" diff --git a/py-src/data_formulator/agents/agent_experience_distill.py b/py-src/data_formulator/agents/agent_workflow_distill.py similarity index 58% rename from py-src/data_formulator/agents/agent_experience_distill.py rename to py-src/data_formulator/agents/agent_workflow_distill.py index cc738495..3f3d9c6d 100644 --- a/py-src/data_formulator/agents/agent_experience_distill.py +++ b/py-src/data_formulator/agents/agent_workflow_distill.py @@ -1,17 +1,17 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Experience distillation agent — extracts reusable knowledge from analysis context. +"""Workflow distillation agent — extracts a replayable workflow from analysis context. Given a user-visible analysis context (timeline of events) plus an optional user instruction, this agent calls an LLM to produce a structured Markdown -experience document with YAML front matter suitable for storage in the +workflow document with YAML front matter suitable for storage in the knowledge base. Usage:: - agent = ExperienceDistillAgent(client) - md_content = agent.run(experience_context, user_instruction="...") + agent = WorkflowDistillAgent(client) + md_content = agent.run(workflow_context, user_instruction="...") """ from __future__ import annotations @@ -25,18 +25,19 @@ logger = logging.getLogger(__name__) -_AGENT_ID = "experience_distill" +_AGENT_ID = "workflow_distill" SYSTEM_PROMPT = """\ -You are a knowledge distiller. Given the chronological events of a data -analysis session plus an optional user instruction, write a short reusable -Markdown note that will help with similar future tasks. +You are a workflow distiller. Given the chronological events of a data +analysis session plus an optional user instruction, extract a short, +**replayable workflow** that captures *what the user wanted and got* — so +the same analysis can be reproduced later on a similarly-shaped dataset. The session contains one or more threads (separate analysis branches in the same session) each rendered under a `### Thread N` header. When -multiple threads are provided, synthesise lessons that hold across them -— do NOT enumerate per-thread. +multiple threads are provided, merge them into one coherent ordered +workflow — do NOT enumerate per-thread. The events use three types: - `message` — directed speech, formatted as `[→/] `. @@ -46,56 +47,136 @@ (followed by columns, row count, sample, and code). - `create_chart` — a chart emitted on a table (mark + encoding summary). -If a user instruction is provided, focus the note on that instruction. -Otherwise, distill the most transferable methodology from the events. +Your job is to recover the **ordered list of requests** the user actually +wanted, and the outputs (tables/charts) they ended up keeping. Beyond the +concrete steps, also distill the analysis at TWO levels of abstraction so +it can be reused later: +- **Adapting to similar data** (concrete) — how to rerun essentially the + same analysis on a near-identical dataset, e.g. the business report for + a different month, region, or product line. Same shape and intent, only + the specific inputs/filters change. +- **Generalizing to other data** (abstract, dataset-agnostic) — the + underlying analytical pattern, independent of this domain: the kinds of + questions, computations, and charts involved, phrased so they transfer + to a different domain or a differently-shaped dataset. + +CRITICAL extraction rules — keep only what the user wanted and got: +- Each step = one user request, written in plain language. Say BOTH the + question being explored AND what was produced to answer it — including + the chart that was created and the key fields it uses (e.g. "Ask how + sales trend over time, and plot monthly total sales as a line chart"; + "Compare regions by breaking revenue down per region as a sorted bar + chart"). Order them as the analysis progressed. +- DROP corrective back-and-forth. If the user changed their mind + ("no, it should be…", "actually use median instead"), keep ONLY the + final resolved intent — not the wrong first attempt or the correction. +- DROP abandoned work. If a chart or table was created and then deleted + or never kept, leave it out entirely. +- DROP mechanics. Do NOT include error-repair loops, dtype fixes, tool + call noise, or low-level code. Describe intent, not implementation. +- Do NOT lean on code or exact column names unless a name is essential to + the request's meaning. Keep steps dataset-agnostic where possible so + they replay on a new slice of similar data. +- Capture genuine gotchas separately as short notes (advisory warnings to + carry forward), NOT as steps to re-perform. + +If a user instruction is provided, let it steer what to keep or emphasise. Output format (Markdown with YAML front matter, nothing else): ``` --- -subtitle: -tags: [] +subtitle: +filename: created: updated: source: distill source_context: --- -## When to Use - - -## Method - - -## Pitfalls & Tips - +## Goal + + +## Steps +1. +2. +3. <…> + +## Adapting to similar data + + +## Generalizing to other data + + +## Notes + ``` Rules: -- Subtitle must be a short, scannable noun phrase (3-8 words) that captures - the technique or pattern. The hosting application prefixes it with the - session name to form the full title (e.g. "Experience from : "), - so do NOT include the session name in the subtitle. Do NOT pack scenario, - takeaway, and steps into the subtitle — leave details for `## When to Use` - and `## Method`. - Good: "Year-over-year volatility comparison". "Repairing pandas dtype mismatches". - Bad: "Time series analysis workflow: aggregate, visualize trends, quantify YoY spikes, and compare volatility across periods". -- Focus on *transferable* methods and caveats, not case-specific details. -- Keep the body under 500 words. -- No raw data, PII, secrets, or specific values unless they show a universal pattern. -- Write the subtitle, headings, body, and tags in {output_language}. +- Subtitle must DESCRIBE what the workflow is about in PLAIN LANGUAGE that + a non-expert can fully understand at a glance, so they can decide + whether to replay it on new data. Favor clarity over brevity: it can be + a full sentence (up to ~25 words) if that makes the analysis genuinely + understandable. Write it like you would explain the analysis to a + colleague in one breath, covering the subject and the main thing you do + with it. The hosting application uses this subtitle directly as the + workflow's display title, so make it self-contained and do NOT prefix it + with the session name. + - Start with a concrete action verb (Plot, Compare, Break down, Rank, + Track, Summarize, Find…). + - Name the real-world subject in everyday words (sales, revenue, + customers, events), NOT the internal mechanics or derived-column + names you happened to create. + - AVOID abstract or technical jargon and invented noun-phrases + ("deltas", "composition", "window", "distribution shift"). If a + technique matters, phrase it plainly ("change from one period to the + next" instead of "deltas"). + Good: "Plot monthly sales over time and compare each year against the + previous one to spot volatile periods". + "Break revenue down by region and show how each region + contributes to the total as a stacked area chart". + "Track how many events happen in each time window and what kinds + of events make up each window". + Bad: "Time series analysis". "Data workflow". "Chart exploration". + "Event window deltas with composition". "Distribution shift inspection". +- Filename must be a SHORT (2-5 word) lowercase name for the file — just + the core subject and action, e.g. "monthly sales trend", "region revenue + breakdown". No dates, no file extension, no session name. It is only + used to name the file on disk; the descriptive subtitle is what users see. +- Steps must be ordered and reproducible. Each step should make clear the + question being explored and the chart/output produced to answer it. +- "Adapting to similar data" stays close to this analysis (same domain, + same shape) — only the concrete inputs change. "Generalizing to other + data" must be domain-neutral: strip out this dataset's subject matter and + describe only the transferable analytical pattern (question types, + computations, chart kinds). Do NOT just repeat the steps in either + section; add genuine reuse guidance. Keep each section brief. +- Be as long as the analysis needs — do not omit meaningful steps, + questions, or charts just to stay short. Stay focused, but completeness + matters more than brevity. +- No raw data, PII, secrets, or specific values unless essential to a request. +- Write the subtitle, headings, and body in {output_language}. YAML front-matter keys stay in English. {language_instruction} """ -class ExperienceDistillAgent: - """Distills analysis context into a reusable experience document.""" - # Language display names for experience-specific prompts +class WorkflowDistillAgent: + """Distills analysis context into a reusable workflow document.""" + + # Language display names for workflow-specific prompts _LANG_NAMES: dict[str, str] = { "zh": "Simplified Chinese (简体中文)", "ja": "Japanese (日本語)", @@ -121,7 +202,7 @@ def __init__( self.timeout_seconds = int(timeout_seconds) if timeout_seconds else self.DEFAULT_TIMEOUT def run(self, context: dict[str, Any], user_instruction: str = "") -> str: - """Distill an experience document from user-visible session context.""" + """Distill a workflow document from user-visible session context.""" summary = self._extract_context_summary(context) today = datetime.now(timezone.utc).strftime("%Y-%m-%d") context_id = str(context.get("context_id", "") or "") @@ -130,7 +211,7 @@ def run(self, context: dict[str, Any], user_instruction: str = "") -> str: instruction_block = ( f"\n[USER INSTRUCTION]\n{user_instruction.strip()}\n" - f"Focus the distilled experience on the above instruction.\n" + f"Focus the distilled workflow on the above instruction.\n" ) if user_instruction and user_instruction.strip() else "" workspace_block = ( @@ -158,9 +239,11 @@ def run(self, context: dict[str, Any], user_instruction: str = "") -> str: {"role": "user", "content": user_msg}, ] - from data_formulator.knowledge.store import KNOWLEDGE_LIMITS + from data_formulator.knowledge.store import KNOWLEDGE_LIMITS, WORKFLOW_HARD_MAX content = self._call_with_length_retry( - messages, KNOWLEDGE_LIMITS.get("experiences", 2000), + messages, + KNOWLEDGE_LIMITS.get("workflows", 6000), + WORKFLOW_HARD_MAX, ) if not content.strip().startswith("---"): @@ -182,7 +265,7 @@ def _prompt_format_kwargs(self) -> dict[str, str]: lang_block = ( f"[LANGUAGE INSTRUCTION]\n" f"The user's language is **{display_name}**.\n" - f"Write the title, all section headings, all body text, and tags " + f"Write the title, all section headings, and all body text " f"in {display_name}. YAML front-matter keys stay in English." ) return { @@ -199,39 +282,43 @@ def _prompt_format_kwargs(self) -> dict[str, str]: def _call_with_length_retry( self, messages: list[dict], - body_limit: int, + soft_limit: int, + hard_limit: int, ) -> str: - """Call LLM and retry once if the body exceeds *body_limit* characters. + """Call the LLM, nudging it to stay near *soft_limit* characters. - If the retry *still* overshoots, hard-truncate the body so the - document is saved instead of the entire distillation being lost. + ``soft_limit`` is advisory guidance: if the first response overshoots + it we retry once asking the model to condense. We only ever + hard-truncate at ``hard_limit`` — a much larger safety ceiling — so + rich, multi-section workflows are kept intact while runaway output + is still bounded. """ from data_formulator.knowledge.store import parse_front_matter content = self._call_llm(messages) _, body = parse_front_matter(content) - if len(body.strip()) <= body_limit: + if len(body.strip()) <= soft_limit: return content - retry_target = max(body_limit - self.RETRY_MARGIN, 1) + retry_target = max(soft_limit - self.RETRY_MARGIN, 1) logger.info( - "Distilled content too long (%d > %d), retrying with condensation prompt (target ≤ %d)", - len(body.strip()), body_limit, retry_target, + "Distilled content over soft target (%d > %d), retrying with condensation prompt (target ≤ %d)", + len(body.strip()), soft_limit, retry_target, ) messages = messages + [ {"role": "assistant", "content": content}, {"role": "user", "content": ( - f"Your output body is {len(body.strip())} characters, which exceeds " - f"the limit of {body_limit}. Please condense the document to fit " - f"within {retry_target} characters while keeping the most important " - f"insights. Output ONLY the revised Markdown document." + f"Your output body is {len(body.strip())} characters, which is " + f"longer than ideal. Please tighten the document to around " + f"{retry_target} characters while keeping the most important " + f"insights and all sections. Output ONLY the revised Markdown document." )}, ] retried = self._call_llm(messages) - # Hard-trim if the retry still overshoots — better a slightly - # truncated experience than a save failure. - return self._truncate_body_to_limit(retried, body_limit) + # Hard-trim only if the retry blows past the absolute ceiling — + # better a slightly truncated workflow than a save failure. + return self._truncate_body_to_limit(retried, hard_limit) @classmethod def _truncate_body_to_limit(cls, content: str, body_limit: int) -> str: @@ -385,7 +472,7 @@ def _render_events(cls, events: list[Any]) -> str: return "\n".join(parts) if parts else "(empty context)" def _call_llm(self, messages: list[dict]) -> str: - """Single LLM call to generate the experience document.""" + """Single LLM call to generate the workflow document.""" resp = self.client.get_completion( messages, reasoning_effort=reasoning_effort_for(_AGENT_ID, self.client.model), timeout=self.timeout_seconds, ) @@ -401,7 +488,6 @@ def _add_fallback_front_matter( header = ( f"---\ntitle: {title}\n" - f"tags: []\n" f"created: {today}\n" f"updated: {today}\n" f"source: distill\n" diff --git a/py-src/data_formulator/agents/data_agent.py b/py-src/data_formulator/agents/data_agent.py index 8e9cd39a..9a9d10b2 100644 --- a/py-src/data_formulator/agents/data_agent.py +++ b/py-src/data_formulator/agents/data_agent.py @@ -153,7 +153,7 @@ def _rescue_validate_action(data: dict) -> list[str]: "function": { "name": "search_knowledge", "description": ( - "Search the user's knowledge base (rules, experiences) " + "Search the user's knowledge base (rules, workflows) " "for relevant entries. Returns title, category, snippet, and " "path for each match. Use read_knowledge to get full content." ), @@ -168,7 +168,7 @@ def _rescue_validate_action(data: dict) -> list[str]: "type": "array", "items": { "type": "string", - "enum": ["rules", "experiences"], + "enum": ["rules", "workflows"], }, "description": "Optional: limit search to specific categories.", }, @@ -190,7 +190,7 @@ def _rescue_validate_action(data: dict) -> list[str]: "properties": { "category": { "type": "string", - "enum": ["rules", "experiences"], + "enum": ["rules", "workflows"], "description": "Knowledge category.", }, "path": { @@ -224,7 +224,7 @@ def _rescue_validate_action(data: dict) -> list[str]: - **inspect_source_data(table_names)** — get schema, stats, and sample rows for source tables (cheaper than explore for basic inspection). - **search_knowledge(query, categories?)** — search the user's knowledge base - (rules, experiences) for relevant entries. + (rules, workflows) for relevant entries. - **read_knowledge(category, path)** — read the full content of a knowledge entry. You analyse data that is **already in the workspace**. If the user's @@ -1379,14 +1379,14 @@ def _build_initial_messages( if peripheral_block: user_content += f"{peripheral_block}\n\n" - # Search and inject relevant knowledge (experiences + non-alwaysApply rules) + # Search and inject relevant knowledge (workflows + non-alwaysApply rules) table_names = [t.get("name", "") for t in input_tables if t.get("name")] relevant_knowledge = self._search_relevant_knowledge(user_question, table_names) - # Always include the experience distilled from the active workspace + # Always include the workflow distilled from the active workspace # (design-docs/24 §3.6) so the session has stable working memory # across turns regardless of search relevance. - session_exp = self._load_active_session_experience() + session_exp = self._load_active_session_workflow() if session_exp: existing_paths = { (item["category"], item["path"]) for item in relevant_knowledge @@ -1891,7 +1891,7 @@ def _search_relevant_knowledge( table_names: list[str], max_items: int = 5, ) -> list[dict[str, Any]]: - """Search experiences and non-alwaysApply rules relevant to the current session. + """Search workflows and non-alwaysApply rules relevant to the current session. Uses the user question as the search query and passes table names separately for tag-overlap boosting. alwaysApply rules are @@ -1904,7 +1904,7 @@ def _search_relevant_knowledge( try: results = self._knowledge_store.search( user_question, - categories=["rules", "experiences"], + categories=["rules", "workflows"], max_results=max_items, table_names=table_names[:5], ) @@ -1913,11 +1913,11 @@ def _search_relevant_knowledge( logger.warning("Failed to search knowledge", exc_info=True) return [] - def _load_active_session_experience(self) -> dict[str, Any] | None: - """Return the experience distilled from the active workspace, if any. + def _load_active_session_workflow(self) -> dict[str, Any] | None: + """Return the workflow distilled from the active workspace, if any. The session-scoped distillation flow (design-docs/24) writes one - experience per workspace, stamped with ``source_workspace_id``. + workflow per workspace, stamped with ``source_workspace_id``. We always inject that file into the agent's context so the agent has stable working memory for the active session in addition to whatever the relevance search picked. @@ -1932,14 +1932,14 @@ def _load_active_session_experience(self) -> dict[str, Any] | None: if not ws_id: return None try: - entry = self._knowledge_store.find_experience_by_workspace_id(ws_id) + entry = self._knowledge_store.find_workflow_by_workspace_id(ws_id) except Exception: - logger.warning("find_experience_by_workspace_id failed", exc_info=True) + logger.warning("find_workflow_by_workspace_id failed", exc_info=True) return None if not entry: return None try: - content = self._knowledge_store.read("experiences", entry["path"]) + content = self._knowledge_store.read("workflows", entry["path"]) except Exception: return None from data_formulator.knowledge.store import parse_front_matter @@ -1948,9 +1948,8 @@ def _load_active_session_experience(self) -> dict[str, Any] | None: if not snippet: return None return { - "category": "experiences", + "category": "workflows", "title": entry.get("title", entry.get("path", "")), - "tags": entry.get("tags", []), "path": entry["path"], "snippet": snippet, "source": entry.get("source", "distill"), diff --git a/py-src/data_formulator/app.py b/py-src/data_formulator/app.py index 47d2bda8..ef9dd4cb 100644 --- a/py-src/data_formulator/app.py +++ b/py-src/data_formulator/app.py @@ -219,7 +219,7 @@ def _register_blueprints(): from data_formulator.routes.credentials import credential_bp app.register_blueprint(credential_bp) - # Register knowledge management API (rules, skills, experiences) + # Register knowledge management API (rules, skills, workflows) from data_formulator.routes.knowledge import knowledge_bp app.register_blueprint(knowledge_bp) diff --git a/py-src/data_formulator/knowledge/store.py b/py-src/data_formulator/knowledge/store.py index 0b290093..08463437 100644 --- a/py-src/data_formulator/knowledge/store.py +++ b/py-src/data_formulator/knowledge/store.py @@ -1,10 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Knowledge store — manages user knowledge files (rules, experiences). +"""Knowledge store — manages user knowledge files (rules, workflows). Each user has a ``knowledge/`` directory under their home with two -sub-directories: ``rules`` and ``experiences``. Every knowledge entry is a +sub-directories: ``rules`` and ``workflows``. Every knowledge entry is a Markdown file with YAML front matter. All file I/O is routed through :class:`ConfinedDir` for path safety. @@ -12,7 +12,7 @@ Directory depth constraints: - ``rules``: flat — only files directly under ``rules/`` (1 path part) -- ``experiences``: one level of sub-directories (up to 2 path parts) +- ``workflows``: one level of sub-directories (up to 2 path parts) """ from __future__ import annotations @@ -27,19 +27,27 @@ logger = logging.getLogger(__name__) -VALID_CATEGORIES = frozenset({"rules", "experiences"}) +VALID_CATEGORIES = frozenset({"rules", "workflows"}) _MAX_DEPTH = { "rules": 1, - "experiences": 2, # one sub-dir: "category/file.md" + "workflows": 2, # one sub-dir: "category/file.md" } KNOWLEDGE_LIMITS: dict[str, int] = { "rule_description_max": 100, "rules": 350, - "experiences": 2000, + # Soft length guidance for distilled workflows: the target the distill + # agent aims for, NOT a hard cap. Workflows may exceed it when an + # analysis genuinely needs the room (e.g. multiple abstraction levels). + # Writes are only rejected past WORKFLOW_HARD_MAX below. + "workflows": 6000, } +# Absolute safety ceiling for a workflow body. Guards against runaway LLM +# output while still letting rich, multi-section workflows through. +WORKFLOW_HARD_MAX: int = 24000 + # --------------------------------------------------------------------------- # Tokenization helpers for improved search scoring # --------------------------------------------------------------------------- @@ -151,14 +159,13 @@ class KnowledgeItemMeta: """ __slots__ = ( - "title", "tags", "source", "created", "description", "always_apply", + "title", "source", "created", "description", "always_apply", "source_workspace_id", "source_workspace_name", ) def __init__( self, title: str, - tags: list[str], source: str, created: str, description: str, @@ -167,7 +174,6 @@ def __init__( source_workspace_name: str = "", ): self.title = title - self.tags = tags self.source = source self.created = created self.description = description @@ -181,14 +187,6 @@ def from_raw(cls, meta: dict[str, Any], fallback_stem: str = "") -> "KnowledgeIt title = meta.get("title", fallback_stem) title = str(title) if title is not None else fallback_stem - raw_tags = meta.get("tags", []) - if isinstance(raw_tags, list): - tags = [str(t) for t in raw_tags] - elif raw_tags is None: - tags = [] - else: - tags = [str(raw_tags)] - source = str(meta.get("source", "manual") or "manual") created = str(meta.get("created", "") or "") description = str(meta.get("description", "") or "") @@ -198,7 +196,6 @@ def from_raw(cls, meta: dict[str, Any], fallback_stem: str = "") -> "KnowledgeIt return cls( title=title, - tags=tags, source=source, created=created, description=description, @@ -246,26 +243,64 @@ class KnowledgeStore: store = KnowledgeStore(user_home) items = store.list_all("rules") - content = store.read("experiences", "data-cleaning/handle-missing.md") + content = store.read("workflows", "data-cleaning/handle-missing.md") store.write("rules", "date-format.md", md_content) store.delete("rules", "date-format.md") - results = store.search("ROI", categories=["rules", "experiences"]) + results = store.search("ROI", categories=["rules", "workflows"]) """ def __init__(self, user_home: Path | str) -> None: user_home = Path(user_home) self._root = ConfinedDir(user_home / "knowledge", mkdir=True) + self._migrate_experiences_to_workflows() self._jails: dict[str, ConfinedDir] = { "rules": ConfinedDir(self._root.root / "rules", mkdir=True), - "experiences": ConfinedDir(self._root.root / "experiences", mkdir=True), + "workflows": ConfinedDir(self._root.root / "workflows", mkdir=True), } self._migrate_flat() # -- migration --------------------------------------------------------- + def _migrate_experiences_to_workflows(self) -> None: + """Move legacy ``experiences/`` files into ``workflows/`` (one-time). + + The feature was renamed from "experiences" to "workflows"; existing + users have files under ``knowledge/experiences/``. Move them so the + rename is transparent. + """ + old_root = self._root.root / "experiences" + if not old_root.is_dir(): + return + new_root = self._root.root / "workflows" + new_root.mkdir(parents=True, exist_ok=True) + for md_file in list(old_root.rglob("*.md")): + rel = md_file.relative_to(old_root) + dest = new_root / rel + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists(): + stem = rel.stem + suffix_n = 1 + while dest.exists(): + dest = dest.parent / f"{stem}-{suffix_n}.md" + suffix_n += 1 + try: + md_file.rename(dest) + logger.info("Migrated experiences/%s → workflows/%s", rel, dest.name) + except Exception: + logger.warning("Failed to migrate experience file %s", md_file, exc_info=True) + # Remove the now-empty legacy tree (best effort) + try: + for sub in sorted(old_root.rglob("*"), reverse=True): + if sub.is_dir() and not any(sub.iterdir()): + sub.rmdir() + if not any(old_root.iterdir()): + old_root.rmdir() + except Exception: + logger.warning("Failed to clean up legacy experiences dir", exc_info=True) + def _migrate_flat(self) -> None: - """Move any experiences/subdir/file.md → experiences/file.md (one-time migration).""" - exp_root = self._jails["experiences"].root + """Move any workflows/subdir/file.md → workflows/file.md (one-time migration).""" + exp_root = self._jails["workflows"].root for md_file in list(exp_root.rglob("*.md")): rel = md_file.relative_to(exp_root) if len(rel.parts) <= 1: @@ -285,9 +320,9 @@ def _migrate_flat(self) -> None: parent = md_file.parent if parent != exp_root and not any(parent.iterdir()): parent.rmdir() - logger.info("Migrated knowledge experience %s → %s", rel, dest.name) + logger.info("Migrated knowledge workflow %s → %s", rel, dest.name) except Exception: - logger.warning("Failed to migrate experience file %s", md_file, exc_info=True) + logger.warning("Failed to migrate workflow file %s", md_file, exc_info=True) # -- path validation --------------------------------------------------- @@ -326,7 +361,7 @@ def _jail(self, category: str) -> ConfinedDir: def list_all(self, category: str) -> list[dict[str, Any]]: """List all knowledge entries in *category*. - Returns a list of dicts with ``title``, ``tags``, ``path``, + Returns a list of dicts with ``title``, ``path``, ``source``, and ``created`` parsed from front matter. For rules, also includes ``description`` and ``alwaysApply``. """ @@ -345,7 +380,6 @@ def list_all(self, category: str) -> list[dict[str, Any]]: rel = str(md_file.relative_to(jail.root)).replace("\\", "/") item: dict[str, Any] = { "title": km.title, - "tags": km.tags, "path": rel, "source": km.source, "created": km.created, @@ -353,9 +387,9 @@ def list_all(self, category: str) -> list[dict[str, Any]]: if category == "rules": item["description"] = km.description item["alwaysApply"] = km.always_apply - if category == "experiences": + if category == "workflows": # Surface session-distillation provenance so the frontend can - # find an existing session experience by workspace id + # find an existing session workflow by workspace id # without re-reading every file. See design-docs/24. if km.source_workspace_id: item["sourceWorkspaceId"] = km.source_workspace_id @@ -394,7 +428,15 @@ def write(self, category: str, path: str, content: str) -> Path: body_limit = KNOWLEDGE_LIMITS.get(category) if body_limit is not None: body_len = len(body.strip()) - if body_len > body_limit: + if category == "workflows": + # Soft guidance: the body_limit is a target the distill agent + # aims for, not a hard cap. Only reject far past the ceiling. + if body_len > WORKFLOW_HARD_MAX: + raise ValueError( + f"workflows body exceeds {WORKFLOW_HARD_MAX} characters " + f"(got {body_len})" + ) + elif body_len > body_limit: raise ValueError( f"{category} body exceeds {body_limit} characters " f"(got {body_len})" @@ -407,12 +449,12 @@ def delete(self, category: str, path: str) -> None: self.validate_path(category, path) self._jail(category).unlink(path) - # -- session experience helpers ---------------------------------------- + # -- session workflow helpers ---------------------------------------- - def find_experience_by_workspace_id( + def find_workflow_by_workspace_id( self, workspace_id: str, ) -> dict[str, Any] | None: - """Return the experience entry whose front matter records this workspace id. + """Return the workflow entry whose front matter records this workspace id. Used by the session-scoped distillation flow (design-docs/24) to upsert: when re-distilling the same session, overwrite the same @@ -421,11 +463,11 @@ def find_experience_by_workspace_id( if not workspace_id or not workspace_id.strip(): return None try: - for item in self.list_all("experiences"): + for item in self.list_all("workflows"): if item.get("sourceWorkspaceId") == workspace_id: return item except Exception: - logger.warning("find_experience_by_workspace_id failed", exc_info=True) + logger.warning("find_workflow_by_workspace_id failed", exc_info=True) return None # -- alwaysApply rules helper ------------------------------------------ @@ -511,12 +553,13 @@ def search( """Search across knowledge categories. Tokenizes *query* into keywords and scores each entry using - multi-field weighted matching (title > tags > filename > body). - Whole-string exact matches and table-name / tag overlaps receive + multi-field weighted matching (title > filename > body). + Whole-string exact matches and table-name overlaps receive additional bonuses. Non-manual sources are slightly discounted. *table_names* (optional) are table names from the current session; - when a table name appears in an entry's tags the entry is boosted. + when a table name appears in an entry's title or body the entry is + boosted. """ if not query or not query.strip(): return [] @@ -542,7 +585,7 @@ def search( continue score = self._match_score( - q, km.title, km.tags, md_file.stem, body[:200], + q, km.title, md_file.stem, body[:200], source=km.source, table_names=table_names, ) if score <= 0: @@ -552,7 +595,6 @@ def search( scored.append((score, { "category": cat, "title": km.title, - "tags": km.tags, "path": rel, "snippet": body[:500].strip(), "source": km.source, @@ -565,7 +607,6 @@ def search( def _match_score( query: str, title: str, - tags: list[str], stem: str, body_prefix: str, *, @@ -589,13 +630,10 @@ def _match_score( title_l = title.lower() stem_l = stem.lower() body_l = body_prefix.lower() - tags_l = [t.lower() for t in tags] for token in tokens: if token in title_l: score += 100 / n - if any(token in tl for tl in tags_l): - score += 50 / n if token in stem_l: score += 30 / n if token in body_l: @@ -604,14 +642,14 @@ def _match_score( # Whole-string bonus (handles short queries like "ROI") if q and q in title.lower(): score += 50 - if q and any(q in t.lower() for t in tags): - score += 50 - # Table-name → tag overlap bonus + # Table-name overlap bonus (title / body) if table_names: - tags_l_set = {t.lower() for t in tags} + title_l = title.lower() + body_l = body_prefix.lower() for tn in table_names: - if any(tn.lower() in tl for tl in tags_l_set): + tnl = tn.lower() + if tnl in title_l or tnl in body_l: score += 30 # Non-manual source slight discount diff --git a/py-src/data_formulator/routes/knowledge.py b/py-src/data_formulator/routes/knowledge.py index 901024c1..1a458ba9 100644 --- a/py-src/data_formulator/routes/knowledge.py +++ b/py-src/data_formulator/routes/knowledge.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Knowledge management API — CRUD + search + experience distillation. +"""Knowledge management API — CRUD + search + workflow distillation. All endpoints use ``POST`` with JSON body. Access is scoped to the current user via ``get_identity_id()`` and confined via ``ConfinedDir``. @@ -155,44 +155,44 @@ def knowledge_search(): return json_ok({"results": results}) -# ── distill experience ──────────────────────────────────────────────────── +# ── distill workflow ──────────────────────────────────────────────────── -@knowledge_bp.route("/distill-experience", methods=["POST"]) -def distill_experience(): - """Distill user-visible analysis context into a reusable experience. +@knowledge_bp.route("/distill-workflow", methods=["POST"]) +def distill_workflow(): + """Distill user-visible analysis context into a reusable workflow. Session-scoped payload (design-docs/24): - ``experience_context`` carries a list of ``threads`` (one per leaf + ``workflow_context`` carries a list of ``threads`` (one per leaf derived table the user has on screen), each with its own chronological ``events`` array. ``workspace_id`` + ``workspace_name`` bind the resulting file to the active session so re-distilling upserts the same file. - Required body fields: ``experience_context`` and ``model``. + Required body fields: ``workflow_context`` and ``model``. Optional: ``user_instruction`` (natural-language focus hint for the LLM), - ``category_hint`` (sub-directory under experiences/). + ``category_hint`` (sub-directory under workflows/). """ data = request.get_json(silent=True) or {} - experience_context = data.get("experience_context") - if not isinstance(experience_context, dict): - raise AppError(ErrorCode.INVALID_REQUEST, "'experience_context' is required") + workflow_context = data.get("workflow_context") + if not isinstance(workflow_context, dict): + raise AppError(ErrorCode.INVALID_REQUEST, "'workflow_context' is required") - threads = experience_context.get("threads") + threads = workflow_context.get("threads") if not isinstance(threads, list) or not threads: raise AppError( ErrorCode.INVALID_REQUEST, - "'experience_context.threads' is required and must be a non-empty list", + "'workflow_context.threads' is required and must be a non-empty list", ) - workspace_id_raw = experience_context.get("workspace_id", "") + workspace_id_raw = workflow_context.get("workspace_id", "") workspace_id = workspace_id_raw.strip() if isinstance(workspace_id_raw, str) else "" - workspace_name_raw = experience_context.get("workspace_name", "") + workspace_name_raw = workflow_context.get("workspace_name", "") workspace_name = workspace_name_raw.strip() if isinstance(workspace_name_raw, str) else "" if not workspace_id or not workspace_name: raise AppError( ErrorCode.INVALID_REQUEST, - "'experience_context.workspace_id' and 'workspace_name' are required", + "'workflow_context.workspace_id' and 'workspace_name' are required", ) model_config = data.get("model") @@ -215,53 +215,55 @@ def distill_experience(): # Build client and run distillation from data_formulator.routes.agents import get_client, _get_ui_lang - from data_formulator.agents.agent_experience_distill import ExperienceDistillAgent + from data_formulator.agents.agent_workflow_distill import WorkflowDistillAgent client = get_client(model_config) - agent = ExperienceDistillAgent( + agent = WorkflowDistillAgent( client=client, language_code=_get_ui_lang(), timeout_seconds=timeout_seconds, ) try: - md_content = agent.run(experience_context, user_instruction=user_instruction) + md_content = agent.run(workflow_context, user_instruction=user_instruction) except Exception as exc: - logger.warning("Experience distillation LLM call failed: %s", type(exc).__name__) + logger.warning("Workflow distillation LLM call failed: %s", type(exc).__name__) from data_formulator.error_handler import classify_and_wrap_llm_error raise classify_and_wrap_llm_error(exc) from exc - # Save to knowledge/experiences/ + # Save to knowledge/workflows/ store = KnowledgeStore(user_home) - # Bind the file to the workspace, override title to - # "Experience from : ", and upsert below. - md_content = _apply_session_front_matter(md_content, workspace_id, workspace_name) + # Bind the file to the workspace, set the title to the agent-generated + # descriptive subtitle, and upsert below. + md_content, title_core, filename_hint = _apply_session_front_matter( + md_content, workspace_id, workspace_name, + ) - filename = _experience_filename(workspace_name) + filename = _workflow_filename(filename_hint or title_core or workspace_name) rel_path = f"{category_hint}/{filename}" if category_hint else filename - # Upsert: if a previous experience exists for this workspace at a + # Upsert: if a previous workflow exists for this workspace at a # different path (e.g. user renamed the workspace), delete it after a # successful write so we keep one file per session. - existing = store.find_experience_by_workspace_id(workspace_id) + existing = store.find_workflow_by_workspace_id(workspace_id) try: - store.write("experiences", rel_path, md_content) + store.write("workflows", rel_path, md_content) except ValueError as exc: raise AppError(ErrorCode.INVALID_REQUEST, str(exc)) from exc if existing and existing.get("path") and existing["path"] != rel_path: try: - store.delete("experiences", existing["path"]) + store.delete("workflows", existing["path"]) except Exception: logger.warning( - "Failed to delete stale session experience at %s", + "Failed to delete stale session workflow at %s", existing.get("path"), exc_info=True, ) - return json_ok({"path": rel_path, "category": "experiences"}) + return json_ok({"path": rel_path, "category": "workflows"}) # ── helpers for session-scoped distillation ─────────────────────────────── @@ -269,16 +271,21 @@ def distill_experience(): def _apply_session_front_matter( content: str, workspace_id: str, workspace_name: str, -) -> str: - """Override / inject session-binding fields in the experience front matter. - - - Composes the visible ``title`` as ``Experience from : `` - using the LLM-emitted ``subtitle`` (preferred) or pre-existing - ``title``. The original ``subtitle`` field is removed from the - front matter once consumed. +) -> tuple[str, str, str]: + """Override / inject session-binding fields in the workflow front matter. + + - Sets the visible ``title`` to the agent-emitted descriptive + ``subtitle`` (preferred) or the pre-existing ``title``, with any + legacy ``Workflow from : `` prefix stripped. The ``subtitle`` + field is removed from the front matter once consumed. + - Consumes the agent-emitted short ``filename`` hint (removed from the + front matter) and returns it so the caller can name the file without + using the long descriptive title. - Stamps ``source_workspace_id`` and ``source_workspace_name`` so the file can be looked up on subsequent distillations. - Forces ``source: distill`` (idempotent if already set). + + Returns ``(content_with_front_matter, title_core, filename_hint)``. """ from data_formulator.knowledge.store import parse_front_matter @@ -287,27 +294,31 @@ def _apply_session_front_matter( meta = {} subtitle = str(meta.pop("subtitle", "") or "").strip() + filename_hint = str(meta.pop("filename", "") or "").strip() existing_title = str(meta.get("title", "") or "").strip() - # Strip any "Experience from : " prefix from a prior pass so - # update-mode runs don't double-prefix when the LLM echoes the title. - title_core = subtitle or _strip_experience_prefix(existing_title) + # Strip any legacy "Workflow from : " (or "Experience from") + # prefix so update-mode runs don't carry it forward. + title_core = subtitle or _strip_workflow_prefix(existing_title) if not title_core: title_core = workspace_name - new_title = f"Experience from {workspace_name}: {title_core}" - meta["title"] = new_title + meta["title"] = title_core meta["source"] = "distill" meta["source_workspace_id"] = workspace_id meta["source_workspace_name"] = workspace_name - return _serialize_front_matter(meta, body) + return _serialize_front_matter(meta, body), title_core, filename_hint + +_EXP_PREFIX_RE = re.compile(r"^\s*(?:Workflow|Experience) from .+?:\s*", re.IGNORECASE) -_EXP_PREFIX_RE = re.compile(r"^\s*Experience from .+?:\s*", re.IGNORECASE) +# Path separators, Windows-reserved chars and control chars that must never +# appear in a filename derived from untrusted LLM output. +_UNSAFE_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|\x00-\x1f]+') -def _strip_experience_prefix(title: str) -> str: +def _strip_workflow_prefix(title: str) -> str: return _EXP_PREFIX_RE.sub("", title).strip() @@ -323,16 +334,22 @@ def _serialize_front_matter(meta: dict, body: str) -> str: return f"---\n{yaml_text}\n---\n\n{body_text}" -def _experience_filename(workspace_name: str) -> str: - """Derive a deterministic filename from the workspace name. +def _workflow_filename(title: str) -> str: + """Slugify an LLM-supplied name into a clean, safe ``.md`` filename. - Re-distilling the same session always lands on the same file. - Falls back to a literal slug when sanitisation rejects the name. + Re-distilling a session upserts by ``source_workspace_id`` (see caller), + so the file is replaced even when the name changes. ``safe_data_filename`` + enforces the security boundary (basename only, no ``.``/``..``); the slug + step just keeps separators and reserved chars out so the name is clean and + portable. Unicode (e.g. CJK) is preserved. """ from data_formulator.datalake.parquet_utils import safe_data_filename - slug = workspace_name.strip().replace(" ", "-").lower()[:80] or "session-experience" + cleaned = _UNSAFE_FILENAME_CHARS.sub("-", title) + cleaned = re.sub(r"\s+", "-", cleaned.strip()) + cleaned = re.sub(r"-{2,}", "-", cleaned) + slug = cleaned.strip(".-").lower()[:80] or "session-workflow" try: return safe_data_filename(f"{slug}.md") except ValueError: - return "session-experience.md" + return "session-workflow.md" diff --git a/src/api/knowledgeApi.ts b/src/api/knowledgeApi.ts index 7c149f3a..a722c00f 100644 --- a/src/api/knowledgeApi.ts +++ b/src/api/knowledgeApi.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. /** - * Knowledge API client — CRUD, search, and experience distillation. + * Knowledge API client — CRUD, search, and workflow distillation. * * All endpoints use POST with JSON body. Requests go through * {@link fetchWithIdentity} for identity headers and 401 retry. @@ -14,11 +14,10 @@ import { apiRequest } from '../app/apiClient'; // ── Types ──────────────────────────────────────────────────────────────── -export type KnowledgeCategory = 'rules' | 'experiences'; +export type KnowledgeCategory = 'rules' | 'workflows'; export interface KnowledgeItem { title: string; - tags: string[]; path: string; source: string; created: string; @@ -27,25 +26,24 @@ export interface KnowledgeItem { /** Rules only: if true the rule is always injected into the agent prompt. */ alwaysApply?: boolean; /** - * Experiences only: workspace id this experience was distilled from. + * Workflows only: workspace id this workflow was distilled from. * Set by the session-scoped distillation flow (design-docs/24); used - * by the KnowledgePanel to find the existing session experience. + * by the KnowledgePanel to find the existing session workflow. */ sourceWorkspaceId?: string; - /** Experiences only: workspace display name at distillation time. */ + /** Workflows only: workspace display name at distillation time. */ sourceWorkspaceName?: string; } export interface KnowledgeLimits { rule_description_max: number; rules: number; - experiences: number; + workflows: number; } export interface KnowledgeSearchResult { category: KnowledgeCategory; title: string; - tags: string[]; path: string; snippet: string; source: string; @@ -122,7 +120,7 @@ export async function searchKnowledge( return data.results ?? []; } -export interface DistillExperienceResult { +export interface DistillWorkflowResult { path: string; category: string; } @@ -134,7 +132,7 @@ export interface DistillExperienceResult { * a deterministic filename + title. `threads` carries one chronological * `events` list per leaf table on screen. */ -export interface SessionExperienceContext { +export interface SessionWorkflowContext { context_id?: string; workspace_id: string; workspace_name: string; @@ -146,18 +144,18 @@ export interface SessionExperienceContext { payload_notes?: string[]; } -export async function distillSessionExperience( - sessionContext: SessionExperienceContext, +export async function distillSessionWorkflow( + sessionContext: SessionWorkflowContext, model: Record, instruction?: string, timeoutSeconds?: number, signal?: AbortSignal, -): Promise { - const { data } = await apiRequest<{ path: string; category: string }>('/api/knowledge/distill-experience', { +): Promise { + const { data } = await apiRequest<{ path: string; category: string }>('/api/knowledge/distill-workflow', { method: 'POST', headers: JSON_HEADERS, body: JSON.stringify({ - experience_context: sessionContext, + workflow_context: sessionContext, model, user_instruction: instruction, timeout_seconds: timeoutSeconds, diff --git a/src/app/useKnowledgeStore.ts b/src/app/useKnowledgeStore.ts index 0a6ea65d..6adeb60c 100644 --- a/src/app/useKnowledgeStore.ts +++ b/src/app/useKnowledgeStore.ts @@ -5,7 +5,7 @@ * Knowledge state management — React hooks for knowledge CRUD & search. * * Uses plain React state (not Redux) because knowledge data is server-side - * and only needed by the KnowledgePanel and save-as-experience flows. + * and only needed by the KnowledgePanel and save-as-workflow flows. * Errors are dispatched to the global MessageSnackbar via dfActions.addMessages. */ @@ -40,16 +40,16 @@ export function useKnowledgeStore() { const { t } = useTranslation(); const [rules, setRules] = useState({ ...EMPTY_CATEGORY }); - const [experiences, setExperiences] = useState({ ...EMPTY_CATEGORY }); + const [workflows, setWorkflows] = useState({ ...EMPTY_CATEGORY }); const [searchResults, setSearchResults] = useState([]); const [searching, setSearching] = useState(false); - const DEFAULT_LIMITS: KnowledgeLimits = { rule_description_max: 100, rules: 350, experiences: 2000 }; + const DEFAULT_LIMITS: KnowledgeLimits = { rule_description_max: 100, rules: 350, workflows: 2000 }; const [limits, setLimits] = useState(DEFAULT_LIMITS); - const stateMap = { rules, experiences }; - const setterMap = useRef({ rules: setRules, experiences: setExperiences }); + const stateMap = { rules, workflows }; + const setterMap = useRef({ rules: setRules, workflows: setWorkflows }); const fetchList = useCallback(async (category: KnowledgeCategory) => { const setter = setterMap.current[category]; @@ -71,7 +71,7 @@ export function useKnowledgeStore() { const fetchAll = useCallback(async () => { await Promise.all([ fetchList('rules'), - fetchList('experiences'), + fetchList('workflows'), fetchKnowledgeLimits().then(setLimits).catch(() => { /* best-effort */ }), ]); }, [fetchList]); @@ -184,7 +184,7 @@ export function useKnowledgeStore() { return { rules, - experiences, + workflows, stateMap, limits, searchResults, diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 7a52125b..8ef952df 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -875,9 +875,9 @@ "knowledge": { "title": "Agent Knowledge", "rules": "Rules", - "experiences": "Experiences", + "workflows": "Workflows", "rulesDescription": "Constraints and standards that agents must follow", - "experiencesDescription": "Reusable methods, tips, and knowledge distilled from analyses", + "workflowsDescription": "Reusable analysis workflows distilled from past sessions that agents can save and replay", "newItem": "New", "search": "Search", "searchPlaceholder": "Search knowledge...", @@ -902,29 +902,29 @@ "failedToSave": "Failed to save knowledge", "failedToDelete": "Failed to delete knowledge", "failedToSearch": "Search failed", - "saveAsExperience": "Save as Experience", - "saveAsExperienceTitle": "Save as Experience", - "distillHint": "Distill experience from this analysis for agents to reuse in future analysis.", + "saveAsExperience": "Save as Workflow", + "saveAsExperienceTitle": "Save as Workflow", + "distillHint": "Distill a workflow from this analysis for agents to save and replay in future sessions.", "distillFromHeading": "Distill from", "distillFromCaption": "Threads below will be sent to the LLM. Click a thread to inspect its events.", - "distillingOverlay": "Distilling experience… this may take a moment.", + "distillingOverlay": "Distilling workflow… this may take a moment.", "userInstruction": "User instruction (optional)", "userInstructionPlaceholder": "what to focus on, what to skip…", "distillationInstructions": "Distillation instructions (optional)", "distillationInstructionsPlaceholder": "e.g. focus on the data cleaning steps; skip exploratory chart variations; emphasise pitfalls we hit when joining tables…", - "distillExperience": "Distill Experience", - "distillStarted": "Distilling experience...", - "distilling": "Distilling experience...", - "distilled": "Experience saved", + "distillWorkflow": "Distill Workflow", + "distillStarted": "Distilling workflow...", + "distilling": "Distilling workflow...", + "distilled": "Workflow saved", "distillFailedRetry": "Save failed, retry", - "failedToDistill": "Failed to distill experience", - "distillSessionTitle": "Distill Session Experience", - "updateSessionTitle": "Update Session Experience", - "distillSessionHint": "Distill lessons from this analysis into a reusable knowledge document.", - "distillSessionUpdateHint": "Re-distill lessons from this analysis into the existing knowledge document.", + "failedToDistill": "Failed to distill workflow", + "distillSessionTitle": "Distill Session Workflow", + "updateSessionTitle": "Update Session Workflow", + "distillSessionHint": "Distill this analysis into a reusable workflow document that agents can replay.", + "distillSessionUpdateHint": "Re-distill this analysis into the existing workflow document.", "distillSessionNothing": "No completed analysis threads in this session yet.", "distillFromSession": "Distill from this session", - "experiencePlaceholderHint": "Save lessons learned", + "workflowPlaceholderHint": "Save this analysis as a workflow", "updateFromSession": "Update from this session", "updateFromSessionHint": "Refresh with new lessons", "addNewRule": "Add new rule", @@ -937,15 +937,21 @@ "itemCount": "({{count}})", "collapse": "Collapse", "expand": "Expand", - "emptyState": "Add rules or experiences to help AI agents work better.", - "rulesHint": "Rules — constraints the agent always follows. Click + to add your own.", - "experiencesHint": "Experiences — lessons distilled from your past analyses. Click the placeholder below to distill one from this session.", + "emptyState": "Add rules or workflows to help AI agents work better.", + "rulesHint": "Constraints the agent always follows.", + "workflowsHint": "Analyses distilled from past sessions that the agent can save and replay.", "markdownEditor": "Markdown Editor", "description": "Description", "descriptionPlaceholder": "Short summary of this rule (max {{max}} chars)", "alwaysApply": "Always loaded into AI", "alwaysApplyHint": "When enabled, this rule is always injected into every AI agent prompt, regardless of context", "charCount": "{{current}} / {{max}}", - "charCountExceeded": "Exceeds {{max}} character limit ({{current}} / {{max}})" + "charCountExceeded": "Exceeds {{max}} character limit ({{current}} / {{max}})", + "replay": "Replay", + "replayTooltip": "Replay this analysis on the current data", + "replayBusy": "The agent is busy — wait for it to finish before replaying.", + "replayNoData": "Load and focus a dataset before replaying a workflow.", + "replayStarted": "Replaying workflow on the current data…", + "replayPrompt": "Reproduce the following analysis workflow on the currently loaded data. Follow the steps in order, adapting any column references to the columns available in the current dataset. It's fine if the result isn't identical — reproduce the same overall analysis.\n\nBefore making large assumptions, check whether the current data can actually support the workflow. If there is a major discrepancy — e.g. a required field or measure is missing, the granularity or shape is very different, or a step has no sensible equivalent on this data — pause and ask me to confirm how to proceed (or briefly explain the mismatch and your proposed adaptation) instead of guessing. Minor differences (renamed columns, extra columns) can be adapted silently.\n\n{{content}}" } } diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index 1740e92a..244f93da 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -875,9 +875,9 @@ "knowledge": { "title": "Agent 知识", "rules": "规则", - "experiences": "经验", + "workflows": "工作流", "rulesDescription": "Agent 必须遵守的约束和编码规范", - "experiencesDescription": "从分析中提炼的可复用方法和技巧", + "workflowsDescription": "从过往会话中提炼、可供 Agent 保存与重放的可复用分析工作流", "newItem": "新建", "search": "搜索", "searchPlaceholder": "搜索知识...", @@ -902,29 +902,29 @@ "failedToSave": "保存知识失败", "failedToDelete": "删除知识失败", "failedToSearch": "搜索失败", - "saveAsExperience": "保存为经验", - "saveAsExperienceTitle": "保存为经验", - "distillHint": "从本次分析中提炼经验，供 Agent 在后续分析中复用。", + "saveAsExperience": "保存为工作流", + "saveAsExperienceTitle": "保存为工作流", + "distillHint": "从本次分析中提炼工作流，供 Agent 在后续会话中保存与重放。", "distillFromHeading": "提炼来源", "distillFromCaption": "以下线索将发送给 LLM。点击线索可查看其事件。", - "distillingOverlay": "正在提炼经验…请稍候。", + "distillingOverlay": "正在提炼工作流…请稍候。", "userInstruction": "用户指令（可选）", "userInstructionPlaceholder": "重点关注什么、跳过什么…", "distillationInstructions": "提炼指令（可选）", "distillationInstructionsPlaceholder": "例如：重点关注数据清洗步骤；跳过探索性图表变体；着重记录表连接时遇到的陷阱…", - "distillExperience": "提炼经验", - "distillStarted": "正在提炼经验...", - "distilling": "正在提炼经验...", - "distilled": "经验已保存", - "distillFailedRetry": "保存经验失败，重试", - "failedToDistill": "提炼经验失败", - "distillSessionTitle": "提炼会话经验", - "updateSessionTitle": "更新会话经验", - "distillSessionHint": "从本次分析中提炼经验，生成一篇可复用的知识文档。", - "distillSessionUpdateHint": "重新提炼本次分析的经验，覆盖现有的知识文档。", + "distillWorkflow": "提炼工作流", + "distillStarted": "正在提炼工作流...", + "distilling": "正在提炼工作流...", + "distilled": "工作流已保存", + "distillFailedRetry": "保存工作流失败，重试", + "failedToDistill": "提炼工作流失败", + "distillSessionTitle": "提炼会话工作流", + "updateSessionTitle": "更新会话工作流", + "distillSessionHint": "将本次分析提炼为一篇可供 Agent 重放的可复用工作流文档。", + "distillSessionUpdateHint": "将本次分析重新提炼到现有的工作流文档中。", "distillSessionNothing": "本会话还没有可提炼的分析线索。", "distillFromSession": "从本会话提炼", - "experiencePlaceholderHint": "保存分析中的经验", + "workflowPlaceholderHint": "将本次分析保存为工作流", "updateFromSession": "从本会话更新", "updateFromSessionHint": "用新经验刷新该条目", "addNewRule": "添加新规则", @@ -937,15 +937,21 @@ "itemCount": "（{{count}}）", "collapse": "收起", "expand": "展开", - "emptyState": "添加规则、技能或经验，帮助 AI Agent 更好地工作。", - "rulesHint": "规则 — Agent 始终遵守的约束。点击 + 添加你自己的规则。", - "experiencesHint": "经验 — 从你过往分析中提炼出的经验。点击下方占位项可从本会话提炼一条。", + "emptyState": "添加规则或工作流，帮助 AI Agent 更好地工作。", + "rulesHint": "Agent 始终遵守的约束。", + "workflowsHint": "从过往会话中提炼、Agent 可保存与重放的分析。", "markdownEditor": "Markdown 编辑器", "description": "描述", "descriptionPlaceholder": "规则的简短描述（最多 {{max}} 字符）", "alwaysApply": "始终加载到 AI", "alwaysApplyHint": "启用后，无论什么场景，此规则都会自动注入到每次 AI Agent 的提示词中", "charCount": "{{current}} / {{max}}", - "charCountExceeded": "超出 {{max}} 字符限制（{{current}} / {{max}}）" + "charCountExceeded": "超过 {{max}} 字符限制（{{current}} / {{max}}）", + "replay": "重放", + "replayTooltip": "在当前数据上重放此分析", + "replayBusy": "Agent 正忙——请等待其完成后再重放。", + "replayNoData": "请先加载并聚焦一个数据集，再重放工作流。", + "replayStarted": "正在当前数据上重放工作流…", + "replayPrompt": "在当前已加载的数据上复现以下分析流程。按顺序执行各步骤，并将其中的列引用调整为当前数据集中可用的列。结果不必完全一致——复现同样的整体分析即可。\n\n在做出较大假设之前，请先确认当前数据是否真的能支撑该流程。如果存在重大差异——例如缺少必需的字段或度量、数据粒度或结构差异很大、或某个步骤在当前数据上没有合理的对应方式——请暂停并向我确认如何继续（或简要说明不匹配之处及你建议的调整方案），而不要凭空猜测。对于细微差异（列被重命名、存在额外的列）可以直接静默调整。\n\n{{content}}" } } diff --git a/src/views/DataFrameTable.tsx b/src/views/DataFrameTable.tsx index dd32ecc9..afb03bbd 100644 --- a/src/views/DataFrameTable.tsx +++ b/src/views/DataFrameTable.tsx @@ -127,17 +127,24 @@ export const DataFrameTable: React.FC = ({ )} {displayCols.map((col, i) => { const desc = col !== '\u2026' ? columnDescriptions?.[col] : undefined; + if (desc) { + return ( + + + {col} + + + ); + } return ( - - - {col} - - + + {col} + ); })} diff --git a/src/views/DataSourceSidebar.tsx b/src/views/DataSourceSidebar.tsx index 7381a423..c0502fd0 100644 --- a/src/views/DataSourceSidebar.tsx +++ b/src/views/DataSourceSidebar.tsx @@ -157,7 +157,7 @@ export const DataSourceSidebar: React.FC<{ // appears when they try to add a new connector or link a folder. const [initialTab, setInitialTab] = useState<'sources' | 'sessions' | 'knowledge'>('sources'); - // External callers (e.g. SaveExperienceButton on success) can ask the + // External callers (e.g. workflow distill on success) can ask the // sidebar to open and switch to a specific tab. useEffect(() => { const handler = (e: Event) => { diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index 940cb4ef..248dbe75 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -1333,17 +1333,6 @@ let SingleThreadGroupView: FC<{ const mergeIds = derivedTable?.derive?.source as string[] | undefined; if (entry.role === 'instruction' && mergeNames && mergeNames.length > 0 && mergeIds && mergeIds.length > 0) { const nextKey = sourceSetKey(mergeIds); - // eslint-disable-next-line no-console - console.log('[merge-node check]', { - tableId, - parentTableId: parentTable?.id, - initialSourceIds, - prevSourceKey, - mergeIds, - mergeNames, - nextKey, - fires: nextKey !== prevSourceKey, - }); if (nextKey !== prevSourceKey) { const mergeColor = highlighted ? theme.palette.primary.main : theme.palette.text.secondary; timelineItems.push({ diff --git a/src/views/InteractionEntryCard.tsx b/src/views/InteractionEntryCard.tsx index 8cfd0000..79686ca2 100644 --- a/src/views/InteractionEntryCard.tsx +++ b/src/views/InteractionEntryCard.tsx @@ -242,6 +242,11 @@ export const InteractionEntryCard: React.FC = memo(({ // so they should read stronger than the agent's bubbles. backgroundColor: palette.bgcolor, border: `1px solid ${borderColor.component}`, + // Cap very long instructions (e.g. a replayed workflow) so the + // card stays compact; the full text scrolls within the cap. + maxHeight: 160, + overflowY: 'auto', + overscrollBehavior: 'contain', ...(highlighted ? { borderLeft: `2px solid ${palette.main}` } : {}), ...clickSx, }}> diff --git a/src/views/KnowledgePanel.tsx b/src/views/KnowledgePanel.tsx index 8d8a111f..05343a0a 100644 --- a/src/views/KnowledgePanel.tsx +++ b/src/views/KnowledgePanel.tsx @@ -4,16 +4,16 @@ /** * KnowledgePanel — panel for browsing and editing knowledge items. * - * Shows two collapsible sections: Rules (flat) and Experiences (flat). + * Shows two collapsible sections: Rules (flat) and Workflows (flat). * Items are tagged for organization; no subdirectory grouping. * Supports search, edit, and delete. Rules can be created directly by - * the user via the "+" affordance; experiences are produced by the + * the user via the "+" affordance; workflows are produced by the * agent's distillation flow (see SessionDistill). */ -import React, { useState, useCallback, useEffect, useRef } from 'react'; +import React, { useState, useCallback, useEffect } from 'react'; import { useTranslation } from 'react-i18next'; -import { useSelector } from 'react-redux'; +import { useSelector, useDispatch } from 'react-redux'; import { Box, Typography, @@ -26,7 +26,6 @@ import { DialogContent, DialogActions, CircularProgress, - Chip, Divider, } from '@mui/material'; import { alpha } from '@mui/material/styles'; @@ -34,6 +33,7 @@ import AddIcon from '@mui/icons-material/Add'; import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; import RefreshIcon from '@mui/icons-material/Refresh'; import Editor from 'react-simple-code-editor'; @@ -41,9 +41,9 @@ import { useKnowledgeStore } from '../app/useKnowledgeStore'; import { deleteKnowledge, type KnowledgeCategory } from '../api/knowledgeApi'; import type { KnowledgeItem } from '../api/knowledgeApi'; import { borderColor, radius } from '../app/tokens'; -import { type DataFormulatorState } from '../app/dfSlice'; -import { isLeafDerivedTable, buildLeafEvents } from './experienceContext'; -import { SessionDistillDialog, findSessionExperience } from './SessionDistill'; +import { dfActions, type DataFormulatorState } from '../app/dfSlice'; +import { isLeafDerivedTable, buildLeafEvents } from './workflowContext'; +import { SessionDistillDialog, findSessionWorkflow } from './SessionDistill'; // Default file name and seed body for a brand-new rule. Rules are plain // Markdown — the user just edits the body; no front matter is required. @@ -58,19 +58,18 @@ Describe the constraints or conventions the agent should follow. interface ActionRowProps { icon: React.ReactNode; label: string; - hint: string; onClick: () => void; } -const ActionRow: React.FC = ({ icon, label, hint, onClick }) => ( +const ActionRow: React.FC = ({ icon, label, onClick }) => ( `1px solid ${alpha(theme.palette.primary.main, 0.5)}`, @@ -88,22 +87,12 @@ const ActionRow: React.FC = ({ icon, label, hint, onClick }) => userSelect: 'none', }} > - {icon} - - - {label} - - - {hint} - - + {icon} + + {label} + ); @@ -112,8 +101,9 @@ const ActionRow: React.FC = ({ icon, label, hint, onClick }) => export const KnowledgePanel: React.FC = () => { const { t } = useTranslation(); const store = useKnowledgeStore(); + const dispatch = useDispatch(); - // For the "distill from this session" placeholder under EXPERIENCES. + // For the "distill from this session" placeholder under WORKFLOWS. const tables = useSelector((s: DataFormulatorState) => s.tables); const charts = useSelector((s: DataFormulatorState) => s.charts); const conceptShelfItems = useSelector((s: DataFormulatorState) => s.conceptShelfItems); @@ -183,35 +173,6 @@ export const KnowledgePanel: React.FC = () => { setEditorLoading(false); }, [store]); - // Pending request to auto-open an entry once it appears in the store - // (e.g. after the SessionDistillDialog finishes distilling). - const pendingOpenRef = useRef<{ category: KnowledgeCategory; path: string } | null>(null); - - useEffect(() => { - const handler = (e: Event) => { - const detail = (e as CustomEvent).detail || {}; - const category = (detail.category as KnowledgeCategory | undefined) ?? 'experiences'; - const path = detail.path as string | undefined; - if (path) { - pendingOpenRef.current = { category, path }; - } - }; - window.addEventListener('open-knowledge-panel', handler); - return () => window.removeEventListener('open-knowledge-panel', handler); - }, []); - - // When the requested entry shows up in the store, open its editor. - useEffect(() => { - const pending = pendingOpenRef.current; - if (!pending) return; - const cat = store.stateMap[pending.category]; - if (!cat?.loaded) return; - const item = cat.items.find(i => i.path === pending.path); - if (!item) return; - pendingOpenRef.current = null; - openEditDialog(pending.category, item); - }, [store.stateMap, openEditDialog]); - const handleSave = useCallback(async () => { if (!editorPath.trim() || !editorContent.trim()) return; setEditorSaving(true); @@ -237,9 +198,9 @@ export const KnowledgePanel: React.FC = () => { }, [deleteTarget, store]); // ── Distill from current session ──────────────────────────────────── - // The EXPERIENCES placeholder under EXPERIENCES is bound to the + // The WORKFLOWS placeholder is bound to the // active workspace. When the workspace already has a distilled - // experience (matched by `sourceWorkspaceId` in front matter) we + // workflow (matched by `sourceWorkspaceId` in front matter) we // expose an inline ⟳ Update affordance on the existing entry; // otherwise the placeholder opens the dialog in *create* mode. // See design-docs/24-session-scoped-distillation.md. @@ -265,9 +226,9 @@ export const KnowledgePanel: React.FC = () => { const selectedModel = allModels.find(m => m.id === selectedModelId); const canDistillFromSession = hasDistillableSession && !!selectedModel && !!activeWorkspace; - const sessionExperience = React.useMemo( - () => findSessionExperience( - store.stateMap['experiences'].items, + const sessionWorkflow = React.useMemo( + () => findSessionWorkflow( + store.stateMap['workflows'].items, activeWorkspace?.id, ), [store.stateMap, activeWorkspace?.id], @@ -278,6 +239,21 @@ export const KnowledgePanel: React.FC = () => { setSessionDialogOpen(true); }, []); + // ── Replay a workflow ──────────────────────────────────────────── + // Reads the workflow body and asks the data agent (in SimpleChartRecBox) + // to reproduce the captured workflow on the currently loaded data. v1 is + // deliberately simple: we hand the whole workflow to the agent in one + // request via a window event and let it figure out the rest. + // See discussion/replayable-experience-workflow.md. + const handleReplay = useCallback(async (item: KnowledgeItem) => { + const content = await store.read('workflows', item.path); + if (content == null) return; + const prompt = t('knowledge.replayPrompt', { content }); + window.dispatchEvent(new CustomEvent('df-replay-workflow', { + detail: { prompt, title: item.title }, + })); + }, [store, t]); + // ── Render section ────────────────────────────────────────────────── @@ -285,81 +261,85 @@ export const KnowledgePanel: React.FC = () => { category: KnowledgeCategory, item: KnowledgeItem, ) => { - const displayName = item.path || item.title; + const displayTitle = (item.title || '').replace(/^\s*(?:Workflow|Experience) from .+?:\s*/i, '').trim(); + const primary = displayTitle || item.title || item.path; return ( openEditDialog(category, item)} sx={{ display: 'flex', alignItems: 'flex-start', gap: 0.75, - px: 1.5, py: 0.75, + px: 1.5, py: 0.625, cursor: 'pointer', color: 'text.primary', '&:hover': { bgcolor: 'action.hover' }, - '&:hover .item-actions': { visibility: 'visible' }, + '&:hover .item-actions': { display: 'inline-flex' }, userSelect: 'none', }} > - + - - {displayName} + + {primary} - {item.tags.length > 0 && ( - - {item.tags.map(tag => ( - - ))} - - )} {item.source === 'agent_summarized' && ( )} - + + {category === 'workflows' && ( + + { e.stopPropagation(); handleReplay(item); }} + sx={{ + p: 0.25, + color: 'primary.main', + '&:hover': { bgcolor: theme => alpha(theme.palette.primary.main, 0.08) }, + }} + > + + + + )} { e.stopPropagation(); setDeleteTarget({ category, path: item.path, title: item.title }); }} - sx={{ p: 0.25, color: 'text.secondary', '&:hover': { color: 'error.main' } }} + sx={{ p: 0.25, display: 'none', color: 'text.secondary', '&:hover': { color: 'error.main' } }} > - + ); - }, [openEditDialog, t]); + }, [openEditDialog, t, handleReplay]); const renderCategorySection = useCallback(( category: KnowledgeCategory, label: string, + hint: string, ) => { const state = store.stateMap[category]; // Persistent action row at the top of the section. Rules: opens - // the create dialog. Experiences: opens the session distill + // the create dialog. Workflows: opens the session distill // dialog in create or update mode depending on whether the active - // workspace already has a distilled experience. + // workspace already has a distilled workflow. // See design-docs/24-session-scoped-distillation.md. const renderActionRow = () => { if (category === 'rules') { return ( } + icon={} label={t('knowledge.addNewRule', { defaultValue: 'Add new rule' })} - hint={t('knowledge.addNewRuleHint', { defaultValue: 'Set a convention for the agent' })} onClick={() => openCreateDialog('rules')} /> ); } - // experiences + // workflows if (!canDistillFromSession) { // No active workspace, no model, or no distillable thread // yet — show a passive hint instead of a dead action. @@ -370,15 +350,12 @@ export const KnowledgePanel: React.FC = () => { ); } - const updateMode = !!sessionExperience; + const updateMode = !!sessionWorkflow; if (sessionDistilling) { return ( } - label={t('knowledge.distilling', { defaultValue: 'Distilling experience…' })} - hint={updateMode - ? t('knowledge.updateFromSessionHint', { defaultValue: 'Refresh with new lessons' }) - : t('knowledge.experiencePlaceholderHint', { defaultValue: 'Save lessons learned' })} + icon={} + label={t('knowledge.distilling', { defaultValue: 'Distilling workflow…' })} onClick={() => openSessionDistillDialog(updateMode)} /> ); @@ -386,32 +363,44 @@ export const KnowledgePanel: React.FC = () => { return ( - : } + ? + : } label={updateMode ? t('knowledge.updateFromSession', { defaultValue: 'Update from this session' }) : t('knowledge.distillFromSession', { defaultValue: 'Distill from this session' })} - hint={updateMode - ? t('knowledge.updateFromSessionHint', { defaultValue: 'Refresh with new lessons' }) - : t('knowledge.experiencePlaceholderHint', { defaultValue: 'Save lessons learned' })} onClick={() => openSessionDistillDialog(updateMode)} /> ); }; return ( - + - + {label} + {/* Always-visible guidance for the section, set off by a + subtle left accent line below the title. */} + alpha(theme.palette.primary.main, 0.25), + }} + > + + {hint} + + + {state.loading && ( @@ -421,36 +410,18 @@ export const KnowledgePanel: React.FC = () => { {state.items.map(item => renderItem(category, item))} ); - }, [store.stateMap, renderItem, openCreateDialog, t, canDistillFromSession, sessionExperience, sessionDistilling, openSessionDistillDialog]); + }, [store.stateMap, renderItem, openCreateDialog, t, canDistillFromSession, sessionWorkflow, sessionDistilling, openSessionDistillDialog]); // ── Main render ───────────────────────────────────────────────────── return ( - {/* Persistent hint — explains Rules vs Experiences without - requiring the user to scroll past empty-state messages. */} - - - {t('knowledge.rulesHint')} - - - {t('knowledge.experiencesHint')} - - - - {/* Content area */} + {/* Content area. Rules vs Workflows guidance is surfaced via an + info icon next to each section title (see renderCategorySection). */} - {renderCategorySection('rules', t('knowledge.rules'))} - {renderCategorySection('experiences', t('knowledge.experiences'))} + {renderCategorySection('rules', t('knowledge.rules'), t('knowledge.rulesHint'))} + {renderCategorySection('workflows', t('knowledge.workflows'), t('knowledge.workflowsHint'))} @@ -515,22 +486,6 @@ export const KnowledgePanel: React.FC = () => { }} /> - {(() => { - const bodyLimit = store.limits[editorCategory as keyof typeof store.limits] as number | undefined; - if (!bodyLimit) return null; - const bodyLen = editorContent.trim().length; - const exceeded = bodyLen > bodyLimit; - return ( - bodyLimit * 0.9 ? 'warning.main' : 'text.disabled', - }}> - {exceeded - ? t('knowledge.charCountExceeded', { max: bodyLimit, current: bodyLen }) - : t('knowledge.charCount', { max: bodyLimit, current: bodyLen })} - - ); - })()} )} @@ -555,7 +510,6 @@ export const KnowledgePanel: React.FC = () => { editorSaving || !editorContent.trim() || !editorPath.trim() - || editorContent.trim().length > (store.limits[editorCategory as keyof typeof store.limits] as number ?? Infinity) } variant="contained" sx={{ textTransform: 'none', fontSize: 12 }} diff --git a/src/views/SessionDistill.tsx b/src/views/SessionDistill.tsx index fbff4f3b..fd9efeae 100644 --- a/src/views/SessionDistill.tsx +++ b/src/views/SessionDistill.tsx @@ -2,19 +2,19 @@ // Licensed under the MIT License. /** - * SessionDistill — session-scoped experience distillation. + * SessionDistill — session-scoped workflow distillation. * * Replaces the old per-result distillation flow with a single * session-bound entry. See design-docs/24-session-scoped-distillation.md. * * Exports: - * - buildSessionExperienceContext(workspace, threads): state-independent + * - buildSessionWorkflowContext(workspace, threads): state-independent * payload builder (with size budgeting, see §3.5 of the design doc). * - collectSessionThreads(tables, charts, fields): leaf discovery + per-leaf * event walk against live DataFormulator state. * - SessionDistillDialog: the dialog used by KnowledgePanel for both * create and update modes. - * - findSessionExperience: lookup an existing session experience by + * - findSessionWorkflow: lookup an existing session workflow by * workspace id. */ @@ -51,16 +51,16 @@ import { import { store, type AppDispatch } from '../app/store'; import { handleApiError } from '../app/errorHandler'; import { - distillSessionExperience, + distillSessionWorkflow, type KnowledgeItem, - type SessionExperienceContext, + type SessionWorkflowContext, } from '../api/knowledgeApi'; import { buildLeafEvents, buildDistillModelConfig, isLeafDerivedTable, TOOL_USES_CODE_FONT, -} from './experienceContext'; +} from './workflowContext'; // --------------------------------------------------------------------------- // Payload size budget (design-docs/24 §3.5) @@ -81,7 +81,7 @@ const SESSION_EVENT_BUDGET = 60_000; // bytes of JSON-serialized events // --------------------------------------------------------------------------- /** - * One pre-built thread, ready for `buildSessionExperienceContext`. + * One pre-built thread, ready for `buildSessionWorkflowContext`. * * Callers produce these by walking their own tables (see * `collectSessionThreads` for the in-app implementation) or with hand-built @@ -97,7 +97,7 @@ export interface SessionThread { export interface BuildSessionResult { /** Payload as it will be sent (after trimming). */ - payload: SessionExperienceContext; + payload: SessionWorkflowContext; /** Display threads with labels for the preview UI (post-trim). */ threads: SessionThread[]; /** Aggregate stats for the preview (post-trim). */ @@ -107,14 +107,14 @@ export interface BuildSessionResult { } // --------------------------------------------------------------------------- -// findSessionExperience +// findSessionWorkflow // --------------------------------------------------------------------------- /** - * Find the experience entry distilled from the given workspace, if any. + * Find the workflow entry distilled from the given workspace, if any. * Returns the first match; the backend ensures at most one per workspace. */ -export function findSessionExperience( +export function findSessionWorkflow( items: KnowledgeItem[] | undefined, workspaceId: string | undefined, ): KnowledgeItem | undefined { @@ -132,7 +132,7 @@ export function findSessionExperience( * * Threads with no user message are filtered out. Returns `[]` when the * session has no distillable thread. Not used in tests — tests construct - * `SessionThread[]` directly and call `buildSessionExperienceContext`. + * `SessionThread[]` directly and call `buildSessionWorkflowContext`. */ export function collectSessionThreads( tables: DictTable[], @@ -162,16 +162,16 @@ export function collectSessionThreads( } // --------------------------------------------------------------------------- -// buildSessionExperienceContext — pure (workspace, threads) → payload +// buildSessionWorkflowContext — pure (workspace, threads) → payload // --------------------------------------------------------------------------- /** - * Assemble the multi-thread payload sent to `/api/knowledge/distill-experience`. + * Assemble the multi-thread payload sent to `/api/knowledge/distill-workflow`. * * State-independent: takes pre-built threads and a workspace identity. * Returns `null` when `threads` is empty. */ -export function buildSessionExperienceContext( +export function buildSessionWorkflowContext( workspace: { id: string; displayName: string }, threads: SessionThread[], ): BuildSessionResult | null { @@ -179,7 +179,7 @@ export function buildSessionExperienceContext( const { trimmedThreads, notes } = trimToBudget(threads, SESSION_EVENT_BUDGET); - const payload: SessionExperienceContext = { + const payload: SessionWorkflowContext = { context_id: workspace.id, workspace_id: workspace.id, workspace_name: workspace.displayName, @@ -308,7 +308,7 @@ export const SessionDistillDialog: React.FC = ({ const built = useMemo(() => { if (!open || !activeWorkspace) return null; const threads = collectSessionThreads(tables, charts, conceptShelfItems); - return buildSessionExperienceContext(activeWorkspace, threads); + return buildSessionWorkflowContext(activeWorkspace, threads); }, [open, activeWorkspace, tables, charts, conceptShelfItems]); const [userInstruction, setUserInstruction] = useState(''); @@ -330,6 +330,10 @@ export const SessionDistillDialog: React.FC = ({ setStatus('running'); onRunningChange?.(true); const instruction = userInstruction.trim() || undefined; + // Close the dialog right away — distillation continues in the + // background and surfaces its result via the events/toast below. + setUserInstruction(''); + onClose(); try { const modelConfig = buildDistillModelConfig(selectedModel as ModelConfig); @@ -338,7 +342,7 @@ export const SessionDistillDialog: React.FC = ({ const timeoutId = setTimeout(() => controller.abort(), timeoutSeconds * 1000); let result; try { - result = await distillSessionExperience( + result = await distillSessionWorkflow( built.payload, modelConfig, instruction, timeoutSeconds, controller.signal, ); } finally { @@ -351,14 +355,12 @@ export const SessionDistillDialog: React.FC = ({ value: t('knowledge.distilled'), })); window.dispatchEvent(new CustomEvent('knowledge-changed', { - detail: { category: 'experiences' }, + detail: { category: 'workflows' }, })); window.dispatchEvent(new CustomEvent('open-knowledge-panel', { - detail: { category: 'experiences', path: result.path }, + detail: { category: 'workflows', path: result.path }, })); setStatus('idle'); - setUserInstruction(''); - onClose(); } catch (e: unknown) { setStatus('failed'); handleApiError(e, 'knowledge'); @@ -375,8 +377,8 @@ export const SessionDistillDialog: React.FC = ({ {updateMode - ? t('knowledge.updateSessionTitle', { defaultValue: 'Update Session Experience' }) - : t('knowledge.distillSessionTitle', { defaultValue: 'Distill Session Experience' })} + ? t('knowledge.updateSessionTitle', { defaultValue: 'Update Session Workflow' }) + : t('knowledge.distillSessionTitle', { defaultValue: 'Distill Session Workflow' })} = ({ {updateMode ? t('knowledge.distillSessionUpdateHint', { - defaultValue: 'Re-distill lessons from this analysis into the existing knowledge document.', + defaultValue: 'Re-distill this analysis into the existing workflow document.', }) : t('knowledge.distillSessionHint', { - defaultValue: 'Distill lessons from this analysis into a reusable knowledge document.', + defaultValue: 'Distill this analysis into a reusable workflow document that agents can replay.', })} @@ -479,7 +481,7 @@ export const SessionDistillDialog: React.FC = ({ ? t('knowledge.distilling') : updateMode ? t('knowledge.updateSession', { defaultValue: 'Update' }) - : t('knowledge.distillExperience')} + : t('knowledge.distillWorkflow')} diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index 23b30c7c..e94d2192 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -38,8 +38,6 @@ import AddIcon from '@mui/icons-material/Add'; import TipsAndUpdatesIcon from '@mui/icons-material/TipsAndUpdates'; import StopIcon from '@mui/icons-material/Stop'; -import AutoGraphIcon from '@mui/icons-material/AutoGraph'; -import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; import InsertDriveFileOutlinedIcon from '@mui/icons-material/InsertDriveFileOutlined'; import { borderColor, transition } from '../app/tokens'; import { Theme } from '@mui/material/styles'; @@ -70,7 +68,7 @@ const AgentWorkingOverlay: FC<{ message?: string; elapsed?: number; theme: Theme }}> - + {t('chartRec.agentWorking')} @@ -96,13 +94,13 @@ const AgentWorkingOverlay: FC<{ message?: string; elapsed?: number; theme: Theme )} {latestMessage}{elapsedSuffix} @@ -1354,6 +1352,39 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ exploreFromChat(prompt, undefined, displayPrompt); }, [reportFromChat, exploreFromChat, selectedAgent, clarificationQuestions, clarifyAnswers]); + // Replay a workflow: the KnowledgePanel fires `df-replay-workflow` + // with a prompt describing the captured workflow; we hand it straight to + // the data agent on the currently focused dataset. v1 is deliberately + // simple — one request, let the agent reproduce the analysis on its own. + // See discussion/replayable-experience-workflow.md. + useEffect(() => { + const handler = (e: Event) => { + const prompt = (e as CustomEvent).detail?.prompt as string | undefined; + if (!prompt) return; + if (isChatFormulating) { + dispatch(dfActions.addMessages({ + timestamp: Date.now(), type: 'error', + component: 'data-agent', value: t('knowledge.replayBusy'), + })); + return; + } + if (!focusedTableId) { + dispatch(dfActions.addMessages({ + timestamp: Date.now(), type: 'error', + component: 'data-agent', value: t('knowledge.replayNoData'), + })); + return; + } + dispatch(dfActions.addMessages({ + timestamp: Date.now(), type: 'info', + component: 'data-agent', value: t('knowledge.replayStarted'), + })); + exploreFromChat(prompt); + }; + window.addEventListener('df-replay-workflow', handler); + return () => window.removeEventListener('df-replay-workflow', handler); + }, [exploreFromChat, isChatFormulating, focusedTableId, dispatch, t]); + const resumeFromClarification = useCallback((responses: ClarificationResponse[]) => { if (!pendingClarification) return; // Pass the formatted display string as `prompt` — it powers both the @@ -1744,9 +1775,6 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ '&:hover': { backgroundColor: alpha(isReportMode ? theme.palette.warning.main : theme.palette.primary.main, 0.08) }, }} > - {selectedAgent === 'explore' - ? - : } {selectedAgent === 'explore' ? t('chartRec.modeExplore') : t('chartRec.modeReport')} @@ -1761,7 +1789,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ submitChat(t('chartRec.exploreIdeasPrompt'), undefined, t('chartRec.askedForRecommendations'))} > diff --git a/src/views/experienceContext.ts b/src/views/workflowContext.ts similarity index 98% rename from src/views/experienceContext.ts rename to src/views/workflowContext.ts index 98ec1c80..dac6e006 100644 --- a/src/views/experienceContext.ts +++ b/src/views/workflowContext.ts @@ -2,8 +2,8 @@ // Licensed under the MIT License. /** - * experienceContext — pure helpers that turn DataFormulator state into - * the timeline payload sent to `/api/knowledge/distill-experience`. + * workflowContext — pure helpers that turn DataFormulator state into + * the timeline payload sent to `/api/knowledge/distill-workflow`. * * No React, no Redux. Used by: * - SessionDistill.collectSessionThreads (live distillation) diff --git a/tests/backend/agents/test_agent_knowledge_integration.py b/tests/backend/agents/test_agent_knowledge_integration.py index 3efc65df..4d738635 100644 --- a/tests/backend/agents/test_agent_knowledge_integration.py +++ b/tests/backend/agents/test_agent_knowledge_integration.py @@ -62,7 +62,7 @@ def user_home(tmp_path): rules_dir.mkdir(parents=True) (rules_dir / "roi.md").write_text(RULE_MD, encoding="utf-8") - exp_dir = tmp_path / "knowledge" / "experiences" / "cleaning" + exp_dir = tmp_path / "knowledge" / "workflows" / "cleaning" exp_dir.mkdir(parents=True) (exp_dir / "missing.md").write_text(SKILL_MD, encoding="utf-8") @@ -170,11 +170,11 @@ def test_no_match_no_injection(self, mock_client, mock_workspace, user_home): def test_max_five_items(self, mock_client, mock_workspace, tmp_path): rules_dir = tmp_path / "knowledge" / "rules" rules_dir.mkdir(parents=True) - exp_dir = tmp_path / "knowledge" / "experiences" / "common" + exp_dir = tmp_path / "knowledge" / "workflows" / "common" exp_dir.mkdir(parents=True) for i in range(10): (exp_dir / f"exp-{i}.md").write_text( - f"---\ntitle: Common Experience {i}\ntags: [common]\n" + f"---\ntitle: Common Workflow {i}\ntags: [common]\n" f"created: 2026-04-26\nupdated: 2026-04-26\n---\n" f"Content about common topic {i}.\n", encoding="utf-8", @@ -247,7 +247,7 @@ def test_agent_works_without_knowledge(self, mock_client, mock_workspace): def test_empty_knowledge_dir(self, mock_client, mock_workspace, tmp_path): """Agent with empty knowledge dir works normally.""" (tmp_path / "knowledge" / "rules").mkdir(parents=True) - (tmp_path / "knowledge" / "experiences").mkdir(parents=True) + (tmp_path / "knowledge" / "workflows").mkdir(parents=True) agent = _make_agent(mock_client, mock_workspace, tmp_path) prompt = agent._build_system_prompt() assert "User Rules" not in prompt diff --git a/tests/backend/agents/test_experience_distill.py b/tests/backend/agents/test_workflow_distill.py similarity index 74% rename from tests/backend/agents/test_experience_distill.py rename to tests/backend/agents/test_workflow_distill.py index a3b823c8..e44d4a86 100644 --- a/tests/backend/agents/test_experience_distill.py +++ b/tests/backend/agents/test_workflow_distill.py @@ -1,13 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Tests for ExperienceDistillAgent and the /api/knowledge/distill-experience endpoint. +"""Tests for WorkflowDistillAgent and the /api/knowledge/distill-workflow endpoint. Covers: -- _extract_context_summary correctly extracts experience context +- _extract_context_summary correctly extracts workflow context - Output Markdown includes valid YAML front matter - front matter contains source: distill and source metadata -- Generated experience file written to correct directory +- Generated workflow file written to correct directory - category_hint controls sub-directory """ @@ -18,8 +18,14 @@ import flask import pytest -from data_formulator.agents.agent_experience_distill import ExperienceDistillAgent -from data_formulator.knowledge.store import parse_front_matter +from data_formulator.agents.agent_workflow_distill import WorkflowDistillAgent +from data_formulator.knowledge.store import ( + KNOWLEDGE_LIMITS, + WORKFLOW_HARD_MAX, + parse_front_matter, +) + +WORKFLOW_SOFT_LIMIT = KNOWLEDGE_LIMITS["workflows"] pytestmark = [pytest.mark.backend] @@ -73,7 +79,7 @@ }, ] -SAMPLE_EXPERIENCE_CONTEXT = { +SAMPLE_WORKFLOW_CONTEXT = { "context_id": "ws-1", "workspace_id": "ws-1", "workspace_name": "Sales Region Analysis", @@ -86,7 +92,7 @@ class TestExtractContextSummary: def test_renders_each_event_type(self): - summary = ExperienceDistillAgent._extract_context_summary(SAMPLE_EXPERIENCE_CONTEXT) + summary = WorkflowDistillAgent._extract_context_summary(SAMPLE_WORKFLOW_CONTEXT) # message events assert "[user→data-agent/prompt]" in summary assert "Show sales by region" in summary @@ -113,7 +119,7 @@ def test_renders_each_event_type(self): assert "encoding: x=region(nominal)" in summary def test_empty_events_returns_marker(self): - summary = ExperienceDistillAgent._extract_context_summary({}) + summary = WorkflowDistillAgent._extract_context_summary({}) assert summary == "(empty context)" def test_user_content_is_not_displaycontent(self): @@ -131,7 +137,7 @@ def test_user_content_is_not_displaycontent(self): }], }], } - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "raw text" in summary def test_skips_non_dict_events(self): @@ -140,7 +146,7 @@ def test_skips_non_dict_events(self): {"type": "message", "from": "user", "to": "data-agent", "role": "prompt", "content": "ok"}, ]}]} - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "[user→data-agent/prompt]" in summary # No crashes; the bogus entries are silently dropped. @@ -154,7 +160,7 @@ def test_create_table_basic(self): "sample_rows": [{"a": 1}], "code": "x = 1", }]}]} - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "[create_table] t1" in summary def test_create_chart_without_encoding(self): @@ -163,7 +169,7 @@ def test_create_chart_without_encoding(self): "related_table_id": "t1", "mark_or_type": "line", }]}]} - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "[create_chart] line on t1" in summary assert "encoding:" not in summary @@ -187,7 +193,7 @@ def test_renders_multi_thread_with_headers(self): }, ], } - summary = ExperienceDistillAgent._extract_context_summary(ctx) + summary = WorkflowDistillAgent._extract_context_summary(ctx) assert "### Thread 1 (id=leaf-a)" in summary assert "### Thread 2 (id=leaf-b)" in summary assert "load gas prices" in summary @@ -236,10 +242,10 @@ def _mock_client(self): def test_produces_valid_markdown(self): client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) with patch.object(agent, "_call_llm", return_value=MOCK_CONTEXT_RESPONSE): - result = agent.run(SAMPLE_EXPERIENCE_CONTEXT) + result = agent.run(SAMPLE_WORKFLOW_CONTEXT) assert result.startswith("---") meta, body = parse_front_matter(result) @@ -249,11 +255,11 @@ def test_produces_valid_markdown(self): def test_fallback_front_matter_added(self): client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) no_fm_response = "# Sales Analysis\n\nJust some content." with patch.object(agent, "_call_llm", return_value=no_fm_response): - result = agent.run(SAMPLE_EXPERIENCE_CONTEXT) + result = agent.run(SAMPLE_WORKFLOW_CONTEXT) assert result.startswith("---") meta, _ = parse_front_matter(result) @@ -261,11 +267,11 @@ def test_fallback_front_matter_added(self): assert meta["source_context"] == "ws-1" def test_retries_once_when_body_too_long(self): - """If first LLM call produces body > limit, agent retries with condensation prompt.""" + """If first LLM call produces body over the soft target, agent retries with condensation prompt.""" client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) - long_body = "x" * 3000 + long_body = "x" * (WORKFLOW_SOFT_LIMIT + 1000) long_response = ( "---\ntitle: Long\ntags: []\ncreated: 2026-01-01\n" "updated: 2026-01-01\nsource: distill\nsource_context: t1\n---\n\n" @@ -283,18 +289,18 @@ def fake_call_llm(messages): return short_response with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - result = agent.run(SAMPLE_EXPERIENCE_CONTEXT) + result = agent.run(SAMPLE_WORKFLOW_CONTEXT) assert call_count == 2 _, body = parse_front_matter(result) - assert len(body.strip()) <= 2000 + assert len(body.strip()) <= WORKFLOW_SOFT_LIMIT def test_retry_asks_for_slack_under_limit(self): - """The retry prompt asks the model for less than the hard limit.""" + """The retry prompt asks the model for less than the soft target.""" client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) - long_body = "x" * 3000 + long_body = "x" * (WORKFLOW_SOFT_LIMIT + 1000) long_response = ( "---\ntitle: L\ntags: []\ncreated: 2026-01-01\n" "updated: 2026-01-01\nsource: distill\nsource_context: t1\n---\n\n" @@ -309,21 +315,21 @@ def fake_call_llm(messages): return long_response if len(captured) == 1 else MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) assert len(captured) == 2 retry_prompt = captured[1][-1]["content"] - # Must mention the slacked target (limit minus margin), not the raw limit. - expected_target = 2000 - agent.RETRY_MARGIN - assert f"within {expected_target} characters" in retry_prompt + # Must mention the slacked target (soft limit minus margin). + expected_target = WORKFLOW_SOFT_LIMIT - agent.RETRY_MARGIN + assert f"around {expected_target} characters" in retry_prompt def test_hard_trims_when_retry_still_over_limit(self): - """If the retry still overshoots, body is hard-trimmed to fit the limit.""" + """If the retry still blows past the hard ceiling, body is hard-trimmed to fit it.""" client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) - first_body = "x" * 3000 - retry_body = "y" * 2014 # mimics the real-world failure: 14 over + first_body = "x" * (WORKFLOW_SOFT_LIMIT + 1000) + retry_body = "y" * (WORKFLOW_HARD_MAX + 14) # mimics retry still over the ceiling front_matter = ( "---\ntitle: T\ntags: []\ncreated: 2026-01-01\n" "updated: 2026-01-01\nsource: distill\nsource_context: t1\n---\n\n" @@ -339,13 +345,13 @@ def fake_call_llm(messages): return resp with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - result = agent.run(SAMPLE_EXPERIENCE_CONTEXT) + result = agent.run(SAMPLE_WORKFLOW_CONTEXT) # Both LLM calls happened. assert call_count == 2 - # Final body fits the hard limit (no save failure). + # Final body fits the hard ceiling (no save failure). _, body = parse_front_matter(result) - assert len(body.strip()) <= 2000 + assert len(body.strip()) <= WORKFLOW_HARD_MAX # Truncation marker is present so the user can see it was trimmed. assert "truncated" in body # Front matter preserved. @@ -355,7 +361,7 @@ def fake_call_llm(messages): def test_no_retry_when_body_within_limit(self): """If first LLM call is within limit, no retry happens.""" client = self._mock_client() - agent = ExperienceDistillAgent(client=client) + agent = WorkflowDistillAgent(client=client) call_count = 0 @@ -365,14 +371,14 @@ def fake_call_llm(messages): return MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) assert call_count == 1 def test_language_instruction_injected_into_system_prompt(self): client = self._mock_client() zh_instruction = "[LANGUAGE INSTRUCTION]\nWrite in Simplified Chinese." - agent = ExperienceDistillAgent(client=client, language_instruction=zh_instruction) + agent = WorkflowDistillAgent(client=client, language_instruction=zh_instruction) captured_messages = [] @@ -381,7 +387,7 @@ def fake_call_llm(messages): return MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) system_content = captured_messages[0]["content"] assert "[LANGUAGE INSTRUCTION]" in system_content @@ -389,7 +395,7 @@ def fake_call_llm(messages): def test_language_code_zh_injects_chinese_instruction(self): client = self._mock_client() - agent = ExperienceDistillAgent(client=client, language_code="zh") + agent = WorkflowDistillAgent(client=client, language_code="zh") captured_messages = [] @@ -398,7 +404,7 @@ def fake_call_llm(messages): return MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) system_content = captured_messages[0]["content"] assert "Simplified Chinese" in system_content @@ -406,7 +412,7 @@ def fake_call_llm(messages): def test_language_code_en_no_extra_instruction(self): client = self._mock_client() - agent = ExperienceDistillAgent(client=client, language_code="en") + agent = WorkflowDistillAgent(client=client, language_code="en") captured_messages = [] @@ -415,27 +421,45 @@ def fake_call_llm(messages): return MOCK_CONTEXT_RESPONSE with patch.object(agent, "_call_llm", side_effect=fake_call_llm): - agent.run(SAMPLE_EXPERIENCE_CONTEXT) + agent.run(SAMPLE_WORKFLOW_CONTEXT) system_content = captured_messages[0]["content"] assert "in English" in system_content assert "[LANGUAGE INSTRUCTION]" not in system_content -# ── _experience_filename ────────────────────────────────────────────────── +# ── _workflow_filename ────────────────────────────────────────────────── -class TestExperienceFilename: - def test_derives_from_workspace_name(self): - from data_formulator.routes.knowledge import _experience_filename - name = _experience_filename("Sales Analysis Pattern") +class TestWorkflowFilename: + def test_derives_from_title(self): + from data_formulator.routes.knowledge import _workflow_filename + name = _workflow_filename("Sales Analysis Pattern") assert name.endswith(".md") assert "sales-analysis-pattern" in name.lower() - def test_fallback_when_workspace_name_blank(self): - from data_formulator.routes.knowledge import _experience_filename - name = _experience_filename(" ") - assert name == "session-experience.md" + def test_fallback_when_title_blank(self): + from data_formulator.routes.knowledge import _workflow_filename + name = _workflow_filename(" ") + assert name == "session-workflow.md" + + def test_rejects_path_traversal(self): + from data_formulator.routes.knowledge import _workflow_filename + # An LLM-supplied name must never escape the workflows directory. + for evil in ("../../etc/passwd", "..\\..\\win", "/etc/shadow", "a/b/c"): + name = _workflow_filename(evil) + assert "/" not in name + assert "\\" not in name + assert ".." not in name + assert name.endswith(".md") + + def test_strips_reserved_and_control_chars(self): + from data_formulator.routes.knowledge import _workflow_filename + name = _workflow_filename('sales:report*?"<>|\x00 v1') + assert name.endswith(".md") + for ch in ':*?"<>|\x00': + assert ch not in name + assert name == "sales-report-v1.md" # ── API endpoint ────────────────────────────────────────────────────────── @@ -453,7 +477,7 @@ def app(self, tmp_path): _app.register_blueprint(knowledge_bp) register_error_handlers(_app) - (tmp_path / "knowledge" / "experiences").mkdir(parents=True) + (tmp_path / "knowledge" / "workflows").mkdir(parents=True) with patch("data_formulator.routes.knowledge.get_identity_id", return_value="test-user"), \ patch("data_formulator.routes.knowledge.get_user_home", return_value=tmp_path): @@ -464,14 +488,14 @@ def client(self, app): return app.test_client() def test_missing_context_returns_error(self, client): - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={"model": {"endpoint": "openai", "model": "gpt-4o"}}) data = resp.get_json() assert data["status"] == "error" def test_missing_model_returns_error(self, client): - resp = client.post("/api/knowledge/distill-experience", - json={"experience_context": SAMPLE_EXPERIENCE_CONTEXT}) + resp = client.post("/api/knowledge/distill-workflow", + json={"workflow_context": SAMPLE_WORKFLOW_CONTEXT}) data = resp.get_json() assert data["status"] == "error" @@ -483,9 +507,9 @@ def test_missing_events_returns_error(self, client): "workspace_name": "Demo", "threads": [], } - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={ - "experience_context": bad_context, + "workflow_context": bad_context, "model": {"endpoint": "openai", "model": "gpt-4o", "api_key": "test"}, }) data = resp.get_json() @@ -497,9 +521,9 @@ def test_missing_events_field_returns_error(self, client): "workspace_id": "ws-1", "workspace_name": "Demo", } # no 'threads' key - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={ - "experience_context": bad_context, + "workflow_context": bad_context, "model": {"endpoint": "openai", "model": "gpt-4o", "api_key": "test"}, }) data = resp.get_json() @@ -508,22 +532,22 @@ def test_missing_events_field_returns_error(self, client): def test_successful_distill(self, client, tmp_path): with patch("data_formulator.routes.agents.get_client") as mock_gc, \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ - patch("data_formulator.agents.agent_experience_distill.ExperienceDistillAgent.run", + patch("data_formulator.agents.agent_workflow_distill.WorkflowDistillAgent.run", return_value=MOCK_CONTEXT_RESPONSE): mock_gc.return_value = MagicMock() - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={ - "experience_context": SAMPLE_EXPERIENCE_CONTEXT, + "workflow_context": SAMPLE_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "model": "gpt-4o", "api_key": "test"}, }) data = resp.get_json() assert data["status"] == "success" - assert data["data"]["category"] == "experiences" + assert data["data"]["category"] == "workflows" assert data["data"]["path"].endswith(".md") # Verify file was written - exp_dir = tmp_path / "knowledge" / "experiences" + exp_dir = tmp_path / "knowledge" / "workflows" md_files = list(exp_dir.rglob("*.md")) assert len(md_files) >= 1 assert not (tmp_path / "agent-logs").exists() @@ -531,13 +555,13 @@ def test_successful_distill(self, client, tmp_path): def test_category_hint_creates_subdir(self, client, tmp_path): with patch("data_formulator.routes.agents.get_client") as mock_gc, \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ - patch("data_formulator.agents.agent_experience_distill.ExperienceDistillAgent.run", + patch("data_formulator.agents.agent_workflow_distill.WorkflowDistillAgent.run", return_value=MOCK_CONTEXT_RESPONSE): mock_gc.return_value = MagicMock() - resp = client.post("/api/knowledge/distill-experience", + resp = client.post("/api/knowledge/distill-workflow", json={ - "experience_context": SAMPLE_EXPERIENCE_CONTEXT, + "workflow_context": SAMPLE_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "model": "gpt-4o", "api_key": "test"}, "category_hint": "sales", }) diff --git a/tests/backend/knowledge/test_knowledge_store.py b/tests/backend/knowledge/test_knowledge_store.py index 2444195b..f69ce37c 100644 --- a/tests/backend/knowledge/test_knowledge_store.py +++ b/tests/backend/knowledge/test_knowledge_store.py @@ -5,11 +5,11 @@ Covers: - list_all, read, write, delete for each category -- path depth constraints (rules=flat, experiences=1 sub-dir) +- path depth constraints (rules=flat, workflows=1 sub-dir) - .md extension enforcement - ConfinedDir traversal rejection - front matter parsing and graceful degradation -- search: title, tags, filename, body matching + ranking + limit +- search: title, filename, body matching + ranking + limit - search skips alwaysApply rules (they are injected via system prompt) - tokenization: English stopwords, CJK/ASCII mixed splitting - scoring: partial token match, source discount, table_names boost @@ -73,21 +73,20 @@ def test_lists_rules(self, store, tmp_path): items = store.list_all("rules") assert len(items) == 1 assert items[0]["title"] == "ROI Calculation" - assert items[0]["tags"] == ["finance", "computation"] assert items[0]["path"] == "roi.md" assert items[0]["source"] == "manual" - def test_lists_experiences_in_subdirs(self, store, tmp_path): - exp_dir = tmp_path / "knowledge" / "experiences" / "cleaning" + def test_lists_workflows_in_subdirs(self, store, tmp_path): + exp_dir = tmp_path / "knowledge" / "workflows" / "cleaning" exp_dir.mkdir(parents=True) (exp_dir / "missing.md").write_text(SAMPLE_MD_SKILL, encoding="utf-8") - items = store.list_all("experiences") + items = store.list_all("workflows") assert len(items) == 1 assert items[0]["path"] == "cleaning/missing.md" def test_empty_category_returns_empty(self, store): - items = store.list_all("experiences") + items = store.list_all("workflows") assert items == [] def test_front_matter_title_fallback_to_stem(self, store, tmp_path): @@ -139,9 +138,9 @@ def test_preserves_existing_front_matter(self, store): content = store.read("rules", "fm.md") assert "title: ROI Calculation" in content - def test_writes_experiences_in_subdir(self, store, tmp_path): - store.write("experiences", "cleaning/handle-missing.md", SAMPLE_MD_SKILL) - assert (tmp_path / "knowledge" / "experiences" / "cleaning" / "handle-missing.md").exists() + def test_writes_workflows_in_subdir(self, store, tmp_path): + store.write("workflows", "cleaning/handle-missing.md", SAMPLE_MD_SKILL) + assert (tmp_path / "knowledge" / "workflows" / "cleaning" / "handle-missing.md").exists() # ── CRUD: delete ────────────────────────────────────────────────────────── @@ -169,12 +168,12 @@ def test_rules_subdir_rejected(self): with pytest.raises(ValueError, match="sub-directories"): KnowledgeStore.validate_path("rules", "sub/file.md") - def test_experiences_one_subdir_ok(self): - KnowledgeStore.validate_path("experiences", "cat/file.md") + def test_workflows_one_subdir_ok(self): + KnowledgeStore.validate_path("workflows", "cat/file.md") - def test_experiences_two_subdirs_rejected(self): + def test_workflows_two_subdirs_rejected(self): with pytest.raises(ValueError, match="one level"): - KnowledgeStore.validate_path("experiences", "cat/sub/file.md") + KnowledgeStore.validate_path("workflows", "cat/sub/file.md") def test_skills_rejected_as_invalid(self): with pytest.raises(ValueError, match="Invalid category"): @@ -228,7 +227,7 @@ def _setup_knowledge(self, store, tmp_path): rules_dir = tmp_path / "knowledge" / "rules" (rules_dir / "roi.md").write_text(SAMPLE_MD, encoding="utf-8") - exp_dir = tmp_path / "knowledge" / "experiences" / "cleaning" + exp_dir = tmp_path / "knowledge" / "workflows" / "cleaning" exp_dir.mkdir(parents=True) (exp_dir / "missing.md").write_text(SAMPLE_MD_SKILL, encoding="utf-8") @@ -237,11 +236,6 @@ def test_search_by_title(self, store): assert len(results) >= 1 assert results[0]["title"] == "Handle Missing Values" - def test_search_by_tags(self, store): - results = store.search("pandas") - assert len(results) >= 1 - assert results[0]["title"] == "Handle Missing Values" - def test_search_by_filename(self, store): results = store.search("missing") assert len(results) >= 1 @@ -269,7 +263,7 @@ def test_max_results_limit(self, store, tmp_path): assert len(results) <= 5 def test_search_filters_by_category(self, store): - results = store.search("ROI", categories=["experiences"]) + results = store.search("ROI", categories=["workflows"]) assert len(results) == 0 def test_search_skips_always_apply_rules(self, store, tmp_path): @@ -304,13 +298,12 @@ def test_partial_token_match_finds_results(self, store): assert results[0]["title"] == "Handle Missing Values" def test_table_names_boost(self, store, tmp_path): - """Entries tagged with a session table name get boosted.""" - exp_dir = tmp_path / "knowledge" / "experiences" / "analysis" + """Entries mentioning a session table name (title/body) get boosted.""" + exp_dir = tmp_path / "knowledge" / "workflows" / "analysis" exp_dir.mkdir(parents=True) (exp_dir / "sales-tip.md").write_text( - "---\ntitle: Sales Analysis Tips\n" - "tags: [sales_data, revenue]\nsource: manual\n---\n" - "When analysing sales, check for seasonality.\n", + "---\ntitle: Sales Analysis Tips\nsource: manual\n---\n" + "When analysing sales_data, check for seasonality.\n", encoding="utf-8", ) results = store.search("analysis tips", table_names=["sales_data"]) @@ -319,7 +312,7 @@ def test_table_names_boost(self, store, tmp_path): def test_non_manual_source_discounted(self, store, tmp_path): """Non-manual entries score lower than equivalent manual entries.""" - exp_dir = tmp_path / "knowledge" / "experiences" + exp_dir = tmp_path / "knowledge" / "workflows" (exp_dir / "auto-tip.md").write_text( "---\ntitle: Tip One\ntags: [tip]\nsource: distill\n---\nSome tip.\n", encoding="utf-8", @@ -328,7 +321,7 @@ def test_non_manual_source_discounted(self, store, tmp_path): "---\ntitle: Tip One\ntags: [tip]\nsource: manual\n---\nSome tip.\n", encoding="utf-8", ) - results = store.search("Tip One", categories=["experiences"]) + results = store.search("Tip One", categories=["workflows"]) assert len(results) == 2 assert results[0]["source"] == "manual" assert results[1]["source"] == "distill" @@ -472,7 +465,7 @@ def test_all_stopwords_returns_empty(self): class TestMatchScore: def test_single_token_title_hit(self): score = KnowledgeStore._match_score( - "ROI", "ROI Calculation", [], "roi", "", + "ROI", "ROI Calculation", "roi", "", ) assert score > 0 @@ -481,49 +474,49 @@ def test_partial_tokens_accumulate(self): score = KnowledgeStore._match_score( "quarterly sales trend", "Sales Trend Analysis", - [], "analysis", "", + "analysis", "", ) assert score > 0 def test_whole_string_bonus(self): full = KnowledgeStore._match_score( - "ROI", "ROI Calculation", [], "roi", "", + "ROI", "ROI Calculation", "roi", "", ) no_title = KnowledgeStore._match_score( - "ROI", "Something Else", [], "roi", "", + "ROI", "Something Else", "roi", "", ) assert full > no_title def test_source_discount(self): manual = KnowledgeStore._match_score( - "ROI", "ROI Guide", ["finance"], "roi", "", + "ROI", "ROI Guide", "roi", "", source="manual", ) auto = KnowledgeStore._match_score( - "ROI", "ROI Guide", ["finance"], "roi", "", + "ROI", "ROI Guide", "roi", "", source="distill", ) assert auto == pytest.approx(manual * 0.9) def test_table_names_boost(self): without = KnowledgeStore._match_score( - "analysis", "Analysis Tips", ["sales_data"], "tips", "", + "analysis", "Analysis Tips", "tips", "about sales_data", ) with_tn = KnowledgeStore._match_score( - "analysis", "Analysis Tips", ["sales_data"], "tips", "", + "analysis", "Analysis Tips", "tips", "about sales_data", table_names=["sales_data"], ) assert with_tn > without def test_no_match_returns_zero(self): score = KnowledgeStore._match_score( - "xyznonexistent", "ROI Calculation", ["finance"], "roi", "body text", + "xyznonexistent", "ROI Calculation", "roi", "body text", ) assert score == 0 def test_cjk_mixed_query_matches(self): """Chinese+English query should match via extracted ASCII tokens.""" score = KnowledgeStore._match_score( - "帮我分析ROI", "ROI Calculation", ["finance"], "roi", "", + "帮我分析ROI", "ROI Calculation", "roi", "", ) assert score > 0 diff --git a/tests/backend/routes/test_knowledge_routes.py b/tests/backend/routes/test_knowledge_routes.py index ddc2b7ab..f5ac69ff 100644 --- a/tests/backend/routes/test_knowledge_routes.py +++ b/tests/backend/routes/test_knowledge_routes.py @@ -167,7 +167,7 @@ def test_delete_nonexistent(self, client): class TestKnowledgeSearch: def test_search_returns_results(self, client, tmp_path): - exp_dir = tmp_path / "knowledge" / "experiences" / "finance" + exp_dir = tmp_path / "knowledge" / "workflows" / "finance" exp_dir.mkdir(parents=True, exist_ok=True) (exp_dir / "roi.md").write_text(SAMPLE_MD, encoding="utf-8") @@ -191,7 +191,7 @@ def test_search_invalid_category(self, client): assert data["status"] == "error" def test_search_filters_by_category(self, client, tmp_path): - exp_dir = tmp_path / "knowledge" / "experiences" / "finance" + exp_dir = tmp_path / "knowledge" / "workflows" / "finance" exp_dir.mkdir(parents=True, exist_ok=True) (exp_dir / "roi.md").write_text(SAMPLE_MD, encoding="utf-8") @@ -202,7 +202,7 @@ def test_search_filters_by_category(self, client, tmp_path): assert len(data["data"]["results"]) == 0 -SESSION_EXPERIENCE_CONTEXT = { +SESSION_WORKFLOW_CONTEXT = { "context_id": "ws-1", "workspace_id": "ws-1", "workspace_name": "Gasoline prices 2024", @@ -233,6 +233,7 @@ def test_search_filters_by_category(self, client, tmp_path): DISTILLED_MD = """\ --- subtitle: monthly sales aggregation +filename: monthly sales tags: [sales, time-series] created: 2026-05-06 updated: 2026-05-06 @@ -251,37 +252,37 @@ def test_search_filters_by_category(self, client, tmp_path): """ -class TestDistillExperience: - def test_distill_experience_from_context(self, client, tmp_path): +class TestDistillWorkflow: + def test_distill_workflow_from_context(self, client, tmp_path): with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", return_value=DISTILLED_MD, ) as run: - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "success" - assert data["data"]["category"] == "experiences" - assert (tmp_path / "knowledge" / "experiences" / data["data"]["path"]).exists() + assert data["data"]["category"] == "workflows" + assert (tmp_path / "knowledge" / "workflows" / data["data"]["path"]).exists() assert not (tmp_path / "agent-logs").exists() run.assert_called_once() - def test_distill_experience_llm_timeout_returns_structured_error(self, client): + def test_distill_workflow_llm_timeout_returns_structured_error(self, client): with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", side_effect=TimeoutError("request timed out"), ): - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) @@ -291,55 +292,60 @@ def test_distill_experience_llm_timeout_returns_structured_error(self, client): assert data["error"]["code"] == "LLM_TIMEOUT" assert data["error"]["retry"] is True - def test_distill_experience_missing_context(self, client): - resp = client.post("/api/knowledge/distill-experience", json={ + def test_distill_workflow_missing_context(self, client): + resp = client.post("/api/knowledge/distill-workflow", json={ "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "error" - def test_distill_experience_missing_threads(self, client): - bad_context = {k: v for k, v in SESSION_EXPERIENCE_CONTEXT.items() if k != "threads"} - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": bad_context, + def test_distill_workflow_missing_threads(self, client): + bad_context = {k: v for k, v in SESSION_WORKFLOW_CONTEXT.items() if k != "threads"} + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": bad_context, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "error" - def test_distill_experience_missing_workspace(self, client): - bad_context = {k: v for k, v in SESSION_EXPERIENCE_CONTEXT.items() + def test_distill_workflow_missing_workspace(self, client): + bad_context = {k: v for k, v in SESSION_WORKFLOW_CONTEXT.items() if k not in ("workspace_id", "workspace_name")} - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": bad_context, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": bad_context, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "error" - def test_distill_session_overrides_title_with_workspace_name(self, client, tmp_path): - """Session-scoped distillation composes 'Experience from : '.""" + def test_distill_session_uses_descriptive_title(self, client, tmp_path): + """Session-scoped distillation uses the agent subtitle as the title.""" with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", return_value=DISTILLED_MD, ): - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "success" path = data["data"]["path"] - # Filename is derived from the workspace name, not the LLM subtitle. - assert path == "gasoline-prices-2024.md" - saved = (tmp_path / "knowledge" / "experiences" / path).read_text(encoding="utf-8") - assert "title: 'Experience from Gasoline prices 2024: monthly sales aggregation'" in saved \ - or "title: \"Experience from Gasoline prices 2024: monthly sales aggregation\"" in saved \ - or "title: Experience from Gasoline prices 2024: monthly sales aggregation" in saved + # Filename is derived from the short agent-emitted `filename` hint, + # not the long descriptive title. + assert path == "monthly-sales.md" + saved = (tmp_path / "knowledge" / "workflows" / path).read_text(encoding="utf-8") + assert "title: monthly sales aggregation" in saved \ + or "title: 'monthly sales aggregation'" in saved \ + or "title: \"monthly sales aggregation\"" in saved + # No legacy "Workflow from :" prefix on the title. + assert "Workflow from" not in saved + # The filename hint is consumed, not persisted in the front matter. + assert "filename:" not in saved # Workspace stamps are present so the file can be looked up later. assert "source_workspace_id: ws-1" in saved assert "source_workspace_name: Gasoline prices 2024" in saved @@ -347,42 +353,46 @@ def test_distill_session_overrides_title_with_workspace_name(self, client, tmp_p assert "## Method" in saved def test_distill_session_upserts_existing_workspace_file(self, client, tmp_path): - """Re-distilling the same workspace overwrites the same file.""" + """Re-distilling the same workspace replaces the prior file.""" + second_md = DISTILLED_MD.replace( + "filename: monthly sales", + "filename: annual revenue", + ) with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", - return_value=DISTILLED_MD, + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", + side_effect=[DISTILLED_MD, second_md], ): - client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) - # Re-distill: workspace renamed, so the slug changes — old file - # should be removed in favour of the new one. - renamed = {**SESSION_EXPERIENCE_CONTEXT, "workspace_name": "Diesel 2024"} - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": renamed, + # Re-distill: the filename hint changes, so the slug changes — old + # file should be removed in favour of the new one (matched by + # source_workspace_id). + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "success" new_path = data["data"]["path"] - exp_dir = tmp_path / "knowledge" / "experiences" + exp_dir = tmp_path / "knowledge" / "workflows" # Stale slug deleted, new slug present. - assert not (exp_dir / "gasoline-prices-2024.md").exists() + assert not (exp_dir / "monthly-sales.md").exists() assert (exp_dir / new_path).exists() - assert new_path == "diesel-2024.md" + assert new_path == "annual-revenue.md" - def test_distill_session_skips_subtitle_double_prefix(self, client, tmp_path): - """Update-mode runs that re-emit a prefixed title don't double-prefix.""" - # Simulate a prior run where the LLM echoed an Experience-prefixed title + def test_distill_session_strips_legacy_title_prefix(self, client, tmp_path): + """Update-mode runs strip any legacy 'Workflow from :' prefix.""" + # Simulate a prior run where the LLM echoed a Workflow-prefixed title # without a subtitle. prior_md = ( "---\n" - "title: 'Experience from Gasoline prices 2024: prior insight'\n" + "title: 'Workflow from Gasoline prices 2024: prior insight'\n" "tags: [a]\n" "created: 2026-05-06\n" "updated: 2026-05-06\n" @@ -392,17 +402,18 @@ def test_distill_session_skips_subtitle_double_prefix(self, client, tmp_path): with patch("data_formulator.routes.agents.get_client", return_value=object()), \ patch("data_formulator.routes.agents.get_language_instruction", return_value=""), \ patch( - "data_formulator.agents.agent_experience_distill." - "ExperienceDistillAgent.run", + "data_formulator.agents.agent_workflow_distill." + "WorkflowDistillAgent.run", return_value=prior_md, ): - resp = client.post("/api/knowledge/distill-experience", json={ - "experience_context": SESSION_EXPERIENCE_CONTEXT, + resp = client.post("/api/knowledge/distill-workflow", json={ + "workflow_context": SESSION_WORKFLOW_CONTEXT, "model": {"endpoint": "openai", "key": "x", "model": "gpt"}, }) data = resp.get_json() assert data["status"] == "success" - saved = (tmp_path / "knowledge" / "experiences" / data["data"]["path"]).read_text(encoding="utf-8") - # The "Experience from ..." prefix is stripped before re-prefixing. - assert saved.count("Experience from") == 1 + saved = (tmp_path / "knowledge" / "workflows" / data["data"]["path"]).read_text(encoding="utf-8") + # The legacy "Workflow from ..." prefix is fully stripped. + assert "Workflow from" not in saved + assert "prior insight" in saved From 1571113ac7b74ce1a0c32b434a9576d6fdce2b40 Mon Sep 17 00:00:00 2001 From: y-agent-ai Date: Sat, 30 May 2026 18:22:30 +0800 Subject: [PATCH 07/36] refactor(loading): Refactor AnvilLoader and add custom parameter support 1. Add custom property support for height , label , and sx to AnvilLoader 2. Replace globally hardcoded loading text with customizable label parameter 3. Optimize loading overlay styles with new frosted glass background effect 4. Unify loading state display in App.tsx and VisualizationView --- src/app/App.tsx | 2 +- src/components/AnvilLoader.tsx | 46 +++++++++++++++++++++------------ src/views/VisualizationView.tsx | 10 ++++--- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/app/App.tsx b/src/app/App.tsx index 17898f0c..22d76510 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -1253,7 +1253,7 @@ export const AppFC: FC = function AppFC(appProps) { {configLoaded && authChecked ? ( ) : ( - + )} {migrationBrowserId && ( ; +} + +export function AnvilLoader({ height = '100vh', label, sx }: AnvilLoaderProps) { return ( - - loading data formulator... - + {label !== undefined && ( + + {label} + + )} ); } diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index 7b6d18b4..585eba79 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -15,7 +15,6 @@ import { ListItemIcon, ListItemText, MenuItem, - LinearProgress, Card, ListSubheader, Menu, @@ -37,6 +36,7 @@ import _ from 'lodash'; import { borderColor, transition } from '../app/tokens'; import { WritingIndicator } from '../components/FunComponents'; +import { AnvilLoader } from '../components/AnvilLoader'; import ButtonGroup from '@mui/material/ButtonGroup'; @@ -1099,10 +1099,12 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { return {synthesisRunning ? - + : ''} {chartUnavailable ? "" : chartResizer} {content} From b9abafb163d59c8c4165075d8f573345b4d70235 Mon Sep 17 00:00:00 2001 From: cat0825 <1759138827@qq.com> Date: Sun, 31 May 2026 14:02:02 +0800 Subject: [PATCH 08/36] test: keep zh locale keys aligned --- src/i18n/locales/zh/dataLoading.json | 1 + tests/frontend/unit/app/i18nLocales.test.ts | 30 +++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 tests/frontend/unit/app/i18nLocales.test.ts diff --git a/src/i18n/locales/zh/dataLoading.json b/src/i18n/locales/zh/dataLoading.json index 4eab6fcf..6ffe595d 100644 --- a/src/i18n/locales/zh/dataLoading.json +++ b/src/i18n/locales/zh/dataLoading.json @@ -39,6 +39,7 @@ "rowLimit": "行数限制", "loadSelected": "加载选中的表", "loadedCount": "✓ 已加载 {{count}} 张表", + "loadedCount_plural": "✓ 已加载 {{count}} 张表", "preview": "预览", "hidePreview": "收起", "previewing": "正在预览...", diff --git a/tests/frontend/unit/app/i18nLocales.test.ts b/tests/frontend/unit/app/i18nLocales.test.ts new file mode 100644 index 00000000..dd6c9933 --- /dev/null +++ b/tests/frontend/unit/app/i18nLocales.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; + +import en from "../../../../src/i18n/locales/en"; +import zh from "../../../../src/i18n/locales/zh"; + +type TranslationValue = string | Record; +type TranslationMap = Record; + +function collectKeys(value: TranslationMap, prefix = ""): Set { + const keys = new Set(); + + for (const [key, child] of Object.entries(value)) { + const nextPrefix = prefix ? `${prefix}.${key}` : key; + if (typeof child === "string") { + keys.add(nextPrefix); + } else { + for (const childKey of collectKeys(child, nextPrefix)) { + keys.add(childKey); + } + } + } + + return keys; +} + +describe("i18n locale bundles", () => { + it("keeps Simplified Chinese translation keys aligned with English", () => { + expect(collectKeys(zh)).toEqual(collectKeys(en)); + }); +}); From 748a30ce45f8e01b95388aced0b2e27106d70b69 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Sun, 31 May 2026 12:26:36 -0700 Subject: [PATCH 09/36] bug fix and clean up --- .../agents/agent_workflow_distill.py | 190 ++++++++++-------- .../data_loader/sample_datasets_loader.py | 13 +- src/app/dfSlice.tsx | 10 + src/components/LoadPlanCard.tsx | 51 ++++- src/i18n/locales/en/common.json | 4 +- src/i18n/locales/en/dataLoading.json | 2 + src/i18n/locales/zh/dataLoading.json | 2 + src/views/DataLoadingChat.tsx | 24 ++- src/views/DataSourceSidebar.tsx | 28 ++- src/views/EncodingShelfCard.tsx | 60 ++++-- src/views/KnowledgePanel.tsx | 6 +- src/views/VisualizationView.tsx | 15 +- 12 files changed, 263 insertions(+), 142 deletions(-) diff --git a/py-src/data_formulator/agents/agent_workflow_distill.py b/py-src/data_formulator/agents/agent_workflow_distill.py index 3f3d9c6d..0d86aa78 100644 --- a/py-src/data_formulator/agents/agent_workflow_distill.py +++ b/py-src/data_formulator/agents/agent_workflow_distill.py @@ -30,9 +30,28 @@ SYSTEM_PROMPT = """\ You are a workflow distiller. Given the chronological events of a data -analysis session plus an optional user instruction, extract a short, -**replayable workflow** that captures *what the user wanted and got* — so -the same analysis can be reproduced later on a similarly-shaped dataset. +analysis session plus an optional user distillation instruction, extract a **replayable +workflow** that captures *what the user wanted and got* — and write it at +TWO levels so it can be reused in two different situations: + +1. An **Abstract workflow** — dataset-independent. The underlying analytical + pattern, stripped of this dataset's subject matter: the sequence of + questions, computations, and chart kinds, phrased in domain-neutral terms. + Following it on a *different and possibly very differently-shaped* dataset + should walk the same process and arrive at structurally similar + visualizations. +2. A **Concrete workflow** — for *similar* data (same shape, only minor + differences — a different period, region, or filter). It names the real + fields, aggregations, filters, and chart encodings used here, so the + analysis can be replayed closely with minimal thought. + +Both describe the SAME analysis at different distances. They should be +consistent, but they do NOT need an exact 1:1 step mapping — let each be as +long as it needs (typically 3-7 steps each). + +Where the analysis hinges on a few choices a user might change on replay (a +period, a filter, a top-N), surface them as named **parameters** with +`{{token}}` placeholders in the steps — see the `## Parameters` section below. The session contains one or more threads (separate analysis branches in the same session) each rendered under a `### Thread N` header. When @@ -47,38 +66,20 @@ (followed by columns, row count, sample, and code). - `create_chart` — a chart emitted on a table (mark + encoding summary). -Your job is to recover the **ordered list of requests** the user actually -wanted, and the outputs (tables/charts) they ended up keeping. Beyond the -concrete steps, also distill the analysis at TWO levels of abstraction so -it can be reused later: -- **Adapting to similar data** (concrete) — how to rerun essentially the - same analysis on a near-identical dataset, e.g. the business report for - a different month, region, or product line. Same shape and intent, only - the specific inputs/filters change. -- **Generalizing to other data** (abstract, dataset-agnostic) — the - underlying analytical pattern, independent of this domain: the kinds of - questions, computations, and charts involved, phrased so they transfer - to a different domain or a differently-shaped dataset. - CRITICAL extraction rules — keep only what the user wanted and got: -- Each step = one user request, written in plain language. Say BOTH the - question being explored AND what was produced to answer it — including - the chart that was created and the key fields it uses (e.g. "Ask how - sales trend over time, and plot monthly total sales as a line chart"; - "Compare regions by breaking revenue down per region as a sorted bar - chart"). Order them as the analysis progressed. +- Recover the ORDERED list of requests the user actually wanted, and the + outputs (tables/charts) they kept. Each step states BOTH the question + explored AND what was produced to answer it — including the chart and the + key fields it uses. - DROP corrective back-and-forth. If the user changed their mind ("no, it should be…", "actually use median instead"), keep ONLY the final resolved intent — not the wrong first attempt or the correction. - DROP abandoned work. If a chart or table was created and then deleted or never kept, leave it out entirely. - DROP mechanics. Do NOT include error-repair loops, dtype fixes, tool - call noise, or low-level code. Describe intent, not implementation. -- Do NOT lean on code or exact column names unless a name is essential to - the request's meaning. Keep steps dataset-agnostic where possible so - they replay on a new slice of similar data. -- Capture genuine gotchas separately as short notes (advisory warnings to - carry forward), NOT as steps to re-perform. + call noise, or low-level code dumps. Describe intent, not implementation. +- Capture genuine gotchas as short Notes (advisory warnings to carry + forward), NOT as steps to re-perform. If a user instruction is provided, let it steer what to keep or emphasise. @@ -86,8 +87,8 @@ ``` --- -subtitle: -filename: +subtitle: +filename: created: updated: source: distill @@ -96,74 +97,85 @@ ## Goal - -## Steps -1. -2. +what it produces. This is where the dataset-grounded explanation belongs — +you MAY name the real subject here (e.g. "originally distilled from a +monthly gasoline-price session").> + +## Parameters + +- `{{period}}` — the time range analysed; used here: 2024; on replay: ask. +- `{{top_n}}` — how many top categories to keep; used here: 10; on replay: keep. +- `{{region}}` — geographic filter applied; used here: National; on replay: ask. + +## Abstract workflow + +1. +2. 3. <…> -## Adapting to similar data - - -## Generalizing to other data - +## Concrete workflow + +1. +2. <…> ## Notes +analysis on new data — e.g. "sort by time before computing period-over-period +change". Omit this section entirely if there is nothing worth warning about.> ``` Rules: -- Subtitle must DESCRIBE what the workflow is about in PLAIN LANGUAGE that - a non-expert can fully understand at a glance, so they can decide - whether to replay it on new data. Favor clarity over brevity: it can be - a full sentence (up to ~25 words) if that makes the analysis genuinely - understandable. Write it like you would explain the analysis to a - colleague in one breath, covering the subject and the main thing you do - with it. The hosting application uses this subtitle directly as the - workflow's display title, so make it self-contained and do NOT prefix it - with the session name. - - Start with a concrete action verb (Plot, Compare, Break down, Rank, - Track, Summarize, Find…). - - Name the real-world subject in everyday words (sales, revenue, - customers, events), NOT the internal mechanics or derived-column - names you happened to create. - - AVOID abstract or technical jargon and invented noun-phrases - ("deltas", "composition", "window", "distribution shift"). If a - technique matters, phrase it plainly ("change from one period to the - next" instead of "deltas"). - Good: "Plot monthly sales over time and compare each year against the - previous one to spot volatile periods". - "Break revenue down by region and show how each region - contributes to the total as a stacked area chart". - "Track how many events happen in each time window and what kinds - of events make up each window". - Bad: "Time series analysis". "Data workflow". "Chart exploration". - "Event window deltas with composition". "Distribution shift inspection". +- The subtitle is the workflow's display TITLE. Make it ABSTRACT and + library-friendly: name the *kind of analysis* — a technique plus a GENERIC + subject (KPI, metric, category, event, cohort) — so someone browsing the + workflow library can tell whether this is the KIND of analysis they want to + reuse. Do NOT pin it to this dataset's specific subject, period, or column + names, and do NOT prefix it with the session name. + - Pair a real technique with a generic subject; avoid bare category words. + Good: "Year-over-year KPI volatility analysis". + "Category contribution-to-total breakdown". + "Time-windowed event composition analysis". + Bad: "Plot monthly gasoline prices in 2024 and compare each year". (too specific) + "Time series analysis". "Data workflow". "Chart exploration". (too vague) + The dataset-grounded, full-sentence explanation goes in `## Goal`, NOT the title. - Filename must be a SHORT (2-5 word) lowercase name for the file — just - the core subject and action, e.g. "monthly sales trend", "region revenue - breakdown". No dates, no file extension, no session name. It is only - used to name the file on disk; the descriptive subtitle is what users see. -- Steps must be ordered and reproducible. Each step should make clear the - question being explored and the chart/output produced to answer it. -- "Adapting to similar data" stays close to this analysis (same domain, - same shape) — only the concrete inputs change. "Generalizing to other - data" must be domain-neutral: strip out this dataset's subject matter and - describe only the transferable analytical pattern (question types, - computations, chart kinds). Do NOT just repeat the steps in either - section; add genuine reuse guidance. Keep each section brief. -- Be as long as the analysis needs — do not omit meaningful steps, - questions, or charts just to stay short. Stay focused, but completeness - matters more than brevity. + the technique/subject, e.g. "kpi volatility analysis", "region revenue + breakdown". No dates, no file extension, no session name. It only names the + file on disk; the subtitle is what users see. +- Abstract workflow must be domain-neutral — strip this dataset's subject + matter and column names; describe only the transferable pattern (question + types, computations, chart kinds). Concrete workflow must be runnable on a + near-identical dataset: real field names, the aggregation, the filter to + vary, the chart mark + key encodings. Do NOT have the two sections merely + repeat each other — each adds its own grain of reuse guidance. +- Parameters are optional and a judgment call: surface only the FEW knobs + that materially change the outcome and that a user would revisit on replay + (often 0-4). When in doubt, leave the value inline — a spurious `{{token}}` + is worse than none. Knobs may be run-specific (period, region, top-N — + usually `ask`) or dataset-specific (a domain value/column — usually `keep`, + and may be skipped in the Abstract workflow). Every `{{token}}` in the steps + must be listed in `## Parameters` and vice versa. +- Steps in both sections must be ordered and reproducible. +- Be as long as the analysis needs — do not omit meaningful steps, questions, + or charts just to stay short. Stay focused, but completeness matters more + than brevity. - No raw data, PII, secrets, or specific values unless essential to a request. - Write the subtitle, headings, and body in {output_language}. YAML front-matter keys stay in English. diff --git a/py-src/data_formulator/data_loader/sample_datasets_loader.py b/py-src/data_formulator/data_loader/sample_datasets_loader.py index 6c3267cf..a84b8302 100644 --- a/py-src/data_formulator/data_loader/sample_datasets_loader.py +++ b/py-src/data_formulator/data_loader/sample_datasets_loader.py @@ -25,7 +25,10 @@ import pyarrow as pa from data_formulator.data_loader.external_data_loader import ExternalDataLoader -from data_formulator.datalake.parquet_utils import df_to_safe_records +from data_formulator.datalake.parquet_utils import ( + df_to_safe_records, + sanitize_dataframe_for_arrow, +) logger = logging.getLogger(__name__) @@ -231,7 +234,13 @@ def fetch_data_as_arrow( logger.info("Returning %d / %d rows from sample dataset: %s", len(df), self._last_total_rows, source_table) - return pa.Table.from_pandas(df, preserve_index=False) + # Public sample JSON/CSV files frequently contain mixed-type object + # columns (e.g. movies.json's ``Title`` holds both strings and + # numeric values), which makes ``pa.Table.from_pandas`` raise + # ArrowTypeError. Coerce such columns to a consistent type first. + return pa.Table.from_pandas( + sanitize_dataframe_for_arrow(df), preserve_index=False + ) # ------------------------------------------------------------------ # Internal: cached full-dataset fetch diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 89b075ab..2ceb55aa 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -245,6 +245,9 @@ export interface DataFormulatorState { /** Whether the data source sidebar is expanded (true) or collapsed to rail (false) */ dataSourceSidebarOpen: boolean; + /** Which data source sidebar tab is active. Persisted so it survives session refresh. */ + dataSourceSidebarTab: 'sources' | 'sessions' | 'knowledge'; + /** * One-shot signal asking the sidebar to focus a specific connector * (open the sidebar, switch to sources tab, expand + scroll-into-view @@ -322,6 +325,8 @@ const initialState: DataFormulatorState = { dataSourceSidebarOpen: false, + dataSourceSidebarTab: 'sources', + focusedConnectorId: undefined, } @@ -762,12 +767,16 @@ export const dataFormulatorSlice = createSlice({ viewMode: state.viewMode, dataLoaderConnectParams: state.dataLoaderConnectParams, dataSourceSidebarOpen: state.dataSourceSidebarOpen, + dataSourceSidebarTab: state.dataSourceSidebarTab, activeWorkspace: action.payload, }; }, setDataSourceSidebarOpen: (state, action: PayloadAction) => { state.dataSourceSidebarOpen = action.payload; }, + setDataSourceSidebarTab: (state, action: PayloadAction<'sources' | 'sessions' | 'knowledge'>) => { + state.dataSourceSidebarTab = action.payload; + }, /** * Ask the data-source sidebar to focus a specific connector. * Opens the sidebar (if collapsed) and stores the target id; the @@ -870,6 +879,7 @@ export const dataFormulatorSlice = createSlice({ activeWorkspace: saved.activeWorkspace ?? state.activeWorkspace ?? null, dataSourceSidebarOpen: state.dataSourceSidebarOpen, + dataSourceSidebarTab: state.dataSourceSidebarTab, // Reset display-rows tick so dependent components re-fetch. displayRowsTick: 0, diff --git a/src/components/LoadPlanCard.tsx b/src/components/LoadPlanCard.tsx index 9b60effd..b91e54ab 100644 --- a/src/components/LoadPlanCard.tsx +++ b/src/components/LoadPlanCard.tsx @@ -16,8 +16,14 @@ import type { LoadPlan, LoadPlanCandidate, PendingTableLoad } from './ComponentT interface LoadPlanCardProps { plan: LoadPlan; - onConfirm: (selected: LoadPlanCandidate[]) => void; + onConfirm: (selected: LoadPlanCandidate[], opts?: { newWorkspace?: boolean }) => void; confirmed?: boolean; + /** When true, a workspace with existing data is already open, so the + * destination of the load is ambiguous. We then offer two explicit + * actions: add to the current workspace, or load into a fresh one. + * When false (empty/new workspace), a single "Load selected" button + * loads directly with no ambiguity. */ + canLoadInNewWorkspace?: boolean; } // Plans this small auto-expand each row's preview on first render so the @@ -48,7 +54,7 @@ const formatFilterValue = (value: any) => { return Array.isArray(value) ? value.join(', ') : String(value); }; -export const LoadPlanCard: React.FC = ({ plan, onConfirm, confirmed }) => { +export const LoadPlanCard: React.FC = ({ plan, onConfirm, confirmed, canLoadInNewWorkspace }) => { const theme = useTheme(); const { t } = useTranslation(); const [selection, setSelection] = useState>( @@ -143,12 +149,12 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con fetchPreview(candidate, idx); }; - const handleConfirm = async () => { + const handleConfirm = async (newWorkspace = false) => { const selected = plan.candidates.filter((c, i) => selection[i] && !c.resolutionError); if (selected.length === 0) return; setLoading(true); try { - await onConfirm(selected); + await onConfirm(selected, { newWorkspace }); } finally { setLoading(false); } @@ -257,12 +263,47 @@ export const LoadPlanCard: React.FC = ({ plan, onConfirm, con defaultValue: '✓ Loaded', })} + ) : canLoadInNewWorkspace ? ( + // A workspace with data is already open — make the load + // destination explicit rather than silently appending. + <> + + + ) : ( - {activeWorkspace && ( - - - - )} + + + {activeWorkspace && ( + + )} )} diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index f6c4a79f..98217775 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -6,11 +6,13 @@ import ReactDOM from 'react-dom'; import _ from 'lodash'; -import { Typography, Box, Link, Breadcrumbs, useTheme, Fade, IconButton, Tooltip } from '@mui/material'; +import { Typography, Box, Link, Breadcrumbs, useTheme, Fade, IconButton, Tooltip, TextField, InputAdornment, Chip } from '@mui/material'; import { alpha } from '@mui/material/styles'; import { useTranslation } from 'react-i18next'; import OpenInFullIcon from '@mui/icons-material/OpenInFull'; import CloseFullscreenIcon from '@mui/icons-material/CloseFullscreen'; +import SearchIcon from '@mui/icons-material/Search'; +import ClearIcon from '@mui/icons-material/Clear'; import '../scss/DataView.scss'; @@ -27,12 +29,33 @@ export interface FreeDataViewProps { // full-canvas overlay. Used wherever the grid is shown inline (under a // chart, or as the focused-table preview). maximizable?: boolean; + // Explicit table to render. When omitted the view derives the table from + // `focusedId` (single-focus behavior). Set by the multi-table canvas to + // render each highlighted table in a stack. + tableId?: string; + // Render the Numbers-style title/metadata + search header inside the grid. + // Used by the focused-table canvas. + showHeaderBar?: boolean; + // Hide the in-grid footer widget; the focused-table canvas surfaces those + // actions (row count / random / download) in its bottom toolbar instead. + hideFooter?: boolean; + // Controlled random-rows trigger + virtual-state report, forwarded to the + // grid so the external toolbar can drive/observe them. + randomizeToken?: number; + onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean }) => void; } -export const FreeDataViewFC: FC = function DataView({ maximizable }) { +export const FreeDataViewFC: FC = function DataView({ maximizable, tableId, showHeaderBar, hideFooter, randomizeToken, onStateReport }) { const { t } = useTranslation(); const [maximized, setMaximized] = React.useState(false); + // Draft holds the live input; query is the committed term the grid/server + // actually searches. Search runs only when the user submits (Enter or the + // search icon), never on every keystroke. + const [searchDraft, setSearchDraft] = React.useState(''); + const [searchQuery, setSearchQuery] = React.useState(''); + const submitSearch = React.useCallback(() => setSearchQuery(searchDraft.trim()), [searchDraft]); + const clearSearch = React.useCallback(() => { setSearchDraft(''); setSearchQuery(''); }, []); const dispatch = useDispatch(); @@ -42,13 +65,20 @@ export const FreeDataViewFC: FC = function DataView({ maximiz // Derive the table to display based on focusedId const focusedTableId = useMemo(() => { + if (tableId) return tableId; if (!focusedId) return undefined; if (focusedId.type === 'table') return focusedId.tableId; if (focusedId.type !== 'chart') return undefined; const chartId = focusedId.chartId; const chart = allCharts.find(c => c.id === chartId); return chart?.tableRef; - }, [focusedId, allCharts]); + }, [focusedId, allCharts, tableId]); + + // The search term is temporary/per-table: clear it when switching tables. + React.useEffect(() => { + setSearchDraft(''); + setSearchQuery(''); + }, [focusedTableId]); // Only subscribe to the focused table and table count — NOT the full tables array. // This prevents re-rendering the entire data grid when the agent adds unrelated tables. @@ -132,14 +162,114 @@ export const FreeDataViewFC: FC = function DataView({ maximiz columnDefs={colDefs} rowCount={targetTable?.virtual?.rowCount || targetTable?.rows.length || 0} virtual={targetTable?.virtual ? true : false} + searchText={searchQuery} + hideFooter={hideFooter} + randomizeToken={randomizeToken} + onStateReport={onStateReport} /> ); + // Numbers-style header rendered OUTSIDE the table card: table title + + // row/column metadata on the left, a quick search box on the right. + // Sort & filter remain in the column headers/kebab menus. + const headerRowCount = targetTable?.virtual?.rowCount || targetTable?.rows.length || 0; + const headerColCount = targetTable?.names.length || 0; + const headerBar = showHeaderBar ? ( + + + + + {targetTable?.displayId || targetTable?.id || 'table'} + + {searchQuery ? ( + } + label={`"${searchQuery}"`} + onDelete={clearSearch} + deleteIcon={} + sx={{ + height: 22, maxWidth: 220, flexShrink: 0, + borderRadius: '6px', + backgroundColor: (theme) => alpha(theme.palette.primary.main, 0.08), + color: 'primary.main', + '& .MuiChip-label': { fontSize: 11.5, px: 0.75, overflow: 'hidden', textOverflow: 'ellipsis' }, + '& .MuiChip-icon': { color: 'primary.main', ml: 0.5 }, + '& .MuiChip-deleteIcon': { color: 'primary.main', '&:hover': { color: 'primary.dark' } }, + }} + /> + ) : null} + + + {t('dataGrid.rowCount', { count: headerRowCount })} · {headerColCount} {headerColCount === 1 ? 'column' : 'columns'} + + + + setSearchDraft(e.target.value)} + autoComplete="off" + inputProps={{ autoComplete: 'off', autoCorrect: 'off', autoCapitalize: 'off', spellCheck: false }} + onKeyDown={(e) => { + if (e.key === 'Enter') { e.preventDefault(); submitSearch(); } + else if (e.key === 'Escape' && searchDraft) { e.preventDefault(); clearSearch(); } + }} + placeholder={t('dataGrid.searchPlaceholder', { defaultValue: 'Search table…' })} + InputProps={{ + startAdornment: ( + + + + + + + + ), + endAdornment: (searchDraft || searchQuery) ? ( + + + + + + ) : undefined, + }} + sx={{ + width: 220, + '& .MuiOutlinedInput-root': { borderRadius: '8px', fontSize: 12, backgroundColor: 'background.paper' }, + '& .MuiOutlinedInput-input': { py: '6px' }, + }} + /> + + ) : null; + + // Wrap any table content with the header above it (when enabled), so the + // title sits outside the card frame. + const withHeader = (content: React.ReactNode) => showHeaderBar ? ( + + {headerBar} + {content} + + ) : content; + + // Focused-table canvas: the table already fills the page, so there's no + // maximize button — just a flat bordered card with the title above it. + if (showHeaderBar) { + return withHeader( + + {grid} + + ); + } + if (!maximizable) { - return grid; + return withHeader(grid); } const toggleButton = ( @@ -160,12 +290,16 @@ export const FreeDataViewFC: FC = function DataView({ maximiz // The toggle button sits just outside the table to the right (a slim panel), // so it never overlaps the column headers and the card keeps its original look. // In maximized mode the surrounding overlay already provides the card frame. + // When the header bar is shown (focused-table canvas) the card stays flat — + // no hover glow/elevation — since the title lives outside the card. const cardSx = maximized ? { overflow: 'hidden' } : { overflow: 'hidden', borderRadius: '8px', border: `1px solid ${borderColor.divider}`, - transition: 'box-shadow 0.2s ease', - '&:hover': { boxShadow: '0 0 8px rgba(25, 118, 210, 0.25)' }, + ...(showHeaderBar ? {} : { + transition: 'box-shadow 0.2s ease', + '&:hover': { boxShadow: '0 0 8px rgba(25, 118, 210, 0.25)' }, + }), }; const framed = ( @@ -209,5 +343,5 @@ export const FreeDataViewFC: FC = function DataView({ maximiz ); } - return framed; + return withHeader(framed); } \ No newline at end of file diff --git a/src/views/SelectableDataGrid.tsx b/src/views/SelectableDataGrid.tsx index 4869ef77..222a883e 100644 --- a/src/views/SelectableDataGrid.tsx +++ b/src/views/SelectableDataGrid.tsx @@ -58,8 +58,20 @@ interface SelectableDataGridProps { rowCount: number; virtual: boolean; columnDefs: ColumnDef[]; + // Controlled quick-search text (owned by the focused-table canvas). Filters + // the loaded rows client-side across all data columns. + searchText?: string; + // Hide the in-grid footer widget (row count / random / download). The + // focused-table canvas surfaces these actions in its bottom toolbar. + hideFooter?: boolean; + // Bumping this number triggers a "random rows" refetch (virtual tables). + randomizeToken?: number; + // Report virtual-pagination state up so an external toolbar can render the + // loaded/total count and enable the random-rows action. + onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean }) => void; } + function descendingComparator(a: T, b: T, orderBy: keyof T) { if (b[orderBy] < a[orderBy]) { return -1; @@ -322,7 +334,7 @@ const VirtuosoTableBody = React.forwardRef((props, ref) const PAGE_SIZE = 500; export const SelectableDataGrid: React.FC = React.memo(({ - tableId, rows, tableName, columnDefs, rowCount, virtual }) => { + tableId, rows, tableName, columnDefs, rowCount, virtual, searchText, hideFooter, randomizeToken, onStateReport }) => { const { t } = useTranslation(); const [orderBy, setOrderBy] = React.useState(undefined); @@ -360,6 +372,9 @@ export const SelectableDataGrid: React.FC = React.memo( const [isLoadingMore, setIsLoadingMore] = React.useState(false); const [isDownloading, setIsDownloading] = React.useState(false); const [hasMore, setHasMore] = React.useState(virtual ? rows.length < rowCount : false); + // Filtered total reported by the server (reflects active search/filters for + // virtual tables). Falls back to the unfiltered prop count. + const [serverRowCount, setServerRowCount] = React.useState(rowCount); const fetchIdRef = React.useRef(0); React.useEffect(() => { @@ -377,6 +392,29 @@ export const SelectableDataGrid: React.FC = React.memo( } }, [rows, order, orderBy]) + // Report virtual-pagination state up to an external toolbar (focused-table + // canvas) so it can render the loaded/total count and the random-rows dice. + React.useEffect(() => { + const effectiveCount = virtual ? serverRowCount : rowCount; + onStateReport?.({ + loadedCount: rowsToDisplay.length, + rowCount: effectiveCount, + virtual, + canRandomize: virtual && effectiveCount > 10000, + }); + }, [rowsToDisplay.length, rowCount, serverRowCount, virtual, onStateReport]); + + // Bumping `randomizeToken` (from the external toolbar's dice) resets sort + // and refetches a fresh random page for virtual tables. + const randomizeMountRef = React.useRef(true); + React.useEffect(() => { + if (randomizeMountRef.current) { randomizeMountRef.current = false; return; } + if (!virtual) return; + setOrderBy(undefined); + setOrder('asc'); + fetchVirtualDataRef.current?.([], 'asc'); + }, [randomizeToken]); + // Only the Table component depends on columnDefs (for colgroup); memoize it // so react-virtuoso keeps a stable reference when columns haven't changed. const columnIds = columnDefs.map(c => c.id).join(','); @@ -446,6 +484,14 @@ export const SelectableDataGrid: React.FC = React.memo( filtersRef.current = Object.values(columnFilters); }, [columnFilters]); + // Committed global search term (server-side pushdown for virtual tables). + // Held in a ref so fetchVirtualData stays stable across sort/scroll while + // still carrying the latest search into every request. + const searchRef = React.useRef(''); + React.useEffect(() => { + searchRef.current = (searchText || '').trim(); + }, [searchText]); + const fetchVirtualData = React.useCallback(( sortByColumnIds: string[], sortOrder: 'asc' | 'desc', @@ -470,6 +516,7 @@ export const SelectableDataGrid: React.FC = React.memo( : 'head', order_by_fields: sortByColumnIds.length > 0 ? sortByColumnIds : ['#rowId'], ...(activeFilters.length > 0 ? { filters: activeFilters } : {}), + ...(searchRef.current ? { search: searchRef.current } : {}), }; apiRequest(getUrls().SAMPLE_TABLE, { @@ -487,6 +534,7 @@ export const SelectableDataGrid: React.FC = React.memo( } else { setRowsToDisplay(newRows); } + setServerRowCount(totalCount); setHasMore(offset + newRows.length < totalCount); setIsLoading(false); setIsLoadingMore(false); @@ -518,6 +566,20 @@ export const SelectableDataGrid: React.FC = React.memo( // eslint-disable-next-line react-hooks/exhaustive-deps }, [columnFilters]); + // Refetch from offset 0 whenever the committed search term changes + // (virtual tables only). Non-virtual tables filter client-side below. + const didMountSearchRef = React.useRef(false); + React.useEffect(() => { + if (!virtual) return; + if (!didMountSearchRef.current) { + didMountSearchRef.current = true; + return; + } + searchRef.current = (searchText || '').trim(); + fetchVirtualData(orderBy ? [orderBy] : [], order, 0, false); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [searchText]); + const handleEndReached = React.useCallback(() => { if (!virtual || !hasMore || isLoadingMore || isLoading) return; fetchVirtualData( @@ -528,6 +590,15 @@ export const SelectableDataGrid: React.FC = React.memo( ); }, [virtual, hasMore, isLoadingMore, isLoading, fetchVirtualData, orderBy, order, rowsToDisplay.length]); + // Virtual tables are already filtered server-side (search pushdown), so we + // render their rows as-is. Non-virtual tables (fully loaded) filter + // client-side over the committed search term — case-insensitive substring. + const trimmedSearch = (searchText || '').trim().toLowerCase(); + const visibleRows = (!virtual && trimmedSearch) + ? rowsToDisplay.filter(row => + columnDefs.some(c => c.id !== '#rowId' && String(row[c.id] ?? '').toLowerCase().includes(trimmedSearch))) + : rowsToDisplay; + return ( = React.memo( = React.memo( )} - {virtual && } - {virtual && rowsToDisplay.length < rowCount - ? t('dataGrid.loadedOfTotal', { loaded: rowsToDisplay.length, total: rowCount }) - : t('dataGrid.rowCount', { count: rowCount })} + {virtual && rowsToDisplay.length < serverRowCount + ? t('dataGrid.loadedOfTotal', { loaded: rowsToDisplay.length, total: serverRowCount }) + : t('dataGrid.rowCount', { count: virtual ? serverRowCount : rowCount })} - {virtual && rowCount > 10000 && ( + {virtual && serverRowCount > 10000 && ( = React.memo( - + } {/* Column filter popover — variant chosen synchronously from metadata (design-doc 31). Server-side pushdown via fetchVirtualData. */} {filterPopover && (() => { diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index 97135c11..c0ceaac2 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -1560,32 +1560,6 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ } }, [pendingClarification, dispatch, t]); - // Landing / "no thread yet" highlight: when the user has loaded data - // but hasn't started an exploration on the focused table (no real - // charts AND the table isn't part of a derivation chain), gently pulse - // a colored ring around the input card to anchor the eye here. - // Suppressed once they start typing, while an agent is running, or - // while a clarification is pending — anything that already draws focus. - const focusedTableHasCharts = !!focusedTableId && charts.some(c => - c.tableRef === focusedTableId - && c.chartType !== '?' - && c.chartType !== 'Auto' - && c.source !== 'trigger' - ); - const focusedTableObj = focusedTableId ? tables.find(t => t.id === focusedTableId) : undefined; - const focusedTableHasDerivation = !!focusedTableObj && ( - focusedTableObj.derive !== undefined - || tables.some(t => t.derive?.trigger?.tableId === focusedTableId) - ); - const isLandingHighlight = ( - !!focusedTableId - && !focusedTableHasCharts - && !focusedTableHasDerivation - && !isChatFormulating - && !pendingClarification - && chatPrompt.trim() === '' - ); - const inputBox = ( void }> = function ({ '&:hover': { boxShadow: '0 2px 10px rgba(32, 33, 36, 0.14), 0 1px 3px rgba(32, 33, 36, 0.08)', }, - ...(isLandingHighlight ? { - animation: 'df-chatinput-landing-pulse 2.4s ease-in-out infinite', - '@keyframes df-chatinput-landing-pulse': { - '0%, 100%': { - boxShadow: `0 0 0 0 ${alpha(theme.palette.primary.main, 0.0)}, 0 4px 14px ${alpha(theme.palette.common.black, 0.06)}`, - }, - '50%': { - boxShadow: `0 0 0 6px ${alpha(theme.palette.primary.main, 0.14)}, 0 4px 18px ${alpha(theme.palette.common.black, 0.08)}`, - }, - }, - } : {}), '&:focus-within': { - animation: 'none', borderColor: theme.palette.primary.main, boxShadow: `0 0 0 2px ${alpha(theme.palette.primary.main, 0.15)}, 0 2px 10px rgba(32, 33, 36, 0.14)`, }, diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index 73d325e1..c08dab82 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -31,6 +31,7 @@ import { StreamIcon, getConnectorIcon, connectorSortOrder } from '../icons'; import RestartAltIcon from '@mui/icons-material/RestartAlt'; import ArrowBackIcon from '@mui/icons-material/ArrowBack'; import AddIcon from '@mui/icons-material/Add'; +import HistoryIcon from '@mui/icons-material/History'; import Paper from '@mui/material/Paper'; import CircularProgress from '@mui/material/CircularProgress'; @@ -210,6 +211,85 @@ const DataSourceCard: React.FC = ({ : card; }; +// Compact pill variant of DataSourceCard. Used by the chat-focused landing +// so data sources read as lightweight affordances orbiting the composer, +// rather than a grid of blocks competing with it. Same click behavior as +// DataSourceCard — only the visual weight differs. The description is +// demoted to a hover tooltip so the row stays dense. +const SourcePill: React.FC = ({ + icon, + title, + description, + onClick, + disabled = false, + variant = 'data', + badge, + tooltip, +}) => { + const theme = useTheme(); + const isAction = variant === 'action'; + + const pill = ( + + + {icon} + + + {title} + + {badge} + + ); + + const tip = tooltip ?? (description || null); + return tip + ? {pill} + : pill; +}; + const getUniqueTableName = (baseName: string, existingNames: Set): string => { let uniqueName = baseName; let counter = 1; @@ -523,15 +603,10 @@ export const DataLoadMenu: React.FC = ({ }, { value: 'url' as UploadTabType, - title: t('upload.loadFromUrlTitle', { defaultValue: 'Load from URL (live)' }), + title: t('upload.loadFromUrlTitle', { defaultValue: 'Load from URL' }), description: t('upload.loadFromUrlDesc'), icon: , disabled: false, - badge: , }, ]; @@ -649,63 +724,21 @@ export const DataLoadMenu: React.FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }), [t, onStartChat]); const agentChatBox = ( - - - - - {t('upload.dataLoadingAgent', { defaultValue: 'Data Loading Agent' })} - - {hasPriorConversation && onResumeChat && ( - - - {t('upload.resumePreviousConversation', { defaultValue: 'Previous conversation →' })} - - - )} - + + + + + + ) : undefined} onNonImageFile={(file) => { // Upload non-image files (Excel, CSV, JSON, …) to the // session scratch space. The filename is shown as a @@ -727,7 +760,7 @@ export const DataLoadMenu: React.FC = ({ }} attachments={agentAttachments} onAttachmentsChange={setAgentAttachments} - minRows={1} + minRows={4} tabSuggestion={t('upload.agentChatTabSuggestion', { defaultValue: 'What dataset do we have here?', })} @@ -746,85 +779,63 @@ export const DataLoadMenu: React.FC = ({ width: '100%', display: 'flex', flexDirection: 'column', - gap: 3, + gap: 2.5, mx: 0, textAlign: 'left', }}> - {/* Data Loading Agent quick-chat */} + {/* Data Loading Agent quick-chat — the hero of this surface */} {agentChatBox} - {/* Upload data */} - - - {t('upload.uploadData', { defaultValue: 'Upload data' })} - + {/* Sources — same width as the chat box so they read as part of it */} + + {/* Connected sources status bar — "Connected: xxx, xxx + connect + link" */} - {regularDataSources.map((source) => ( - + {t('upload.connectedLabel', { defaultValue: 'Connected:' })} + + {connectionSources.map((source) => ( + onSelectTab(source.value)} + onClick={() => handleConnectionClick(source.value)} disabled={source.disabled} - badge={source.badge} + variant={source.variant} + tooltip={source.tooltip} /> ))} - - {/* Data Connections */} - - - - {t('upload.dataConnections')} - + {/* Manual upload — one-off sources (file, paste, URL) */} - {connectionSources.map((source) => ( - + {t('upload.loadDirectlyLabel', { defaultValue: 'or load data directly:' })} + + {regularDataSources.map((source) => ( + handleConnectionClick(source.value)} + onClick={() => onSelectTab(source.value)} disabled={source.disabled} - variant={source.variant} - tooltip={source.tooltip} /> ))} diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index 698b5a72..e886b049 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -32,6 +32,7 @@ import { Alert, Fade, Grow, + CircularProgress, } from '@mui/material'; import _ from 'lodash'; @@ -62,6 +63,8 @@ import CasinoIcon from '@mui/icons-material/Casino'; import SaveAltIcon from '@mui/icons-material/SaveAlt'; import OpenInNewIcon from '@mui/icons-material/OpenInNew'; import CloseIcon from '@mui/icons-material/Close'; +import AddchartIcon from '@mui/icons-material/Addchart'; +import * as d3dsv from 'd3-dsv'; import { AgentToyIcon, AnimatedAgentToyIcon } from './AgentToyIcon'; import { CHART_TEMPLATES, getChartTemplate } from '../components/ChartTemplates'; @@ -1161,6 +1164,131 @@ const EmptyStateHero: FC<{ chartSelectionBox: React.ReactNode }> = ({ chartSelec ); }; +// Content-first action dock shown when a *table* (not a chart) is focused. +// The table becomes the primary content at the top of the canvas; this +// sticky bottom bar keeps a lightweight "pick a chart" affordance and a +// pointer to the chat available without a wall of buttons dominating the +// page. The full CHART_TEMPLATES palette lives inside the popover. +// NOTE: designed to extend to multi-table selection — when several tables +// are highlighted this dock can summarize the selection and offer actions +// that span them (e.g. "combine", "chart each"). +const TableActionDock: FC<{ + chartSelectionBox: React.ReactNode; + tableId: string; + tableName: string; + rows: any[]; + virtual: boolean; + gridReport?: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean } | null; + onRandomize?: () => void; +}> = ({ chartSelectionBox, tableId, tableName, rows, virtual, gridReport, onRandomize }) => { + const { t } = useTranslation(); + const [anchorEl, setAnchorEl] = useState(null); + const [isDownloading, setIsDownloading] = useState(false); + const open = Boolean(anchorEl); + + const handleDownload = async () => { + if (isDownloading) return; + setIsDownloading(true); + try { + if (virtual) { + const response = await fetchWithIdentity(getUrls().EXPORT_TABLE_CSV, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ table_name: tableId, delimiter: ',' }), + }); + if (!response.ok) throw new Error('Export failed'); + const blob = await response.blob(); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = `${tableName}.csv`; + a.click(); + URL.revokeObjectURL(a.href); + } else { + const csvContent = d3dsv.dsvFormat(',').format(rows); + const blob = new Blob([csvContent], { type: 'text/csv' }); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = `${tableName}.csv`; + a.click(); + URL.revokeObjectURL(a.href); + } + } catch (error) { + console.error('Error downloading table:', error); + } finally { + setIsDownloading(false); + } + }; + + return ( + + + + + + {gridReport?.virtual && ( + <> + + + {gridReport.loadedCount < gridReport.rowCount + ? t('dataGrid.loadedOfTotal', { loaded: gridReport.loadedCount, total: gridReport.rowCount }) + : t('dataGrid.rowCount', { count: gridReport.rowCount })} + + {gridReport.canRandomize && ( + + + + + + )} + + )} + + setAnchorEl(null)} + anchorOrigin={{ vertical: 'top', horizontal: 'left' }} + transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} + slotProps={{ paper: { sx: { p: 2, maxWidth: 'min(90vw, 1140px)' } } }} + > + {/* Clicking a template creates the chart and this whole + empty-state unmounts; closing the popover here keeps + things tidy in the transient frame. */} + setAnchorEl(null)}> + {chartSelectionBox} + + + + ); +}; + export const VisualizationViewFC: FC = function VisualizationView({ }) { const { t } = useTranslation(); @@ -1180,6 +1308,11 @@ export const VisualizationViewFC: FC = function VisualizationView let tables = useSelector((state: DataFormulatorState) => state.tables); + // Virtual-pagination state reported up from the focused-table grid, so the + // bottom toolbar can show the loaded/total count and drive the random dice. + const [tableGridReport, setTableGridReport] = React.useState<{ loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean } | null>(null); + const [tableRandomizeToken, setTableRandomizeToken] = React.useState(0); + let focusedChart = allCharts.find(c => c.id == focusedChartId) as Chart; let synthesisRunning = focusedChartId ? chartSynthesisInProgress.includes(focusedChartId) : false; @@ -1231,10 +1364,12 @@ export const VisualizationViewFC: FC = function VisualizationView )) } + const isTableFocus = focusedId?.type === 'table' && !!focusedTableId; return ( + {!isTableFocus && ( {(() => { @@ -1267,6 +1402,7 @@ export const VisualizationViewFC: FC = function VisualizationView })()} + )} {focusedId?.type === 'table' && focusedTableId && (() => { const focusedTable = tables.find(t => t.id === focusedTableId); if (!focusedTable) return null; @@ -1274,12 +1410,10 @@ export const VisualizationViewFC: FC = function VisualizationView const HEADER_HEIGHT = 32; const FOOTER_HEIGHT = 32; const MIN_TABLE_HEIGHT = 150; - const MAX_TABLE_HEIGHT = 400; const MIN_TABLE_WIDTH = 300; const MAX_TABLE_WIDTH = 900; const rowCount = focusedTable.virtual?.rowCount || focusedTable.rows?.length || 0; const contentHeight = HEADER_HEIGHT + rowCount * ROW_HEIGHT + FOOTER_HEIGHT; - const adaptiveHeight = Math.max(MIN_TABLE_HEIGHT, Math.min(MAX_TABLE_HEIGHT, contentHeight)); const ROW_ID_COL_WIDTH = 56; const sampleSize = Math.min(29, focusedTable.rows.length); const step = focusedTable.rows.length > sampleSize ? focusedTable.rows.length / sampleSize : 1; @@ -1295,16 +1429,40 @@ export const VisualizationViewFC: FC = function VisualizationView const SCROLLBAR_WIDTH = 17; // +34px gutter so the maximize button can sit just outside the table on the right. const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)) + 34; + // Single card that fills the available canvas height as much + // as possible (tall tables scroll inside), but never grows past + // its own content so short tables stay compact and centered. + // Width fills up to adaptiveWidth with an 80% floor. +48px is + // the focused-view header bar. + const HEADER_BAR_HEIGHT = 48; + const maxCardHeight = contentHeight + HEADER_BAR_HEIGHT; return ( - + + + ); })()} + {isTableFocus && focusedTableId && (() => { + const ft = tables.find(t => t.id === focusedTableId); + return ( + setTableRandomizeToken(x => x + 1)} + /> + ); + })()} diff --git a/tests/backend/agents/test_data_loading_discovery_tools.py b/tests/backend/agents/test_data_loading_discovery_tools.py index 61e65de2..7c9be80f 100644 --- a/tests/backend/agents/test_data_loading_discovery_tools.py +++ b/tests/backend/agents/test_data_loading_discovery_tools.py @@ -417,3 +417,107 @@ def test_includes_current_date_and_time(self) -> None: now = datetime.now() assert f"Current date and time: {now.strftime('%Y-%m-%d')}" in prompt assert f"({now.strftime('%A')})" in prompt + + +# ------------------------------------------------------------------ +# probe_data (design 37 §4.2 / §7) +# ------------------------------------------------------------------ + +class _StubLoader: + """Records the probe call and returns a canned result.""" + + def __init__(self, result=None): + self.result = result if result is not None else { + "rows": [{"n": 3}], "columns": ["n"], "row_count": 1, "exact": True, + } + self.calls = [] + + def probe(self, path, query): + self.calls.append((path, query)) + return self.result + + +class TestProbeData: + def test_missing_ids_returns_error(self, tmp_path: Path) -> None: + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + assert "error" in agent._tool_probe_data({"table_key": "k_orders"}) + assert "error" in agent._tool_probe_data({"source_id": "pg_prod"}) + + def test_unknown_table_key_returns_error(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + + result = agent._tool_probe_data({ + "source_id": "pg_prod", "table_key": "nope", "query": {}, + }) + assert "error" in result + assert "not found" in result["error"] + + def test_budget_exhaustion_returns_error(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 0 + + result = agent._tool_probe_data({ + "source_id": "pg_prod", "table_key": "k_orders", "query": {}, + }) + assert "error" in result + assert "budget" in result["error"].lower() + + def test_resolves_path_and_delegates_to_loader(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + stub = _StubLoader() + + with patch( + "data_formulator.data_connector.resolve_live_loader", + return_value=stub, + ): + result = agent._tool_probe_data({ + "source_id": "pg_prod", + "table_key": "k_orders", + "query": {"aggregates": [{"op": "count", "as": "n"}]}, + }) + + # The model-facing table_key is mapped to the catalog path. + assert stub.calls == [(["Sales", "monthly_orders"], + {"aggregates": [{"op": "count", "as": "n"}]})] + assert result["rows"] == [{"n": 3}] + assert "note" in result # row-cap guidance attached + assert agent._probe_budget == 4 # decremented once + + def test_not_connected_source_returns_error(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + + with patch( + "data_formulator.data_connector.resolve_live_loader", + side_effect=RuntimeError("no such connector"), + ): + result = agent._tool_probe_data({ + "source_id": "pg_prod", "table_key": "k_orders", "query": {}, + }) + assert "error" in result + assert "not connected" in result["error"] + # Budget is not consumed when the loader can't be resolved. + assert agent._probe_budget == 5 + + def test_loader_error_result_passes_through(self, tmp_path: Path) -> None: + save_catalog(tmp_path, "pg_prod", _SAMPLE_TABLES) + agent = DataLoadingAgent(client=None, workspace=_FakeWorkspace(tmp_path)) + agent._probe_budget = 5 + stub = _StubLoader(result={"error": "bad column"}) + + with patch( + "data_formulator.data_connector.resolve_live_loader", + return_value=stub, + ): + result = agent._tool_probe_data({ + "source_id": "pg_prod", "table_key": "k_orders", "query": {}, + }) + assert result == {"error": "bad column"} + assert "note" not in result # no cap note on error results From 932626aac28e397347bdabbdf08ef5d9cded2b43 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Wed, 8 Jul 2026 16:57:22 -0700 Subject: [PATCH 35/36] improvements --- .../agents/agent_starter_questions.py | 119 ++++++++++++ py-src/data_formulator/routes/agents.py | 29 +++ src/app/dfSlice.tsx | 98 +++++++++- src/app/store.ts | 2 +- src/app/useAutoSave.tsx | 3 + src/app/utils.tsx | 3 + src/i18n/index.ts | 3 + src/i18n/locales/en/chart.json | 3 + src/i18n/locales/en/common.json | 3 + src/i18n/locales/zh/chart.json | 3 + src/i18n/locales/zh/common.json | 3 + src/views/AgentChatInput.tsx | 18 +- src/views/ChartQuickConfig.tsx | 103 +++++++++-- src/views/DataThread.tsx | 2 +- src/views/DataView.tsx | 8 +- src/views/SelectableDataGrid.tsx | 48 +++-- src/views/SimpleChartRecBox.tsx | 171 +++++++++++++++++- src/views/UnifiedDataUploadDialog.tsx | 2 +- src/views/VisualizationView.tsx | 155 ++++++++++------ 19 files changed, 665 insertions(+), 111 deletions(-) create mode 100644 py-src/data_formulator/agents/agent_starter_questions.py diff --git a/py-src/data_formulator/agents/agent_starter_questions.py b/py-src/data_formulator/agents/agent_starter_questions.py new file mode 100644 index 00000000..54e704e9 --- /dev/null +++ b/py-src/data_formulator/agents/agent_starter_questions.py @@ -0,0 +1,119 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +from data_formulator.agent_config import reasoning_effort_for +from data_formulator.agents.agent_utils import extract_json_objects +from data_formulator.agents.agent_language import inject_language_instruction + +import logging + +logger = logging.getLogger(__name__) + +_AGENT_ID = "starter_questions" + + +SYSTEM_PROMPT = '''You are a data analyst helping a user get started exploring a freshly loaded dataset. +You are given a summary of the available tables (their names, columns, and a few sample rows) and one designated "primary_table". +Propose a small number of short, concrete starter questions the user could ask to explore the data. + +Guidelines: +- Center the questions on the primary_table (about its own columns / trends / comparisons / distributions / top-N). +- If other tables are present and share a plausible key with the primary table, you MAY include ONE cross-table question that relates the primary table to another table. +- Each question must be answerable by charting or analyzing the provided data (do not invent columns that are not present). +- Keep each question short and natural — under 12 words, phrased as a request (e.g. "Compare sales across regions"). +- Make the questions diverse and prefer referencing specific column names so they feel tailored. +- Do NOT include a generic "show high-level trends" question — that one is already provided separately. + +Return ONLY a json object of the following form: + +{ + "questions": ["", ""] +} + +Example: + +[INPUT] + +{ + "primary_table": "sales", + "tables": [ + { + "name": "sales", + "columns": ["date", "region", "product", "revenue", "units"], + "sample_rows": [ + {"date": "2023-01-01", "region": "West", "product": "A", "revenue": 1200, "units": 30}, + {"date": "2023-01-02", "region": "East", "product": "B", "revenue": 800, "units": 20} + ] + } + ] +} + +[OUTPUT] + +{ + "questions": ["Compare revenue across regions", "Which products sell the most units?"] +} +''' + + +class StarterQuestionsAgent(object): + + def __init__(self, client, language_instruction: str = ""): + self.client = client + self.language_instruction = language_instruction + + def run(self, tables, primary_table=None, n=2): + """Generate a short list of starter exploration questions. + + ``tables`` is a list of dicts with ``name``, optional ``description`` + and either ``columns`` and/or ``sample_rows``. ``primary_table`` is + the name of the table the questions should center on. Returns a list + of question strings (best effort, may be empty on failure). + """ + + input_obj = {"primary_table": primary_table, "tables": tables, "num_questions": n} + + user_query = f"[INPUT]\n\n{json.dumps(input_obj, ensure_ascii=False, default=str)}\n\n[OUTPUT]" + + logger.info("[StarterQuestionsAgent] run start") + + system_prompt = inject_language_instruction( + SYSTEM_PROMPT, self.language_instruction, + ) + + messages = [{"role": "system", "content": system_prompt}, + {"role": "user", "content": user_query}] + + response = self.client.get_completion( + messages=messages, + reasoning_effort=reasoning_effort_for(_AGENT_ID, self.client.model), + ) + + for choice in response.choices: + logger.debug("\n=== Starter questions agent ===>\n") + logger.debug(choice.message.content + "\n") + + content = choice.message.content or "" + + questions = [] + json_blocks = extract_json_objects(content + "\n") + candidate = None + if len(json_blocks) > 0: + candidate = json_blocks[0] + else: + try: + candidate = json.loads(content + "\n") + except (json.JSONDecodeError, ValueError, TypeError): + candidate = None + + if isinstance(candidate, dict): + raw = candidate.get("questions", []) + if isinstance(raw, list): + questions = [str(q).strip() for q in raw if str(q).strip()] + elif isinstance(candidate, list): + questions = [str(q).strip() for q in candidate if str(q).strip()] + + return questions[:n] + + return [] diff --git a/py-src/data_formulator/routes/agents.py b/py-src/data_formulator/routes/agents.py index ab77f6e5..950feeaa 100644 --- a/py-src/data_formulator/routes/agents.py +++ b/py-src/data_formulator/routes/agents.py @@ -19,6 +19,7 @@ import pandas as pd from data_formulator.agents.agent_sort_data import SortDataAgent +from data_formulator.agents.agent_starter_questions import StarterQuestionsAgent from data_formulator.agents.agent_simple import SimpleAgents from data_formulator.auth.identity import get_identity_id from data_formulator.security.code_signing import sign_result, verify_code, MAX_CODE_SIZE @@ -289,6 +290,34 @@ def sort_data_request(): logger.error("Error in sort-data", exc_info=e) raise classify_and_wrap_llm_error(e) from e +@agent_bp.route('/derive-starter-questions', methods=['GET', 'POST']) +def derive_starter_questions_request(): + """Generate a few short, data-tailored starter exploration questions. + + Called once when a workspace's set of root tables changes (e.g. after + data is loaded). Input: ``input_tables`` (list of {name, columns, + sample_rows, description}) and ``model``. Returns ``{"result": [..]}``. + """ + if not request.is_json: + raise AppError(ErrorCode.INVALID_REQUEST, "Invalid request format") + + logger.info("# derive-starter-questions request") + content = request.get_json() + + try: + client = get_client(content['model']) + + n = content.get('n', 2) + language_instruction = get_language_instruction(mode="compact") + agent = StarterQuestionsAgent(client=client, language_instruction=language_instruction) + questions = agent.run(content.get('input_tables', []), primary_table=content.get('primary_table'), n=n) + + questions = questions if questions is not None else [] + return json_ok({"result": questions}) + except Exception as e: + logger.error("Error in derive-starter-questions", exc_info=e) + raise classify_and_wrap_llm_error(e) from e + @agent_bp.route('/analyst-streaming', methods=['GET', 'POST']) def analyst_streaming(): """Unified AnalystAgent streaming endpoint (design-docs/35 + /36). diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index fdc24d72..8264dba5 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -272,6 +272,17 @@ export interface DataFormulatorState { * + briefly highlight). Cleared by the sidebar after consumption. */ focusedConnectorId?: string; + + /** + * AI-generated starter exploration questions, stored per root table. + * Keyed by table id; each entry's `questions` are tailored to that table + * (plus optional cross-table questions when other tables exist). + * `signature` is the root-table-set signature the questions were generated + * for (so they refresh when the set of tables changes). Persisted; + * `starterQuestionsStatus` is transient. + */ + starterQuestions: { [tableId: string]: { questions: string[]; signature: string } }; + starterQuestionsStatus: { [tableId: string]: 'idle' | 'loading' | 'error' }; } // Define the initial state using that type @@ -346,8 +357,10 @@ const initialState: DataFormulatorState = { dataSourceSidebarTab: 'sources', focusedConnectorId: undefined, -} + starterQuestions: {}, + starterQuestionsStatus: {}, +} /** * Non-memoized equivalent of `dfSelectors.getAllCharts` for use inside * reducers. Reducers receive an Immer draft `state`; passing a draft into @@ -530,6 +543,10 @@ let removeTableStateRoutine = (state: DataFormulatorState, tableId: string) => { // Delete reports triggered from this table state.generatedReports = state.generatedReports.filter(r => r.triggerTableId !== tableId); + // Drop this table's starter questions / generation status + delete state.starterQuestions[tableId]; + delete state.starterQuestionsStatus[tableId]; + if (state.focusedId?.type === 'table' && state.focusedId.tableId === tableId) { state.focusedId = state.tables.length > 0 ? { type: 'table', tableId: state.tables[0].id } : undefined; } @@ -564,6 +581,60 @@ export const fetchFieldSemanticType = createAsyncThunk( } ); +/** + * Generate a few short, table-tailored starter exploration questions for the + * currently focused root table. Called (debounced) when a root table is + * focused and it has no fresh questions for the current table-set signature. + * The focused table is the "primary" table; all root tables are passed as + * context so the agent can add a cross-table question when relevant. + * Dispatches `startStarterQuestions` up front, then `setStarterQuestions` / + * `setStarterQuestionsError`. Results are only applied if the signature is + * still current (guards against stale responses when data changed mid-flight). + */ +export const generateStarterQuestions = createAsyncThunk( + "dataFormulatorSlice/generateStarterQuestions", + async (arg: { tableId: string; signature: string; tableIds: string[] }, { getState, dispatch }) => { + const state = getState() as DataFormulatorState; + + dispatch(dfActions.startStarterQuestions({ tableId: arg.tableId, signature: arg.signature })); + + const inputTables = arg.tableIds + .map(id => state.tables.find(t => t.id === id)) + .filter((t): t is DictTable => !!t) + .map(t => ({ + name: t.id, + columns: t.names, + sample_rows: (t.rows || []).slice(0, 10), + description: typeof t.description === 'string' ? t.description : '', + })); + + if (inputTables.length === 0) { + dispatch(dfActions.setStarterQuestions({ tableId: arg.tableId, signature: arg.signature, questions: [] })); + return; + } + + try { + const { data } = await apiRequest(getUrls().DERIVE_STARTER_QUESTIONS, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + input_tables: inputTables, + primary_table: arg.tableId, + model: dfSelectors.getActiveModel(state), + n: 2, + }), + }); + const questions: string[] = Array.isArray(data?.result) + ? data.result.map((q: any) => String(q)).filter((q: string) => q.trim() !== '') + : []; + dispatch(dfActions.setStarterQuestions({ tableId: arg.tableId, signature: arg.signature, questions: questions.slice(0, 2) })); + } catch (err) { + console.warn('generateStarterQuestions failed', err); + dispatch(dfActions.setStarterQuestionsError({ tableId: arg.tableId, signature: arg.signature })); + } + } +); + /** * Fetch backend-computed per-column statistics for a workspace-stored * (virtual) table and merge them into ``table.metadata``. Powers the @@ -737,6 +808,27 @@ export const dataFormulatorSlice = createSlice({ clearFocusedConnector: (state) => { state.focusedConnectorId = undefined; }, + /** Mark a starter-questions generation as started for a table. */ + startStarterQuestions: (state, action: PayloadAction<{ tableId: string; signature: string }>) => { + state.starterQuestions[action.payload.tableId] = { + questions: [], + signature: action.payload.signature, + }; + state.starterQuestionsStatus[action.payload.tableId] = 'loading'; + }, + /** Store generated starter questions (only if signature still current). */ + setStarterQuestions: (state, action: PayloadAction<{ tableId: string; signature: string; questions: string[] }>) => { + const cur = state.starterQuestions[action.payload.tableId]; + if (!cur || cur.signature !== action.payload.signature) return; + cur.questions = action.payload.questions; + state.starterQuestionsStatus[action.payload.tableId] = 'idle'; + }, + /** Mark starter-questions generation as failed (only if signature current). */ + setStarterQuestionsError: (state, action: PayloadAction<{ tableId: string; signature: string }>) => { + const cur = state.starterQuestions[action.payload.tableId]; + if (!cur || cur.signature !== action.payload.signature) return; + state.starterQuestionsStatus[action.payload.tableId] = 'error'; + }, loadState: (state, action: PayloadAction) => { const saved = action.payload; @@ -834,6 +926,10 @@ export const dataFormulatorSlice = createSlice({ // repopulates this slice from the module cache / fresh // renders after load. chartThumbnails: {}, + + // Restore persisted starter questions (status is transient). + starterQuestions: saved.starterQuestions ?? {}, + starterQuestionsStatus: {}, }; }, setServerConfig: (state, action: PayloadAction) => { diff --git a/src/app/store.ts b/src/app/store.ts index c230f5a6..a4a0828b 100644 --- a/src/app/store.ts +++ b/src/app/store.ts @@ -16,7 +16,7 @@ const persistConfig = { // globalModels are always fetched fresh from the server on each app start, // so there is no need (and it would cause stale-data issues) to persist them. // In-progress flags are transient and should not survive page refreshes. - blacklist: ['serverConfig', 'globalModels', 'chartSynthesisInProgress'], + blacklist: ['serverConfig', 'globalModels', 'chartSynthesisInProgress', 'starterQuestionsStatus'], } const persistedReducer = persistReducer(persistConfig, dataFormulatorReducer) diff --git a/src/app/useAutoSave.tsx b/src/app/useAutoSave.tsx index 3a167f14..e2888a17 100644 --- a/src/app/useAutoSave.tsx +++ b/src/app/useAutoSave.tsx @@ -17,6 +17,9 @@ const EXCLUDED_FIELDS = new Set([ // Transient fields that shouldn't trigger or be included in saves 'chartSynthesisInProgress', 'cleanInProgress', 'sessionLoading', 'sessionLoadingLabel', + // Starter-questions status is transient (loading/error); the questions + // themselves are persisted, but the fetch status should reset on reload. + 'starterQuestionsStatus', // Thumbnails are derived from chart specs + table data; re-rendered // from the module cache on reload, so don't waste bandwidth saving them. 'chartThumbnails', diff --git a/src/app/utils.tsx b/src/app/utils.tsx index 196c403d..e7de0b56 100644 --- a/src/app/utils.tsx +++ b/src/app/utils.tsx @@ -49,6 +49,9 @@ export function getUrls() { GET_RECOMMENDATION_QUESTIONS: `/api/agent/get-recommendation-questions`, + // Starter exploration questions (generated on data load) + DERIVE_STARTER_QUESTIONS: `/api/agent/derive-starter-questions`, + // Workspace display name (auto-naming) WORKSPACE_NAME: `/api/agent/workspace-name`, diff --git a/src/i18n/index.ts b/src/i18n/index.ts index ea75a071..2912d1d5 100644 --- a/src/i18n/index.ts +++ b/src/i18n/index.ts @@ -6,6 +6,9 @@ import { initReactI18next } from 'react-i18next'; import LanguageDetector from 'i18next-browser-languagedetector'; import { en, zh } from './locales'; +// NOTE: locale JSON is ingested into the i18next store once, here, at init(). +// Adding keys to a locale file requires a full page reload (not just HMR) for +// the running app to pick them up, since HMR won't re-run this init(). const resources = { en: { translation: en }, zh: { translation: zh }, diff --git a/src/i18n/locales/en/chart.json b/src/i18n/locales/en/chart.json index 68dd50c2..60e4c96b 100644 --- a/src/i18n/locales/en/chart.json +++ b/src/i18n/locales/en/chart.json @@ -20,6 +20,9 @@ "duplicate": "duplicate the chart", "delete": "delete", "deleteChart": "delete chart", + "deleteChartConfirm": "Delete this chart?", + "deleteChartCancel": "cancel", + "deleteChartYes": "delete", "sampleSize": "Sample size", "sampleSizeAria": "Sample size", "sampleAgain": "sample again!", diff --git a/src/i18n/locales/en/common.json b/src/i18n/locales/en/common.json index 7357326d..12999d79 100644 --- a/src/i18n/locales/en/common.json +++ b/src/i18n/locales/en/common.json @@ -540,6 +540,8 @@ "generateReport": "Generate a report", "reportPrompt": "Write a report summarizing the key findings from this exploration.", "askedForReport": "Write a report summarizing the exploration.", + "expandStarters": "Show suggestions", + "collapseStarters": "Hide suggestions", "endConversation": "End conversation", "sendReply": "Send reply", "explore": "Explore", @@ -590,6 +592,7 @@ "rowCount": "{{count}} rows", "loadedOfTotal": "{{loaded}} / {{total}} rows", "viewRandomRows": "view 10000 random rows from this table", + "restoreOrder": "Restore original order", "downloadAsCsv": "Download as CSV", "downloading": "Downloading...", "columnMenu": { diff --git a/src/i18n/locales/zh/chart.json b/src/i18n/locales/zh/chart.json index d08ce521..3b9f02d5 100644 --- a/src/i18n/locales/zh/chart.json +++ b/src/i18n/locales/zh/chart.json @@ -20,6 +20,9 @@ "duplicate": "复制图表", "delete": "删除", "deleteChart": "删除图表", + "deleteChartConfirm": "确定删除此图表？", + "deleteChartCancel": "取消", + "deleteChartYes": "删除", "sampleSize": "样本大小", "sampleSizeAria": "样本大小", "sampleAgain": "重新采样！", diff --git a/src/i18n/locales/zh/common.json b/src/i18n/locales/zh/common.json index e08983fb..07d2e376 100644 --- a/src/i18n/locales/zh/common.json +++ b/src/i18n/locales/zh/common.json @@ -476,6 +476,7 @@ "rowCount": "{{count}} 行", "loadedOfTotal": "已加载 {{loaded}} / {{total}} 行", "viewRandomRows": "查看此表的 10000 行随机数据", + "restoreOrder": "恢复原始顺序", "downloadAsCsv": "下载为 CSV", "downloading": "下载中...", "columnMenu": { @@ -591,6 +592,8 @@ "generateReport": "生成报告", "reportPrompt": "撰写一份报告，总结本次探索的主要发现。", "askedForReport": "撰写一份报告，总结本次探索。", + "expandStarters": "显示建议", + "collapseStarters": "隐藏建议", "endConversation": "结束对话", "sendReply": "发送回复", "explore": "探索", diff --git a/src/views/AgentChatInput.tsx b/src/views/AgentChatInput.tsx index b4d9ea0e..6a0598ba 100644 --- a/src/views/AgentChatInput.tsx +++ b/src/views/AgentChatInput.tsx @@ -470,7 +470,7 @@ export const AgentChatInput: React.FC = ({ px: 1.5, py: 0.5, cursor: 'pointer', - color: 'text.secondary', + color: 'text.primary', '&:hover': { bgcolor: 'action.hover', color: 'text.primary' }, }} > @@ -482,7 +482,7 @@ export const AgentChatInput: React.FC = ({ alignItems: 'center', justifyContent: 'center', width: 16, flexShrink: 0, - color: 'text.disabled', + color: 'text.secondary', }} > {s.icon} @@ -509,20 +509,6 @@ export const AgentChatInput: React.FC = ({ > {s.label} - {s.kind && ( - - {s.kind} - - )} ))} diff --git a/src/views/ChartQuickConfig.tsx b/src/views/ChartQuickConfig.tsx index 570a6b89..b4b4b968 100644 --- a/src/views/ChartQuickConfig.tsx +++ b/src/views/ChartQuickConfig.tsx @@ -11,7 +11,7 @@ import { FC } from 'react'; import React from 'react'; import { useTranslation } from 'react-i18next'; import { useSelector, useDispatch } from 'react-redux'; -import { Box, Typography, Select, MenuItem, useTheme, Tooltip, IconButton, Divider } from '@mui/material'; +import { Box, Typography, Select, MenuItem, useTheme, Tooltip, IconButton, Divider, Popover, Button } from '@mui/material'; import DeleteIcon from '@mui/icons-material/Delete'; import { DataFormulatorState, dfActions, dfSelectors } from '../app/dfSlice'; @@ -124,10 +124,15 @@ type QuickControl = { }; export const ChartQuickConfig: FC = function ({ chartId, tableMetadata, options, deleteDisabled }) { - const { t } = useTranslation('chart'); + const { t } = useTranslation(); const theme = useTheme(); const dispatch = useDispatch(); + // Confirmation popover for the destructive delete action: clicking the + // trash opens a small "delete?" popover so an accidental click can't remove + // the chart. Anchored to the button; explicit confirm/dismiss. + const [deleteAnchor, setDeleteAnchor] = React.useState(null); + const allCharts = useSelector(dfSelectors.getAllCharts); const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); const chart = allCharts.find((c: Chart) => c.id == chartId) as Chart | undefined; @@ -326,22 +331,84 @@ export const ChartQuickConfig: FC = function ({ chartId, })} {/* Built-in chart-level action: delete this chart. Lives in the property-config bar so the chart's controls and its delete sit - together; a hairline divider sets it apart from the property - controls. Only shown when there are controls to set it apart - from — otherwise it stands alone in the bar. */} - - - dispatch(dfActions.deleteChartById(chartId))} - sx={{ color: 'text.disabled','&:hover': { color: 'error.main', backgroundColor: 'rgba(211, 47, 47, 0.08)' } }} - > - - - - + together; a hairline divider + extra spacing sets it apart from + the property controls so it isn't easy to mis-fire. Clicking + opens a confirmation popover since it's destructive. */} + + + + + setDeleteAnchor(e.currentTarget)} + sx={{ + color: deleteAnchor ? 'error.main' : 'text.disabled', + backgroundColor: deleteAnchor ? 'rgba(211, 47, 47, 0.08)' : 'transparent', + '&:hover': { color: 'error.main', backgroundColor: 'rgba(211, 47, 47, 0.08)' }, + }} + > + + + + + setDeleteAnchor(null)} + anchorOrigin={{ vertical: 'top', horizontal: 'center' }} + transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} + slotProps={{ paper: { sx: { + p: 1.25, + borderRadius: '8px', + mt: '-10px', + ml: '-14px', + overflow: 'visible', + boxShadow: '0 4px 16px rgba(0,0,0,0.16)', + // Callout arrow pointing straight down at the delete button + // (button center sits under this arrow via the ml offset). + '&::before': { + content: '""', + position: 'absolute', + bottom: -5, + left: 14, + width: 10, + height: 10, + backgroundColor: 'background.paper', + transform: 'rotate(45deg)', + boxShadow: '2px 2px 3px rgba(0,0,0,0.06)', + }, + } } }} + > + + {t('chart.deleteChartConfirm')} + + + + + + + ); diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index fff265d2..8a194a66 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -3443,7 +3443,7 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { display: 'flex', flexDirection: 'row', flexWrap: 'nowrap', - justifyContent: 'center', + justifyContent: 'flex-start', gap: `${CARD_GAP}px`, py: 1, // Bottom padding leaves room so the scroll handler can position diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index 98217775..f5dd9214 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -42,10 +42,13 @@ export interface FreeDataViewProps { // Controlled random-rows trigger + virtual-state report, forwarded to the // grid so the external toolbar can drive/observe them. randomizeToken?: number; - onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean }) => void; + // Controlled "restore natural order" trigger, forwarded to the grid so the + // toolbar can undo a random sample. + resetOrderToken?: number; + onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean }) => void; } -export const FreeDataViewFC: FC = function DataView({ maximizable, tableId, showHeaderBar, hideFooter, randomizeToken, onStateReport }) { +export const FreeDataViewFC: FC = function DataView({ maximizable, tableId, showHeaderBar, hideFooter, randomizeToken, resetOrderToken, onStateReport }) { const { t } = useTranslation(); const [maximized, setMaximized] = React.useState(false); @@ -165,6 +168,7 @@ export const FreeDataViewFC: FC = function DataView({ maximiz searchText={searchQuery} hideFooter={hideFooter} randomizeToken={randomizeToken} + resetOrderToken={resetOrderToken} onStateReport={onStateReport} /> diff --git a/src/views/SelectableDataGrid.tsx b/src/views/SelectableDataGrid.tsx index 222a883e..62b77633 100644 --- a/src/views/SelectableDataGrid.tsx +++ b/src/views/SelectableDataGrid.tsx @@ -66,9 +66,12 @@ interface SelectableDataGridProps { hideFooter?: boolean; // Bumping this number triggers a "random rows" refetch (virtual tables). randomizeToken?: number; + // Bumping this number restores the natural (#rowId head) order after a + // random sample (virtual tables). + resetOrderToken?: number; // Report virtual-pagination state up so an external toolbar can render the // loaded/total count and enable the random-rows action. - onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean }) => void; + onStateReport?: (s: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean }) => void; } @@ -334,7 +337,7 @@ const VirtuosoTableBody = React.forwardRef((props, ref) const PAGE_SIZE = 500; export const SelectableDataGrid: React.FC = React.memo(({ - tableId, rows, tableName, columnDefs, rowCount, virtual, searchText, hideFooter, randomizeToken, onStateReport }) => { + tableId, rows, tableName, columnDefs, rowCount, virtual, searchText, hideFooter, randomizeToken, resetOrderToken, onStateReport }) => { const { t } = useTranslation(); const [orderBy, setOrderBy] = React.useState(undefined); @@ -354,11 +357,12 @@ export const SelectableDataGrid: React.FC = React.memo( // Ref-based bridge to fetchVirtualData (declared further below); lets stable // sort handlers call into the latest fetch function without re-creating themselves. - const fetchVirtualDataRef = React.useRef<((sortByColumnIds: string[], sortOrder: 'asc' | 'desc', offset?: number, append?: boolean) => void) | null>(null); + const fetchVirtualDataRef = React.useRef<((sortByColumnIds: string[], sortOrder: 'asc' | 'desc', offset?: number, append?: boolean, random?: boolean) => void) | null>(null); const applySort = React.useCallback((newOrderBy: string | undefined, newOrder: 'asc' | 'desc') => { setOrder(newOrder); setOrderBy(newOrderBy); + setIsRandom(false); if (virtual) { fetchVirtualDataRef.current?.(newOrderBy ? [newOrderBy] : [], newOrder); } @@ -367,7 +371,9 @@ export const SelectableDataGrid: React.FC = React.memo( let theme = useTheme(); const [rowsToDisplay, setRowsToDisplay] = React.useState(rows); - + // True while the grid is showing a random sample (virtual tables). Lets the + // external toolbar offer a "restore order" action alongside the dice. + const [isRandom, setIsRandom] = React.useState(false); const [isLoading, setIsLoading] = React.useState(true); const [isLoadingMore, setIsLoadingMore] = React.useState(false); const [isDownloading, setIsDownloading] = React.useState(false); @@ -401,20 +407,35 @@ export const SelectableDataGrid: React.FC = React.memo( rowCount: effectiveCount, virtual, canRandomize: virtual && effectiveCount > 10000, + isRandom: virtual && isRandom, }); - }, [rowsToDisplay.length, rowCount, serverRowCount, virtual, onStateReport]); + }, [rowsToDisplay.length, rowCount, serverRowCount, virtual, isRandom, onStateReport]); // Bumping `randomizeToken` (from the external toolbar's dice) resets sort - // and refetches a fresh random page for virtual tables. + // and refetches a fresh random sample (ORDER BY RANDOM() on the server, + // which also respects any active filters/search) for virtual tables. const randomizeMountRef = React.useRef(true); React.useEffect(() => { if (randomizeMountRef.current) { randomizeMountRef.current = false; return; } if (!virtual) return; setOrderBy(undefined); setOrder('asc'); - fetchVirtualDataRef.current?.([], 'asc'); + setIsRandom(true); + fetchVirtualDataRef.current?.([], 'asc', 0, false, true); }, [randomizeToken]); + // Bumping `resetOrderToken` (from the toolbar's "restore order" button) + // returns a randomized virtual table to its natural #rowId head order. + const resetOrderMountRef = React.useRef(true); + React.useEffect(() => { + if (resetOrderMountRef.current) { resetOrderMountRef.current = false; return; } + if (!virtual) return; + setOrderBy(undefined); + setOrder('asc'); + setIsRandom(false); + fetchVirtualDataRef.current?.([], 'asc', 0, false, false); + }, [resetOrderToken]); + // Only the Table component depends on columnDefs (for colgroup); memoize it // so react-virtuoso keeps a stable reference when columns haven't changed. const columnIds = columnDefs.map(c => c.id).join(','); @@ -497,6 +518,7 @@ export const SelectableDataGrid: React.FC = React.memo( sortOrder: 'asc' | 'desc', offset: number = 0, append: boolean = false, + random: boolean = false, ) => { if (!append) { setIsLoading(true); @@ -511,9 +533,11 @@ export const SelectableDataGrid: React.FC = React.memo( table: tableId, size: PAGE_SIZE, offset, - method: sortByColumnIds.length > 0 - ? (sortOrder === 'asc' ? 'head' : 'bottom') - : 'head', + method: random + ? 'random' + : (sortByColumnIds.length > 0 + ? (sortOrder === 'asc' ? 'head' : 'bottom') + : 'head'), order_by_fields: sortByColumnIds.length > 0 ? sortByColumnIds : ['#rowId'], ...(activeFilters.length > 0 ? { filters: activeFilters } : {}), ...(searchRef.current ? { search: searchRef.current } : {}), @@ -535,7 +559,7 @@ export const SelectableDataGrid: React.FC = React.memo( setRowsToDisplay(newRows); } setServerRowCount(totalCount); - setHasMore(offset + newRows.length < totalCount); + setHasMore(!random && offset + newRows.length < totalCount); setIsLoading(false); setIsLoadingMore(false); }) @@ -562,6 +586,7 @@ export const SelectableDataGrid: React.FC = React.memo( didMountFiltersRef.current = true; return; } + setIsRandom(false); fetchVirtualData(orderBy ? [orderBy] : [], order, 0, false); // eslint-disable-next-line react-hooks/exhaustive-deps }, [columnFilters]); @@ -576,6 +601,7 @@ export const SelectableDataGrid: React.FC = React.memo( return; } searchRef.current = (searchText || '').trim(); + setIsRandom(false); fetchVirtualData(orderBy ? [orderBy] : [], order, 0, false); // eslint-disable-next-line react-hooks/exhaustive-deps }, [searchText]); diff --git a/src/views/SimpleChartRecBox.tsx b/src/views/SimpleChartRecBox.tsx index c0ceaac2..3fe84f18 100644 --- a/src/views/SimpleChartRecBox.tsx +++ b/src/views/SimpleChartRecBox.tsx @@ -14,6 +14,7 @@ import { Card, LinearProgress, Chip, + Collapse, Popper, Paper, MenuList, @@ -21,7 +22,7 @@ import { } from '@mui/material'; import { useDispatch, useSelector } from 'react-redux'; -import { DataFormulatorState, dfActions, dfSelectors, fetchCodeExpl, fetchFieldSemanticType, generateFreshChart, GeneratedReport } from '../app/dfSlice'; +import { DataFormulatorState, dfActions, dfSelectors, fetchCodeExpl, fetchFieldSemanticType, generateFreshChart, generateStarterQuestions, GeneratedReport } from '../app/dfSlice'; import { AppDispatch } from '../app/store'; import { resolveRecommendedChart, getUrls, getTriggers, translateBackend } from '../app/utils'; import { streamRequest } from '../app/apiClient'; @@ -33,9 +34,9 @@ import { normalizeClarifyEvent, formatClarificationResponses } from '../app/clar import { alpha } from '@mui/material/styles'; import { WritingPencil } from '../components/FunComponents'; import ArrowUpwardRoundedIcon from '@mui/icons-material/ArrowUpwardRounded'; -import CloseIcon from '@mui/icons-material/Close'; import AddIcon from '@mui/icons-material/Add'; import TipsAndUpdatesIcon from '@mui/icons-material/TipsAndUpdates'; +import BoltIcon from '@mui/icons-material/Bolt'; import EditOutlinedIcon from '@mui/icons-material/EditOutlined'; import StopIcon from '@mui/icons-material/Stop'; @@ -45,12 +46,43 @@ import { Theme } from '@mui/material/styles'; import { useTranslation } from 'react-i18next'; import { shouldAutoFocusGeneratedChart } from '../app/agentInteractionPolicy'; import { ClarificationPanel, DelegatePanel, ExplanationPanel } from './AgentPausePanel'; +import { CARD_WIDTH } from './threadLayout'; + +// Approx footprint of the leading lightning-bolt IconButton (size small, +// p:0.5 + 16px icon). Used to cap a starter chip so a single chip fits +// within one thread-column width alongside the toggle icon. +const STARTER_ICON_WIDTH = 28; // Seed prompt used when the user invokes "report" mode (or a report hand-off) // without typing an explicit instruction. The unified analyst loads its // `report` skill and emits `write_report` within a normal explore run. const REPORT_SEED_PROMPT = 'Write a report summarizing the exploration.'; +// A starter-question chip that only shows a tooltip when its label is +// truncated (i.e. the text is too long to fit within the capped width). +const StarterChip: FC<{ label: string; onClick: () => void; sx: any }> = ({ label, onClick, sx }) => { + const labelRef = useRef(null); + const [isClipped, setIsClipped] = useState(false); + const checkClipped = () => { + // The MUI `.MuiChip-label` element (parent of our span) is the one that + // applies overflow:hidden + ellipsis, so measure that container. + const el = labelRef.current?.parentElement; + if (el) setIsClipped(el.scrollWidth > el.clientWidth); + }; + return ( + + {label}} + onClick={onClick} + onMouseEnter={checkClipped} + sx={sx} + /> + + ); +}; + const AgentWorkingOverlay: FC<{ message?: string; elapsed?: number; theme: Theme; onCancel?: () => void; color?: 'primary' | 'warning' }> = ({ message, elapsed, theme, onCancel, color = 'primary' }) => { const { t } = useTranslation(); // `message` is the running plan: steps joined by the STEP_SEP control char @@ -134,7 +166,8 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ const tables = useSelector((state: DataFormulatorState) => state.tables); const focusedId = useSelector((state: DataFormulatorState) => state.focusedId); const charts = useSelector(dfSelectors.getAllCharts); - const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); + const starterQuestions = useSelector((state: DataFormulatorState) => state.starterQuestions); + const starterQuestionsStatus = useSelector((state: DataFormulatorState) => state.starterQuestionsStatus); const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); const config = useSelector((state: DataFormulatorState) => state.config); const activeModel = useSelector(dfSelectors.getActiveModel); const workspaceBackend = useSelector((state: DataFormulatorState) => state.serverConfig.WORKSPACE_BACKEND); @@ -160,9 +193,11 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // clears). Tracks the draftId we've already auto-submitted for. const clarifySubmittedRef = useRef(null); const [isChatFormulating, setIsChatFormulating] = useState(false); + // Whether the getting-started starter questions are collapsed (click the + // lightning bolt to expand/collapse). + const [starterCollapsed, setStarterCollapsed] = useState(false); const [mentionedTableIds, setMentionedTableIds] = useState([]); - const [mentionDropdownOpen, setMentionDropdownOpen] = useState(false); - const [mentionHighlightIdx, setMentionHighlightIdx] = useState(0); + const [mentionDropdownOpen, setMentionDropdownOpen] = useState(false); const [mentionHighlightIdx, setMentionHighlightIdx] = useState(0); const [attachedImages, setAttachedImages] = useState([]); const [attachedFiles, setAttachedFiles] = useState<{ name: string; content: string }[]>([]); const fileInputRef = useRef(null); @@ -198,6 +233,14 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ // Stale draft detection is handled by loadState in dfSlice (marks running/clarifying drafts as interrupted) const inputCardRef = useRef(null); + // Ref to the chat textarea so getting-started starter prompts can seed + // the input and immediately focus it for editing. + const chatInputRef = useRef(null); + + const seedChatPrompt = useCallback((text: string) => { + setChatPrompt(text); + requestAnimationFrame(() => chatInputRef.current?.focus()); + }, []); const generatedReports = useSelector((state: DataFormulatorState) => state.generatedReports); @@ -270,6 +313,35 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ setMentionHighlightIdx(0); }, [mentionFilter]); + // ── Starter-questions generation trigger ───────────────────────── + // Questions are stored per root table (each table has its own, plus an + // optional cross-table question). When a root table is focused and it has + // no fresh questions for the current table set, generate them lazily. The + // signature (all root table ids) refreshes questions when tables change; + // the 500ms debounce collapses batch loads into a single call. + const rootTableSignature = React.useMemo( + () => rootTables.map(t => t.id).sort().join('|'), + [rootTables] + ); + const focusedRootTableId = (focusedTableId && rootTables.some(t => t.id === focusedTableId)) + ? focusedTableId + : undefined; + React.useEffect(() => { + if (!focusedRootTableId) return; + const entry = starterQuestions[focusedRootTableId]; + if (entry && entry.signature === rootTableSignature) return; // already fresh + if (starterQuestionsStatus[focusedRootTableId] === 'loading') return; // in flight + const timer = setTimeout(() => { + dispatch(generateStarterQuestions({ + tableId: focusedRootTableId, + signature: rootTableSignature, + tableIds: rootTableSignature.split('|'), + })); + }, 500); + return () => clearTimeout(timer); + }, [focusedRootTableId, rootTableSignature, starterQuestions, starterQuestionsStatus, dispatch]); + + // Helper: confirm selection of a mention (table only) const confirmMention = (optionId: string) => { const tbl = tables.find(t => t.id === optionId); @@ -1799,6 +1871,7 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ inputLabel: { shrink: true }, input: { readOnly: isChatFormulating }, }} + inputRef={chatInputRef} value={chatPrompt} placeholder={ pendingClarification @@ -1920,8 +1993,96 @@ export const SimpleChartRecBox: FC<{ onInputFocus?: () => void }> = function ({ ); + // ── Getting-started guidance ───────────────────────────────────── + // When a root table is focused, show a muted row of AI-generated starter + // questions tailored to that table (see the trigger effect above — each + // table has its own set, plus an optional cross-table question). Clicking + // a chip runs it; clicking the lightning bolt collapses/expands the row. + const focusedStarterEntry = focusedRootTableId ? starterQuestions[focusedRootTableId] : undefined; + const focusedStarterStatus = focusedRootTableId ? starterQuestionsStatus[focusedRootTableId] : undefined; + const focusedStarterFresh = !!focusedStarterEntry && focusedStarterEntry.signature === rootTableSignature; + const starterLoading = !!focusedRootTableId && (!focusedStarterFresh || focusedStarterStatus === 'loading'); + + const showGettingStarted = !!focusedRootTableId + && !isChatFormulating + && !pendingClarification + && (starterLoading || (focusedStarterEntry?.questions?.length ?? 0) > 0); + + const starterChipSx = { + height: 24, borderRadius: '6px', fontSize: 11, + color: 'text.secondary', + backgroundColor: 'transparent', + border: `1px solid ${borderColor.divider}`, + // Cap a single chip to one column width minus the toggle icon, and + // allow it to shrink (flex) when the row would otherwise overflow. + maxWidth: CARD_WIDTH - STARTER_ICON_WIDTH, + minWidth: 0, + flexShrink: 1, + '& .MuiChip-label': { + px: '8px', + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + }, + '&:hover': { + backgroundColor: alpha(theme.palette.text.primary, 0.04), + color: 'text.primary', + borderColor: alpha(theme.palette.text.primary, 0.24), + }, + } as const; + + const gettingStartedBlock = showGettingStarted ? ( + + + setStarterCollapsed(c => !c)} + sx={{ + flexShrink: 0, + p: 0.5, borderRadius: '6px', color: 'text.disabled', + transition: 'background-color 0.15s, color 0.15s', + '&:hover': { color: 'text.secondary', backgroundColor: alpha(theme.palette.text.primary, 0.06) }, + }} + > + + + + {starterCollapsed && ( + setStarterCollapsed(false)} + sx={{ fontSize: 11, color: 'text.disabled', cursor: 'pointer', '&:hover': { color: 'text.secondary' } }} + > + {t('chartRec.expandStarters', { defaultValue: 'Show suggestions' })} + + )} + + {starterLoading + ? + : (focusedStarterEntry?.questions ?? []).map((q, i) => ( + submitChat(q)} + sx={starterChipSx} + /> + )) + } + + + ) : null; + return ( + {gettingStartedBlock} {/* The input box */} {inputBox} diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx index c08dab82..32061b33 100644 --- a/src/views/UnifiedDataUploadDialog.tsx +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -259,7 +259,7 @@ const SourcePill: React.FC = ({ justifyContent: 'center', width: 22, height: 22, - borderRadius: 0.75, + borderRadius: '50%', backgroundColor: alpha(theme.palette.primary.main, 0.08), flexShrink: 0, '& .MuiSvgIcon-root': { fontSize: 15 }, diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index e886b049..3161b094 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -532,6 +532,12 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { const [encodingOpen, setEncodingOpen] = useState(false); const editButtonRef = useRef(null); + // State for the compact action dock that sits below the chart-mode data + // table (mirrors the table-focus dock; replaces the grid's inline footer). + const [chartTableGridReport, setChartTableGridReport] = useState<{ loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean } | null>(null); + const [chartRandomizeToken, setChartRandomizeToken] = useState(0); + const [chartResetOrderToken, setChartResetOrderToken] = useState(0); + // Reset local UI state when focused chart changes useEffect(() => { setCodeDialogOpen(false); @@ -883,13 +889,13 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { {(() => { const ROW_HEIGHT = 25; const HEADER_HEIGHT = 32; - const FOOTER_HEIGHT = 32; - const MIN_TABLE_HEIGHT = 150; + const MIN_TABLE_HEIGHT = 60; const MAX_TABLE_HEIGHT = 400; const MIN_TABLE_WIDTH = 300; const MAX_TABLE_WIDTH = 900; const rowCount = table.virtual?.rowCount || table.rows?.length || 0; - const contentHeight = HEADER_HEIGHT + rowCount * ROW_HEIGHT + FOOTER_HEIGHT; + // Footer is hidden in chart mode (hideFooter), so don't reserve its height. + const contentHeight = HEADER_HEIGHT + rowCount * ROW_HEIGHT + 12; const adaptiveHeight = Math.max(MIN_TABLE_HEIGHT, Math.min(MAX_TABLE_HEIGHT, contentHeight)); // Estimate total width from columns (generous: account for type icons, sort arrows, padding) @@ -912,11 +918,27 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { const adaptiveWidth = Math.max(MIN_TABLE_WIDTH, Math.min(MAX_TABLE_WIDTH, totalColWidth + SCROLLBAR_WIDTH + 16)) + 34; return ( - - + + ); })()} + setChartRandomizeToken(x => x + 1)} + onResetOrder={() => setChartResetOrderToken(x => x + 1)} + /> ; })()} , @@ -1173,18 +1195,23 @@ const EmptyStateHero: FC<{ chartSelectionBox: React.ReactNode }> = ({ chartSelec // are highlighted this dock can summarize the selection and offer actions // that span them (e.g. "combine", "chart each"). const TableActionDock: FC<{ - chartSelectionBox: React.ReactNode; + chartSelectionBox?: React.ReactNode; tableId: string; tableName: string; rows: any[]; virtual: boolean; - gridReport?: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean } | null; + // Compact variant (smaller paddings) used below the chart-mode data table. + compact?: boolean; + gridReport?: { loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean } | null; onRandomize?: () => void; -}> = ({ chartSelectionBox, tableId, tableName, rows, virtual, gridReport, onRandomize }) => { + onResetOrder?: () => void; +}> = ({ chartSelectionBox, tableId, tableName, rows, virtual, compact, gridReport, onRandomize, onResetOrder }) => { const { t } = useTranslation(); const [anchorEl, setAnchorEl] = useState(null); const [isDownloading, setIsDownloading] = useState(false); const open = Boolean(anchorEl); + // The Quick chart action only renders when a template picker is supplied. + const showQuickChart = !!chartSelectionBox; const handleDownload = async () => { if (isDownloading) return; @@ -1221,70 +1248,86 @@ const TableActionDock: FC<{ return ( - - + {showQuickChart && ( + <> + + + + )} - {gridReport?.virtual && ( - <> - - - {gridReport.loadedCount < gridReport.rowCount - ? t('dataGrid.loadedOfTotal', { loaded: gridReport.loadedCount, total: gridReport.rowCount }) - : t('dataGrid.rowCount', { count: gridReport.rowCount })} - - {gridReport.canRandomize && ( - - - - - - )} - + + + {gridReport?.virtual + ? (gridReport.loadedCount < gridReport.rowCount + ? t('dataGrid.loadedOfTotal', { loaded: gridReport.loadedCount, total: gridReport.rowCount }) + : t('dataGrid.rowCount', { count: gridReport.rowCount })) + : t('dataGrid.rowCount', { count: rows.length })} + + {gridReport?.virtual && gridReport.canRandomize && ( + + + + + )} - setAnchorEl(null)} - anchorOrigin={{ vertical: 'top', horizontal: 'left' }} - transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} - slotProps={{ paper: { sx: { p: 2, maxWidth: 'min(90vw, 1140px)' } } }} - > - {/* Clicking a template creates the chart and this whole - empty-state unmounts; closing the popover here keeps - things tidy in the transient frame. */} - setAnchorEl(null)}> - {chartSelectionBox} - - + {showQuickChart && ( + setAnchorEl(null)} + anchorOrigin={{ vertical: 'top', horizontal: 'left' }} + transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} + slotProps={{ paper: { sx: { p: 2, maxWidth: 'min(90vw, 1140px)' } } }} + > + {/* Clicking a template creates the chart and this whole + empty-state unmounts; closing the popover here keeps + things tidy in the transient frame. */} + setAnchorEl(null)}> + {chartSelectionBox} + + + )} ); }; @@ -1310,8 +1353,9 @@ export const VisualizationViewFC: FC = function VisualizationView // Virtual-pagination state reported up from the focused-table grid, so the // bottom toolbar can show the loaded/total count and drive the random dice. - const [tableGridReport, setTableGridReport] = React.useState<{ loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean } | null>(null); + const [tableGridReport, setTableGridReport] = React.useState<{ loadedCount: number; rowCount: number; virtual: boolean; canRandomize: boolean; isRandom: boolean } | null>(null); const [tableRandomizeToken, setTableRandomizeToken] = React.useState(0); + const [tableResetOrderToken, setTableResetOrderToken] = React.useState(0); let focusedChart = allCharts.find(c => c.id == focusedChartId) as Chart; let synthesisRunning = focusedChartId ? chartSynthesisInProgress.includes(focusedChartId) : false; @@ -1444,7 +1488,7 @@ export const VisualizationViewFC: FC = function VisualizationView px: 3, pt: 2, pb: 2, boxSizing: 'border-box', }}> - + ); @@ -1460,6 +1504,7 @@ export const VisualizationViewFC: FC = function VisualizationView virtual={!!ft?.virtual} gridReport={tableGridReport} onRandomize={() => setTableRandomizeToken(x => x + 1)} + onResetOrder={() => setTableResetOrderToken(x => x + 1)} /> ); })()} From 1a6530d980969230af56ecf78f08fa450392b911 Mon Sep 17 00:00:00 2001 From: Chenglong Wang Date: Wed, 8 Jul 2026 22:29:39 -0700 Subject: [PATCH 36/36] fixes --- py-src/data_formulator/datalake/__init__.py | 4 - .../datalake/azure_blob_workspace.py | 223 +++-- .../datalake/blob_disk_cache.py | 238 ++++++ .../data_formulator/datalake/cache_manager.py | 372 -------- .../datalake/cached_azure_blob_workspace.py | 794 ------------------ pyproject.toml | 8 + .../backend/benchmarks/benchmark_workspace.py | 10 +- .../security/test_confined_dir_migration.py | 48 +- 8 files changed, 408 insertions(+), 1289 deletions(-) create mode 100644 py-src/data_formulator/datalake/blob_disk_cache.py delete mode 100644 py-src/data_formulator/datalake/cache_manager.py delete mode 100644 py-src/data_formulator/datalake/cached_azure_blob_workspace.py diff --git a/py-src/data_formulator/datalake/__init__.py b/py-src/data_formulator/datalake/__init__.py index 6199804a..1dc9a0cf 100644 --- a/py-src/data_formulator/datalake/__init__.py +++ b/py-src/data_formulator/datalake/__init__.py @@ -43,8 +43,6 @@ ) from data_formulator.datalake.workspace_manager import WorkspaceManager from data_formulator.datalake.azure_blob_workspace import AzureBlobWorkspace -from data_formulator.datalake.cached_azure_blob_workspace import CachedAzureBlobWorkspace -from data_formulator.datalake.cache_manager import GlobalCacheManager # Metadata types and operations from data_formulator.datalake.workspace_metadata import ( @@ -92,8 +90,6 @@ "Workspace", "WorkspaceWithTempData", "AzureBlobWorkspace", - "CachedAzureBlobWorkspace", - "GlobalCacheManager", "get_data_formulator_home", "get_default_workspace_root", "get_user_home", diff --git a/py-src/data_formulator/datalake/azure_blob_workspace.py b/py-src/data_formulator/datalake/azure_blob_workspace.py index 52fc9400..245a7fab 100644 --- a/py-src/data_formulator/datalake/azure_blob_workspace.py +++ b/py-src/data_formulator/datalake/azure_blob_workspace.py @@ -64,6 +64,21 @@ logger = logging.getLogger(__name__) +def _data_cache_ttl() -> float: + """Seconds a cached *data* blob may be served without re-validating. + + Metadata is always re-validated (TTL 0). Data blobs (parquet) are + effectively immutable per table version, so a small TTL lets rapid repeat + reads (agent tool loops, UI refreshes) skip Azure entirely. Override with + ``AZURE_BLOB_CACHE_TTL_SECONDS``. + """ + try: + return max(0.0, float(os.getenv("AZURE_BLOB_CACHE_TTL_SECONDS", "3"))) + except (TypeError, ValueError): + return 3.0 + + + class AzureBlobWorkspace(Workspace): """ Workspace backed by Azure Blob Storage. @@ -111,6 +126,7 @@ def __init__( # --- blob storage ---------------------------------------------------- self._container: ContainerClient = container_client + self._container_name = getattr(container_client, "container_name", "") or "" if blob_prefix is not None: # Direct prefix mode (used by AzureBlobWorkspaceManager) self._datalake_root = "" @@ -141,6 +157,13 @@ def __init__( scratch_base.mkdir(parents=True, exist_ok=True) self._scratch_dir = scratch_base self._confined_scratch = ConfinedDir(scratch_base, mkdir=False) + # Blob storage has no local workspace root, so ``confined_root`` + # (inherited from :class:`Workspace`, returns ``self._confined_root``) + # would otherwise be undefined. Point it at the same local scratch jail + # so agents that read/list from the workspace root (e.g. the + # data-loading chat's read_file/list_directory tools) operate on the + # local working dir instead of raising AttributeError. + self._confined_root = ConfinedDir(scratch_base, mkdir=False) # --- in-memory metadata cache ---------------------------------------- # Avoids re-downloading workspace.yaml on every method call. @@ -153,16 +176,22 @@ def __init__( self._metadata_lock = threading.Lock() # --- blob data cache ------------------------------------------------- - # Caches downloaded blob bytes keyed by filename. Avoids repeated - # downloads of the same data file within one request (e.g. - # analyze_table calls run_parquet_sql once per column, each of - # which would otherwise re-download the entire parquet blob). + # Request-local in-memory cache of downloaded blob bytes keyed by + # filename. Avoids repeated downloads of the same data file within one + # request (e.g. analyze_table calls run_parquet_sql once per column). + # Backed by the process-global :mod:`blob_disk_cache`, which persists + # bytes + ETag across the short-lived per-request workspace instances. # Invalidated per-file on upload/delete, cleared on cleanup. self._blob_data_cache: dict[str, bytes] = {} # --- metadata -------------------------------------------------------- - if not self._blob_exists(METADATA_FILENAME): - self._init_metadata() + # Skip the existence HEAD when the metadata blob is already cached on + # disk (warm container) — this runs on every request-scoped construction. + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + + if get_blob_disk_cache().get(self._cache_key(METADATA_FILENAME)) is None: + if not self._blob_exists(METADATA_FILENAME): + self._init_metadata() logger.debug("Initialized AzureBlobWorkspace at %s", self._prefix) @@ -178,6 +207,10 @@ def _data_blob_key(self, filename: str) -> str: """Blob-internal key for a data file (under data/ subdirectory).""" return f"data/{filename}" + def _cache_key(self, filename: str) -> str: + """Globally-unique key for the disk cache: container + full blob name.""" + return f"{self._container_name}/{self._blob_name(filename)}" + def _get_blob(self, filename: str): """Return a ``BlobClient`` for *filename*.""" return self._container.get_blob_client(self._blob_name(filename)) @@ -193,25 +226,79 @@ def _blob_exists(self, filename: str) -> bool: def _upload_bytes( self, filename: str, data: bytes | str, *, overwrite: bool = True ) -> int: - """Upload *data* to blob. Returns size in bytes.""" + """Upload *data* to blob. Returns size in bytes. + + Write-through: the disk cache is updated with the new bytes + ETag so + this worker serves the fresh copy immediately without a round trip. + """ + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + raw = data.encode("utf-8") if isinstance(data, str) else data - self._get_blob(filename).upload_blob(raw, overwrite=overwrite) - # Invalidate cached copy of this file + resp = self._get_blob(filename).upload_blob(raw, overwrite=overwrite) + cache = get_blob_disk_cache() + key = self._cache_key(filename) + etag = resp.get("etag") if isinstance(resp, dict) else None + if etag: + cache.put(key, raw, etag) + else: + cache.invalidate(key) + # Invalidate request-local copy of this file self._blob_data_cache.pop(filename, None) if hasattr(self, "_temp_file_cache") and filename in self._temp_file_cache: self._temp_file_cache.pop(filename).unlink(missing_ok=True) return len(raw) + def _ensure_cached(self, filename: str): + """Return a disk-cache :class:`CacheEntry` for *filename*, fetching or + re-validating from Azure only when necessary. + + - Fresh within TTL (data blobs only) → served from disk, no Azure call. + - Otherwise revalidate via a cheap ``get_blob_properties`` HEAD (no data + transfer): matching ETag → keep the cached bytes; changed ETag → full + download to refresh the cache. + - Cold cache → full download. + Raises ``ResourceNotFoundError`` if the blob does not exist. + + Note: we deliberately avoid a conditional ``download_blob`` here. The + ``StorageStreamDownloader`` injects an ``If-Match`` on multi-chunk + continuation requests, which combined with an ``If-None-Match`` makes + large (multi-chunk) downloads fail with ``ResourceModifiedError``. + Comparing ETags ourselves after a HEAD is equally cheap and robust. + """ + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + + cache = get_blob_disk_cache() + key = self._cache_key(filename) + ttl = 0.0 if filename == METADATA_FILENAME else _data_cache_ttl() + blob = self._get_blob(filename) + + entry = cache.get(key) + if entry is not None: + if cache.is_fresh(key, ttl): + return entry + # Cheap revalidation: compare ETags via a HEAD (no data transfer). + if blob.get_blob_properties().etag == entry.etag: + cache.mark_validated(key) + return entry + + # Cold cache or changed blob — full download. + stream = blob.download_blob() + data = stream.readall() + return cache.put(key, data, stream.properties.etag) + def _download_bytes(self, filename: str) -> bytes: cached = self._blob_data_cache.get(filename) if cached is not None: return cached - data = self._get_blob(filename).download_blob().readall() + data = self._ensure_cached(filename).read_bytes() self._blob_data_cache[filename] = data return data def _delete_blob(self, filename: str) -> None: + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + self._get_blob(filename).delete_blob() + get_blob_disk_cache().invalidate(self._cache_key(filename)) self._blob_data_cache.pop(filename, None) if hasattr(self, "_temp_file_cache") and filename in self._temp_file_cache: self._temp_file_cache.pop(filename).unlink(missing_ok=True) @@ -220,27 +307,14 @@ def _delete_blob(self, filename: str) -> None: def _temp_local_copy(self, filename: str): """Yield a local file path containing the blob's data. - The file is cached on disk for the lifetime of this workspace - instance so that repeated calls (e.g. ``run_parquet_sql`` once - per column in ``analyze_table``) don't re-write the temp file. - The cache is keyed by filename and cleaned up when the instance - is garbage-collected or when :meth:`cleanup` is called. + Backed by the process-global disk cache: the yielded path points at the + cached ``.bin`` file (ETag-validated against Azure), so repeated calls + across requests reuse the same local file with no re-download. Callers + only read the file (DuckDB / pyarrow), so sharing the cached path is + safe. """ - if not hasattr(self, "_temp_file_cache"): - self._temp_file_cache: dict[str, Path] = {} - - tmp_path = self._temp_file_cache.get(filename) - if tmp_path is None or not tmp_path.exists(): - data = self._download_bytes(filename) - suffix = Path(filename).suffix - tmp = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) - tmp.write(data) - tmp.close() - tmp_path = Path(tmp.name) - self._temp_file_cache[filename] = tmp_path - - yield tmp_path - # Don't delete — reused across calls, cleaned up on GC / cleanup() + entry = self._ensure_cached(filename) + yield entry.path def _cleanup_temp_files(self) -> None: """Remove all cached temp files from disk.""" @@ -384,8 +458,12 @@ def _cleanup(m: WorkspaceMetadata) -> None: def cleanup(self) -> None: """Delete **all** blobs under this workspace's prefix.""" + from data_formulator.datalake.blob_disk_cache import get_blob_disk_cache + + cache = get_blob_disk_cache() for blob in self._container.list_blobs(name_starts_with=self._prefix): self._container.delete_blob(blob.name) + cache.invalidate(f"{self._container_name}/{blob.name}") self._metadata_cache = None self._blob_data_cache.clear() self._cleanup_temp_files() @@ -397,19 +475,25 @@ def cleanup(self) -> None: # ------------------------------------------------------------------ def read_data_as_df(self, table_name: str) -> pd.DataFrame: + from azure.core.exceptions import ResourceNotFoundError + meta = self.get_table_metadata(table_name) if meta is None: raise FileNotFoundError(f"Table not found: {table_name}") - if not self._blob_exists(self._data_blob_key(meta.filename)): + # Read straight from the ETag-validated disk cache path (no redundant + # existence HEAD, no full in-memory copy); the download validates + # existence and raises ResourceNotFoundError if the blob is gone. + try: + entry = self._ensure_cached(self._data_blob_key(meta.filename)) + except ResourceNotFoundError: raise FileNotFoundError(f"Blob not found: {meta.filename}") - buf = io.BytesIO(self._download_bytes(self._data_blob_key(meta.filename))) readers = { - "parquet": lambda b: pd.read_parquet(b), - "csv": lambda b: pd.read_csv(b), - "excel": lambda b: pd.read_excel(b), - "json": lambda b: pd.read_json(b), - "txt": lambda b: pd.read_csv(b, sep="\t"), + "parquet": lambda p: pd.read_parquet(p), + "csv": lambda p: pd.read_csv(p), + "excel": lambda p: pd.read_excel(p), + "json": lambda p: pd.read_json(p), + "txt": lambda p: pd.read_csv(p, sep="\t"), } reader = readers.get(meta.file_type) if reader is None: @@ -417,7 +501,7 @@ def read_data_as_df(self, table_name: str) -> pd.DataFrame: f"Unsupported file type '{meta.file_type}' for table '{table_name}'. " f"Supported: {', '.join(readers)}." ) - return reader(buf) + return reader(entry.path) # ------------------------------------------------------------------ # Parquet write @@ -531,31 +615,34 @@ def write_parquet( # ------------------------------------------------------------------ def get_parquet_schema(self, table_name: str) -> dict: + from azure.core.exceptions import ResourceNotFoundError + meta = self.get_table_metadata(table_name) if meta is None: raise FileNotFoundError(f"Table not found: {table_name}") if meta.file_type != "parquet": raise ValueError(f"Table {table_name} is not a parquet file") - if not self._blob_exists(self._data_blob_key(meta.filename)): - raise FileNotFoundError(f"Parquet blob not found: {meta.filename}") - with self._temp_local_copy(self._data_blob_key(meta.filename)) as tmp_path: - pf = pq.ParquetFile(tmp_path) - schema = pf.schema_arrow - return { - "table_name": table_name, - "filename": meta.filename, - "num_rows": pf.metadata.num_rows, - "num_columns": len(schema), - "columns": [ - {"name": f.name, "type": str(f.type), "nullable": f.nullable} - for f in schema - ], - "created_at": meta.created_at.isoformat(), - "last_synced": ( - meta.last_synced.isoformat() if meta.last_synced else None - ), - } + try: + entry = self._ensure_cached(self._data_blob_key(meta.filename)) + except ResourceNotFoundError: + raise FileNotFoundError(f"Parquet blob not found: {meta.filename}") + pf = pq.ParquetFile(entry.path) + schema = pf.schema_arrow + return { + "table_name": table_name, + "filename": meta.filename, + "num_rows": pf.metadata.num_rows, + "num_columns": len(schema), + "columns": [ + {"name": f.name, "type": str(f.type), "nullable": f.nullable} + for f in schema + ], + "created_at": meta.created_at.isoformat(), + "last_synced": ( + meta.last_synced.isoformat() if meta.last_synced else None + ), + } def get_parquet_path(self, table_name: str) -> str: # type: ignore[override] """Return the full blob name for the parquet file. @@ -580,25 +667,27 @@ def run_parquet_sql(self, table_name: str, sql: str) -> pd.DataFrame: the query, so DuckDB can use its native parquet reader. """ import duckdb + from azure.core.exceptions import ResourceNotFoundError meta = self.get_table_metadata(table_name) if meta is None: raise FileNotFoundError(f"Table not found: {table_name}") if meta.file_type != "parquet": raise ValueError(f"Table {table_name} is not a parquet file") - if not self._blob_exists(self._data_blob_key(meta.filename)): - raise FileNotFoundError(f"Parquet blob not found: {meta.filename}") if "{parquet}" not in sql: raise ValueError("SQL must contain {parquet} placeholder") - with self._temp_local_copy(self._data_blob_key(meta.filename)) as tmp_path: - escaped = str(tmp_path).replace("\\", "\\\\").replace("'", "''") - full_sql = sql.format(parquet=f"read_parquet('{escaped}')") - conn = duckdb.connect(":memory:") - try: - return conn.execute(full_sql).fetchdf() - finally: - conn.close() + try: + entry = self._ensure_cached(self._data_blob_key(meta.filename)) + except ResourceNotFoundError: + raise FileNotFoundError(f"Parquet blob not found: {meta.filename}") + escaped = str(entry.path).replace("\\", "\\\\").replace("'", "''") + full_sql = sql.format(parquet=f"read_parquet('{escaped}')") + conn = duckdb.connect(":memory:") + try: + return conn.execute(full_sql).fetchdf() + finally: + conn.close() # ------------------------------------------------------------------ # Local directory materialisation (for sandbox execution) diff --git a/py-src/data_formulator/datalake/blob_disk_cache.py b/py-src/data_formulator/datalake/blob_disk_cache.py new file mode 100644 index 00000000..c4090b31 --- /dev/null +++ b/py-src/data_formulator/datalake/blob_disk_cache.py @@ -0,0 +1,238 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Process-persistent, ETag-validated local disk cache for Azure Blob reads. + +Azure blob workspaces build a *fresh* :class:`AzureBlobWorkspace` on every +request, so their per-instance in-memory caches are always cold and every +request re-downloads ``workspace.yaml`` and data blobs (parquet files can be +many megabytes). This module provides a single, process-global cache that +survives across those short-lived instances (and across requests within a +worker container), backed by real files on local disk. + +Layout under ``/blob_cache/``:: + + .bin # the blob bytes + .meta.json # {"key", "blob_name", "container", "etag", "size", "cached_at"} + +The cache stores ``bytes + etag``. Freshness (whether a conditional GET is +needed) is tracked *in memory per process* via a monotonic timestamp, so we +never write to disk just to record a read. Callers (the workspace) decide the +TTL and issue conditional GETs; this module only stores/serves bytes and etags +and tracks last-validation times. + +Eviction is best-effort LRU by total bytes, capped by +``AZURE_BLOB_CACHE_MAX_BYTES`` (default 2 GiB). +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from data_formulator.datalake.workspace import get_data_formulator_home + +logger = logging.getLogger(__name__) + +CACHE_DIR_NAME = "blob_cache" +_DEFAULT_MAX_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB + + +@dataclass +class CacheEntry: + """A cached blob: its bytes live at ``path``, validated by ``etag``.""" + + key: str + path: Path + etag: str + size: int + + def read_bytes(self) -> bytes: + return self.path.read_bytes() + + +class BlobDiskCache: + """Thread-safe on-disk cache of blob bytes keyed by ``container/blob_name``. + + Safe to share across threads. Multiple *processes* (gunicorn workers) + share the same directory; writes are atomic (temp + ``os.replace``) so a + concurrent reader never sees a half-written file. In-memory bookkeeping + (index, validation timestamps, total size) is per-process — that only + affects eviction accounting and TTL freshness, both of which are + best-effort and remain correct via ETag validation. + """ + + def __init__(self, root: Path, max_bytes: int = _DEFAULT_MAX_BYTES) -> None: + self._root = root + self._max_bytes = max_bytes + self._lock = threading.RLock() + self._index: dict[str, CacheEntry] = {} + self._validated_at: dict[str, float] = {} + self._total_bytes = 0 + self._root.mkdir(parents=True, exist_ok=True) + self._load_index() + + # ------------------------------------------------------------------ + # Paths / keys + # ------------------------------------------------------------------ + + @staticmethod + def _stem(key: str) -> str: + return hashlib.sha256(key.encode("utf-8")).hexdigest() + + def _bin_path(self, key: str) -> Path: + return self._root / f"{self._stem(key)}.bin" + + def _meta_path(self, key: str) -> Path: + return self._root / f"{self._stem(key)}.meta.json" + + # ------------------------------------------------------------------ + # Index bootstrap + # ------------------------------------------------------------------ + + def _load_index(self) -> None: + """Populate the in-memory index by scanning existing meta files.""" + try: + meta_files = list(self._root.glob("*.meta.json")) + except OSError: + return + for meta_file in meta_files: + try: + meta = json.loads(meta_file.read_text(encoding="utf-8")) + key = meta["key"] + bin_path = self._bin_path(key) + if not bin_path.exists(): + continue + size = int(meta.get("size", bin_path.stat().st_size)) + self._index[key] = CacheEntry( + key=key, path=bin_path, etag=meta["etag"], size=size + ) + self._total_bytes += size + except Exception: + logger.debug("blob cache: skipping bad meta %s", meta_file, exc_info=True) + + # ------------------------------------------------------------------ + # Read side + # ------------------------------------------------------------------ + + def get(self, key: str) -> Optional[CacheEntry]: + """Return the cached entry for *key*, or ``None`` if absent.""" + with self._lock: + entry = self._index.get(key) + if entry is not None and entry.path.exists(): + return entry + if entry is not None: + # bin vanished underneath us — drop the stale index record + self._drop_locked(key) + return None + + def is_fresh(self, key: str, ttl_seconds: float) -> bool: + """Whether *key* was validated within the last ``ttl_seconds``.""" + if ttl_seconds <= 0: + return False + with self._lock: + last = self._validated_at.get(key) + return last is not None and (time.monotonic() - last) < ttl_seconds + + def mark_validated(self, key: str) -> None: + """Record that *key* was just confirmed up-to-date against Azure.""" + with self._lock: + self._validated_at[key] = time.monotonic() + + # ------------------------------------------------------------------ + # Write side + # ------------------------------------------------------------------ + + def put(self, key: str, data: bytes, etag: str) -> CacheEntry: + """Store *data*/*etag* for *key* and return the resulting entry.""" + bin_path = self._bin_path(key) + meta_path = self._meta_path(key) + self._atomic_write(bin_path, data) + meta = { + "key": key, + "etag": etag, + "size": len(data), + "cached_at": time.time(), + } + self._atomic_write( + meta_path, json.dumps(meta, ensure_ascii=False).encode("utf-8") + ) + with self._lock: + old = self._index.get(key) + if old is not None: + self._total_bytes -= old.size + entry = CacheEntry(key=key, path=bin_path, etag=etag, size=len(data)) + self._index[key] = entry + self._total_bytes += entry.size + self._validated_at[key] = time.monotonic() + self._evict_if_needed_locked() + return entry + + def invalidate(self, key: str) -> None: + """Remove *key* from the cache (disk + memory).""" + with self._lock: + self._drop_locked(key) + + # ------------------------------------------------------------------ + # Internals + # ------------------------------------------------------------------ + + @staticmethod + def _atomic_write(path: Path, data: bytes) -> None: + tmp = path.with_name(f"{path.name}.{os.getpid()}.{threading.get_ident()}.tmp") + try: + tmp.write_bytes(data) + os.replace(tmp, path) + finally: + if tmp.exists(): + tmp.unlink(missing_ok=True) + + def _drop_locked(self, key: str) -> None: + entry = self._index.pop(key, None) + if entry is not None: + self._total_bytes -= entry.size + self._validated_at.pop(key, None) + self._bin_path(key).unlink(missing_ok=True) + self._meta_path(key).unlink(missing_ok=True) + + def _evict_if_needed_locked(self) -> None: + if self._total_bytes <= self._max_bytes: + return + # Evict least-recently-validated first; entries never validated this + # process (timestamp 0) go first. + candidates = sorted( + self._index.keys(), + key=lambda k: self._validated_at.get(k, 0.0), + ) + for key in candidates: + if self._total_bytes <= self._max_bytes: + break + self._drop_locked(key) + + +_cache_singleton: Optional[BlobDiskCache] = None +_singleton_lock = threading.Lock() + + +def get_blob_disk_cache() -> BlobDiskCache: + """Return the process-global :class:`BlobDiskCache` (created on first use).""" + global _cache_singleton + if _cache_singleton is None: + with _singleton_lock: + if _cache_singleton is None: + try: + max_bytes = int( + os.getenv("AZURE_BLOB_CACHE_MAX_BYTES", str(_DEFAULT_MAX_BYTES)) + ) + except ValueError: + max_bytes = _DEFAULT_MAX_BYTES + root = get_data_formulator_home() / CACHE_DIR_NAME + _cache_singleton = BlobDiskCache(root, max_bytes=max_bytes) + return _cache_singleton diff --git a/py-src/data_formulator/datalake/cache_manager.py b/py-src/data_formulator/datalake/cache_manager.py deleted file mode 100644 index b7cc8948..00000000 --- a/py-src/data_formulator/datalake/cache_manager.py +++ /dev/null @@ -1,372 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -""" -Global cache manager for multi-user deployments. - -While each :class:`CachedAzureBlobWorkspace` enforces its own per-workspace -limit (default 1 GB), this module enforces a **server-wide ceiling** across -ALL user caches, preventing the aggregate local cache from consuming all -available disk space when many users are active. - -Architecture -~~~~~~~~~~~~ -``GlobalCacheManager`` is a **thread-safe singleton**. The first call to -:meth:`get_instance` configures it (cache root, max bytes, scan interval); -subsequent calls return the same object. - -Cross-user eviction -~~~~~~~~~~~~~~~~~~~ -When the global limit is exceeded, files are evicted across *all* user -cache directories using **LRU by mtime** (oldest files first, regardless -of which user owns them). Protected files (``workspace.yaml``) are never -evicted. Eviction targets 80 % of the global max to avoid thrashing. - -Graceful degradation -~~~~~~~~~~~~~~~~~~~~ -When the global cache is full and eviction cannot free enough space, -individual workspaces fall through to direct Azure reads. User-initiated -writes always succeed locally (correctness), but read-path caching is -skipped so the disk isn't filled further. - -Disk scanning -~~~~~~~~~~~~~ -Total disk usage is computed via ``os.walk()`` over the cache root. -To avoid excessive I/O on servers with many files, the scan is -**debounced** — at most once per ``scan_interval`` seconds (default 10 s). -""" - -from __future__ import annotations - -import logging -import os -import threading -import time -from pathlib import Path -from typing import Any - -from data_formulator.datalake.workspace_metadata import METADATA_FILENAME - -logger = logging.getLogger(__name__) - -# Default global cache limit: 50 GB -_DEFAULT_GLOBAL_MAX_BYTES = 50 * 1024**3 - -# Default interval between filesystem scans (seconds) -_DEFAULT_SCAN_INTERVAL = 10.0 - - -class GlobalCacheManager: - """Thread-safe singleton managing total cache disk usage across all users. - - Usage:: - - mgr = GlobalCacheManager.get_instance( - cache_root=Path("~/.data_formulator/cache"), - max_global_bytes=10 * 1024**3, # 10 GB - ) - - # Before caching a downloaded file (optional write): - if mgr.try_acquire_space(len(data)): - cache_file.write_bytes(data) - - # After a mandatory write (e.g. user upload): - cache_file.write_bytes(data) - mgr.notify_write(len(data)) - - # Monitoring: - stats = mgr.get_global_stats() - """ - - _instance: GlobalCacheManager | None = None - _init_lock = threading.Lock() - - # ------------------------------------------------------------------ - # Singleton access - # ------------------------------------------------------------------ - - @classmethod - def get_instance( - cls, - cache_root: Path | str | None = None, - max_global_bytes: int = _DEFAULT_GLOBAL_MAX_BYTES, - scan_interval: float = _DEFAULT_SCAN_INTERVAL, - ) -> GlobalCacheManager: - """Return the singleton, creating it on first call. - - Args: - cache_root: Root of the local cache tree. Defaults to - ``~/.data_formulator/cache``. - max_global_bytes: Global ceiling in bytes (default 10 GB). - scan_interval: Min seconds between full filesystem scans - (default 10). - - Subsequent calls return the existing singleton — arguments are - ignored after the first call. - """ - if cls._instance is not None: - return cls._instance - with cls._init_lock: - if cls._instance is not None: - return cls._instance - if cache_root is None: - from data_formulator.datalake.workspace import ( - get_data_formulator_home, - ) - - cache_root = get_data_formulator_home() / "cache" - cls._instance = cls( - Path(cache_root), max_global_bytes, scan_interval - ) - return cls._instance - - @classmethod - def reset_instance(cls) -> None: - """Discard the singleton (for testing only).""" - with cls._init_lock: - cls._instance = None - - # ------------------------------------------------------------------ - # Construction (private — use get_instance) - # ------------------------------------------------------------------ - - def __init__( - self, - cache_root: Path, - max_global_bytes: int, - scan_interval: float, - ): - self._cache_root = cache_root - self._max_global_bytes = max_global_bytes - self._scan_interval = scan_interval - - self._lock = threading.Lock() - self._last_scan_time: float = 0.0 - self._cached_total_bytes: int = 0 - - logger.info( - "GlobalCacheManager: root=%s max=%d MB scan_interval=%.1fs", - cache_root, - max_global_bytes // (1024 * 1024), - scan_interval, - ) - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def max_global_bytes(self) -> int: - return self._max_global_bytes - - @property - def cache_root(self) -> Path: - return self._cache_root - - # ------------------------------------------------------------------ - # Disk scanning (debounced) - # ------------------------------------------------------------------ - - def _scan_total_size(self) -> int: - """Walk the cache root and sum file sizes. - - Debounced: returns the cached value if the last scan was less - than ``scan_interval`` seconds ago. - - **Must be called with ``self._lock`` held.** - """ - now = time.monotonic() - if now - self._last_scan_time < self._scan_interval: - return self._cached_total_bytes - - total = 0 - try: - for dirpath, _dirnames, filenames in os.walk(self._cache_root): - for fn in filenames: - try: - total += os.path.getsize(os.path.join(dirpath, fn)) - except OSError: - pass - except OSError: - pass - - self._cached_total_bytes = total - self._last_scan_time = now - return total - - # ------------------------------------------------------------------ - # Space management - # ------------------------------------------------------------------ - - def try_acquire_space(self, needed_bytes: int) -> bool: - """Try to make room for *needed_bytes* of new cache data. - - 1. If total + needed is under the limit, return ``True``. - 2. Otherwise, run cross-user LRU eviction. - 3. If still insufficient, return ``False`` (caller should skip - local caching and serve from Azure directly). - - This is intended for **optional** cache writes (e.g. caching a - download). For **mandatory** writes (user uploads), use - :meth:`notify_write` after writing instead. - """ - with self._lock: - total = self._scan_total_size() - if total + needed_bytes <= self._max_global_bytes: - self._cached_total_bytes = total + needed_bytes - return True - - # Try cross-user eviction - freed = self._evict_global_unlocked( - target_free=needed_bytes + int(self._max_global_bytes * 0.1) - ) - - # Re-scan after eviction - if freed > 0: - self._last_scan_time = 0.0 # force fresh scan - total = self._scan_total_size() - if total + needed_bytes <= self._max_global_bytes: - self._cached_total_bytes = total + needed_bytes - return True - - return False - - def notify_write(self, nbytes: int) -> None: - """Notify the manager of a mandatory write (e.g. user upload). - - The write has already happened. This bumps the cached counter - and triggers global eviction if the limit is exceeded. - """ - with self._lock: - self._cached_total_bytes += nbytes - if self._cached_total_bytes > self._max_global_bytes: - self._evict_global_unlocked( - target_free=int( - self._cached_total_bytes - - self._max_global_bytes * 0.8 - ) - ) - - def maybe_evict_global(self) -> int: - """Run cross-user eviction if total exceeds the global limit. - - Returns bytes freed. - """ - with self._lock: - total = self._scan_total_size() - if total <= self._max_global_bytes: - return 0 - return self._evict_global_unlocked( - target_free=int(total - self._max_global_bytes * 0.8) - ) - - # ------------------------------------------------------------------ - # Cross-user LRU eviction (internal) - # ------------------------------------------------------------------ - - def _evict_global_unlocked(self, target_free: int) -> int: - """Evict files across all user caches, LRU by mtime. - - **Must be called with ``self._lock`` held.** - - Skips: - * ``workspace.yaml`` — correctness-critical metadata. - * Hidden files (starting with ``.``). - - Args: - target_free: Bytes to free. - - Returns: - Total bytes actually freed. - """ - if target_free <= 0: - return 0 - - # Collect all candidate files across all user caches - candidates: list[tuple[str, float, int]] = [] - try: - for dirpath, _, filenames in os.walk(self._cache_root): - for fn in filenames: - if fn == METADATA_FILENAME: - continue - if fn.startswith("."): - continue - full = os.path.join(dirpath, fn) - try: - st = os.stat(full) - candidates.append((full, st.st_mtime, st.st_size)) - except OSError: - pass - except OSError: - return 0 - - # Sort by mtime ascending (oldest first = evict first) - candidates.sort(key=lambda x: x[1]) - - freed = 0 - evicted = 0 - for full_path, _mtime, size in candidates: - if freed >= target_free: - break - try: - os.unlink(full_path) - freed += size - evicted += 1 - except OSError: - pass - - if evicted: - logger.info( - "Global cache eviction: removed %d file(s), freed %.1f MB " - "(target was %.1f MB)", - evicted, - freed / (1024 * 1024), - target_free / (1024 * 1024), - ) - # Invalidate cached scan so next check is accurate - self._last_scan_time = 0.0 - - return freed - - # ------------------------------------------------------------------ - # Monitoring - # ------------------------------------------------------------------ - - def get_global_stats(self) -> dict[str, Any]: - """Return global cache statistics for monitoring / debugging.""" - with self._lock: - total = self._scan_total_size() - - # Count user-level cache directories (root/datalake_root/user_id/) - user_dirs = 0 - try: - for entry in os.scandir(self._cache_root): - if entry.is_dir(): - for sub in os.scandir(entry.path): - if sub.is_dir(): - user_dirs += 1 - except OSError: - pass - - return { - "cache_root": str(self._cache_root), - "total_size_bytes": total, - "total_size_mb": round(total / (1024 * 1024), 2), - "max_size_mb": round(self._max_global_bytes / (1024 * 1024), 2), - "utilization_pct": ( - round(total / self._max_global_bytes * 100, 1) - if self._max_global_bytes > 0 - else 0 - ), - "user_cache_count": user_dirs, - } - - # ------------------------------------------------------------------ - # Representation - # ------------------------------------------------------------------ - - def __repr__(self) -> str: - return ( - f"GlobalCacheManager(root={self._cache_root!r}, " - f"max={self._max_global_bytes // (1024**2)} MB)" - ) diff --git a/py-src/data_formulator/datalake/cached_azure_blob_workspace.py b/py-src/data_formulator/datalake/cached_azure_blob_workspace.py deleted file mode 100644 index 3a121bf4..00000000 --- a/py-src/data_formulator/datalake/cached_azure_blob_workspace.py +++ /dev/null @@ -1,794 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -""" -Cached Azure Blob workspace with persistent local file mirror. - -Wraps :class:`AzureBlobWorkspace` with a **write-through local cache** -under ``~/.data_formulator/cache/``. Reads come from the local mirror -(filesystem speed), writes go to the local mirror immediately and are -uploaded to Azure Blob Storage in a background thread. - -Key performance improvements over plain ``AzureBlobWorkspace``: - -* ``read_data_as_df()`` — reads local parquet directly (no blob download) -* ``local_dir()`` — yields the cache directory (no temp-dir downloads) -* ``run_parquet_sql()`` — runs DuckDB against local parquet (no temp copy) -* ``write_parquet()`` — writes local file immediately, Azure upload in bg - -Cache eviction --------------- -An LRU eviction mechanism prevents unbounded disk growth: - -* **Max size** — configurable, default 1 GB per workspace. -* **Trigger** — checked after every write. -* **Policy** — evict least-recently-used files (oldest ``mtime``) until - total cache size drops to 80 % of the max. -* **Protected** — ``workspace.yaml`` and files with pending background - uploads are never evicted. - -Usage:: - - from azure.storage.blob import ContainerClient - from data_formulator.datalake.cached_azure_blob_workspace import ( - CachedAzureBlobWorkspace, - ) - - container = ContainerClient.from_connection_string(conn_str, "my-container") - ws = CachedAzureBlobWorkspace( - "user:42", container, - datalake_root="workspaces", - max_cache_bytes=2 * 1024**3, # 2 GB cache - ) -""" - -from __future__ import annotations - -import atexit -import collections -import io -import logging -import os -import shutil -import threading -import time -from concurrent.futures import Future, ThreadPoolExecutor -from contextlib import contextmanager -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Optional, TYPE_CHECKING - -import pandas as pd -import pyarrow as pa -import pyarrow.parquet as pq -import yaml - -from data_formulator.datalake.azure_blob_workspace import AzureBlobWorkspace -from data_formulator.datalake.cache_manager import GlobalCacheManager -from data_formulator.datalake.workspace_metadata import ( - METADATA_FILENAME, - WorkspaceMetadata, -) -from data_formulator.datalake.parquet_utils import sanitize_table_name -from data_formulator.datalake.workspace import Workspace, get_data_formulator_home -from data_formulator.security.path_safety import ConfinedDir - -if TYPE_CHECKING: - from azure.storage.blob import ContainerClient - -logger = logging.getLogger(__name__) - -# Default cache size limit per workspace (1 GB). -_DEFAULT_MAX_CACHE_BYTES = 1 * 1024 ** 3 - - -class CachedAzureBlobWorkspace(AzureBlobWorkspace): - """Azure Blob workspace with a persistent local file cache. - - Every file written to Azure is also written to a local cache directory. - Reads are served from the local cache whenever possible — falling back - to Azure only for files that have been evicted or written by another - process. - - Background uploads - ~~~~~~~~~~~~~~~~~~ - Data-file uploads are submitted to a :class:`ThreadPoolExecutor` so - they don't block the request thread. Metadata (``workspace.yaml``) - is always uploaded **synchronously** because it is small and - correctness-critical. Call :meth:`wait_for_uploads` to block until - all pending uploads finish (e.g. before shutdown or tests). - - Multi-instance safety - ~~~~~~~~~~~~~~~~~~~~~~ - When the same user is served by multiple server instances (e.g. - behind a load balancer), each instance has its own local cache. - Stale-cache detection compares the local ``workspace.yaml``'s - ``updated_at`` timestamp against Azure on a configurable interval - (default: 30 s). If Azure is newer, the local metadata and any - changed data files are re-downloaded. - - Global cache budget - ~~~~~~~~~~~~~~~~~~~~ - A :class:`GlobalCacheManager` singleton enforces a server-wide - ceiling (default 10 GB) across **all** user caches. When the - global limit is exceeded, cross-user LRU eviction removes the - oldest files server-wide. If eviction cannot free enough space, - download-path caching is skipped (graceful degradation) so reads - fall through to Azure Blob Storage directly. - - Thread safety - ~~~~~~~~~~~~~ - The local cache directory is per-user so there are no cross-user - conflicts. Per-file upload locks prevent concurrent background - uploads from racing on the same blob. The ``_pending_uploads`` - set is protected by a global lock. - """ - - # ------------------------------------------------------------------ - # Construction - # ------------------------------------------------------------------ - - def __init__( - self, - identity_id: str, - container_client: "ContainerClient", - datalake_root: str = "", - *, - blob_prefix: str | None = None, - cache_root: str | Path | None = None, - max_cache_bytes: int = _DEFAULT_MAX_CACHE_BYTES, - max_global_cache_bytes: int | None = None, - bg_upload_workers: int = 2, - staleness_check_interval: float = 30.0, - ): - """ - Args: - identity_id: Unique user identifier. - container_client: Azure ``ContainerClient``. - datalake_root: Path prefix inside the blob container. - cache_root: Root of the local cache tree. Defaults to - ``~/.data_formulator/cache``. - max_cache_bytes: Maximum total size of cached files for this - workspace before LRU eviction kicks in. - max_global_cache_bytes: Server-wide ceiling across **all** - user caches (default: 10 GB). ``None`` uses the - :class:`GlobalCacheManager` default. - bg_upload_workers: Thread pool size for background uploads. - staleness_check_interval: Seconds between Azure metadata - freshness checks (default: 30). Set to 0 to check on - every ``get_metadata()`` call. - """ - # We must set up the cache directory *before* calling - # super().__init__ because the parent's __init__ may call - # _upload_bytes (via _init_metadata / save_metadata). - safe_id = Workspace._sanitize_identity_id(identity_id) - - if cache_root is None: - cache_root = get_data_formulator_home() / "cache" - base = Path(cache_root) - - if blob_prefix is not None: - # Multi-workspace mode: cache dir based on blob prefix - safe_prefix = blob_prefix.strip("/").replace("/", os.sep) - self._cache_dir = base / safe_prefix - else: - root = datalake_root.strip("/") - if root: - self._cache_dir = base / root / safe_id - else: - self._cache_dir = base / safe_id - self._cache_dir.mkdir(parents=True, exist_ok=True) - self._cache_jail = ConfinedDir(self._cache_dir, mkdir=False) - - self._max_cache_bytes = max_cache_bytes - - # Background upload machinery - self._pending_uploads: set[str] = set() - self._upload_lock = threading.Lock() - # Per-file locks to serialise concurrent uploads to the same blob - self._file_locks: dict[str, threading.Lock] = collections.defaultdict( - threading.Lock - ) - self._upload_executor = ThreadPoolExecutor( - max_workers=bg_upload_workers, - thread_name_prefix="df_cache_upload", - ) - self._upload_futures: list[Future] = [] - # Register atexit hook to flush pending uploads on interpreter exit - atexit.register(self._atexit_flush) - - # Staleness detection for multi-instance deployments - self._staleness_check_interval = staleness_check_interval - self._last_staleness_check: float = 0.0 # epoch seconds - self._local_metadata_updated_at: Optional[datetime] = None - - # Initialise the global cache manager (singleton) — must be - # before super().__init__ because _upload_bytes references it. - gcm_kwargs: dict[str, Any] = {"cache_root": base} - if max_global_cache_bytes is not None: - gcm_kwargs["max_global_bytes"] = max_global_cache_bytes - self._global_cache = GlobalCacheManager.get_instance(**gcm_kwargs) - - # Now safe to call super().__init__ — our _upload_bytes / etc. - # overrides are in place and _cache_dir is ready. - super().__init__(identity_id, container_client, datalake_root, blob_prefix=blob_prefix) - - # Run initial eviction if cache is over-sized (e.g. from prev run) - self._maybe_evict() - - logger.info( - "Initialized CachedAzureBlobWorkspace: prefix=%s cache=%s " - "max=%d MB global_max=%d MB", - self._prefix, - self._cache_dir, - self._max_cache_bytes // (1024 * 1024), - self._global_cache.max_global_bytes // (1024 * 1024), - ) - - # ------------------------------------------------------------------ - # Cache path helper - # ------------------------------------------------------------------ - - def _cache_path(self, filename: str) -> Path: - """Return the local cache path for *filename*. - - Delegates to :class:`ConfinedDir` which raises ``ValueError`` - if the resolved path escapes the cache directory. - """ - return self._cache_jail.resolve(filename) - - # ------------------------------------------------------------------ - # Low-level blob overrides (write-through cache) - # ------------------------------------------------------------------ - - def _upload_bytes( - self, - filename: str, - data: bytes | str, - *, - overwrite: bool = True, - ) -> int: - """Write *data* to local cache **and** Azure. - - Metadata (``workspace.yaml``) is uploaded synchronously. - Data files are uploaded in a background thread. - """ - raw = data.encode("utf-8") if isinstance(data, str) else data - - # 1. Write to local cache immediately - cache_file = self._cache_path(filename) - cache_file.parent.mkdir(parents=True, exist_ok=True) - cache_file.write_bytes(raw) - - # 2. Invalidate in-memory caches (inherited from AzureBlobWorkspace) - self._blob_data_cache.pop(filename, None) - if hasattr(self, "_temp_file_cache") and filename in self._temp_file_cache: - self._temp_file_cache.pop(filename).unlink(missing_ok=True) - - # 3. Upload to Azure - if filename == METADATA_FILENAME: - # Metadata: synchronous (small, correctness-critical) - self._get_blob(filename).upload_blob(raw, overwrite=overwrite) - else: - # Data files: background upload with per-file lock - with self._upload_lock: - self._pending_uploads.add(filename) - - def _bg_upload(fn: str, payload: bytes) -> None: - # Per-file lock ensures concurrent writes to the same - # blob are serialised — last write always wins. - file_lock = self._file_locks[fn] - file_lock.acquire() - try: - self._get_blob(fn).upload_blob(payload, overwrite=True) - logger.debug("Background upload complete: %s", fn) - except Exception: - logger.warning( - "Background upload FAILED for %s — data is safe in " - "local cache and will be retried on next write.", - fn, - exc_info=True, - ) - finally: - file_lock.release() - with self._upload_lock: - self._pending_uploads.discard(fn) - - fut = self._upload_executor.submit(_bg_upload, filename, raw) - self._upload_futures.append(fut) - - # 4. Evict if needed (per-workspace, then global) - self._maybe_evict() - # Notify global manager of the mandatory write - self._global_cache.notify_write(len(raw)) - - return len(raw) - - def _download_bytes(self, filename: str) -> bytes: - """Read from local cache first, then Azure, then populate cache.""" - # 1. Local cache hit - cache_file = self._cache_path(filename) - if cache_file.exists(): - data = cache_file.read_bytes() - # Touch to update mtime for LRU tracking - try: - os.utime(cache_file, None) - except OSError: - pass - # Also populate in-memory cache for extra speed on hot paths - self._blob_data_cache[filename] = data - return data - - # 2. In-memory cache (shouldn't happen if local cache is warm) - cached = self._blob_data_cache.get(filename) - if cached is not None: - # Backfill local cache — only if global budget allows - if self._global_cache.try_acquire_space(len(cached)): - try: - cache_file.parent.mkdir(parents=True, exist_ok=True) - cache_file.write_bytes(cached) - except OSError: - logger.debug("Failed to backfill cache for %s", filename) - return cached - - # 3. Azure download (cache miss — evicted or written by other instance) - data = self._get_blob(filename).download_blob().readall() - self._blob_data_cache[filename] = data - - # Persist to local cache only if global budget allows - if self._global_cache.try_acquire_space(len(data)): - try: - cache_file.parent.mkdir(parents=True, exist_ok=True) - cache_file.write_bytes(data) - except OSError: - logger.debug( - "Global cache full — serving %s from Azure without " - "local caching", - filename, - ) - else: - logger.debug( - "Global cache full — serving %s from Azure without " - "local caching", - filename, - ) - return data - - def _blob_exists(self, filename: str) -> bool: - """Check local cache first, then Azure.""" - if self._cache_path(filename).exists(): - return True - # Fall back to Azure (file may have been evicted) - return super()._blob_exists(filename) - - # ------------------------------------------------------------------ - # Staleness detection (multi-instance safety) - # ------------------------------------------------------------------ - - def _check_staleness(self) -> None: - """Compare local metadata timestamp with Azure's. - - If Azure has a newer ``updated_at``, invalidate local metadata - and any data files whose table metadata has changed. - Called by ``get_metadata()`` at most once per - ``staleness_check_interval``. - """ - now = time.monotonic() - if now - self._last_staleness_check < self._staleness_check_interval: - return - self._last_staleness_check = now - - try: - # Fetch fresh metadata from Azure (bypass all caches) - raw = self._get_blob(METADATA_FILENAME).download_blob().readall() - parsed = yaml.safe_load(raw) - if parsed is None: - return - remote_meta = WorkspaceMetadata.from_dict(parsed) - except Exception: - # If Azure is unreachable, use local cache silently - logger.debug("Staleness check: Azure unreachable, using local cache") - return - - local_meta = self._metadata_cache - if local_meta is None: - # No local metadata cached yet — will be loaded fresh anyway - return - - if remote_meta.updated_at <= local_meta.updated_at: - return # local is up to date - - logger.info( - "Stale cache detected: local=%s remote=%s — refreshing", - local_meta.updated_at.isoformat(), - remote_meta.updated_at.isoformat(), - ) - - # Find data files that changed (different hash or new tables) - for table_name, remote_table in remote_meta.tables.items(): - local_table = local_meta.tables.get(table_name) - if ( - local_table is None - or local_table.content_hash != remote_table.content_hash - ): - # Invalidate cached file so next read re-downloads - self._cache_path(remote_table.filename).unlink(missing_ok=True) - self._blob_data_cache.pop(remote_table.filename, None) - logger.debug("Invalidated stale cached file: %s", remote_table.filename) - - # Find tables deleted remotely - for table_name in list(local_meta.tables.keys()): - if table_name not in remote_meta.tables: - old_fn = local_meta.tables[table_name].filename - self._cache_path(old_fn).unlink(missing_ok=True) - self._blob_data_cache.pop(old_fn, None) - - # Update local metadata cache file and in-memory cache - self._cache_path(METADATA_FILENAME).write_bytes(raw) - self._metadata_cache = remote_meta - self._blob_data_cache[METADATA_FILENAME] = raw - - def _delete_blob(self, filename: str) -> None: - """Delete from local cache, in-memory caches, and Azure.""" - # Local cache - self._cache_path(filename).unlink(missing_ok=True) - - # In-memory caches - self._blob_data_cache.pop(filename, None) - if hasattr(self, "_temp_file_cache") and filename in self._temp_file_cache: - self._temp_file_cache.pop(filename).unlink(missing_ok=True) - - # Azure - try: - self._get_blob(filename).delete_blob() - except Exception: - logger.debug("Azure delete failed for %s (may not exist)", filename) - - # ------------------------------------------------------------------ - # Temp-local-copy override (use cache file directly) - # ------------------------------------------------------------------ - - # ------------------------------------------------------------------ - # Metadata override with staleness check - # ------------------------------------------------------------------ - - def get_metadata(self) -> WorkspaceMetadata: - """Return workspace metadata, checking Azure for staleness.""" - self._check_staleness() - return super().get_metadata() - - # ------------------------------------------------------------------ - # Temp-local-copy override (use cache file directly) - # ------------------------------------------------------------------ - - @contextmanager - def _temp_local_copy(self, filename: str): - """Yield the local cache path directly — no temp file needed.""" - cache_file = self._cache_path(filename) - if not cache_file.exists(): - # Ensure file is in cache - self._download_bytes(filename) - yield cache_file - - # ------------------------------------------------------------------ - # Read overrides (read directly from local cache files) - # ------------------------------------------------------------------ - - def read_data_as_df(self, table_name: str) -> pd.DataFrame: - """Read table from local cache (fast!) with Azure fallback.""" - meta = self.get_table_metadata(table_name) - if meta is None: - raise FileNotFoundError(f"Table not found: {table_name}") - - cache_file = self._cache_path(meta.filename) - - # Ensure file is in cache - if not cache_file.exists(): - # Download from Azure and populate cache - self._download_bytes(meta.filename) - - # Update mtime for LRU - try: - os.utime(cache_file, None) - except OSError: - pass - - # Read directly from local file — fastest path - readers = { - "parquet": lambda p: pd.read_parquet(p), - "csv": lambda p: pd.read_csv(p), - "excel": lambda p: pd.read_excel(p), - "json": lambda p: pd.read_json(p), - "txt": lambda p: pd.read_csv(p, sep="\t"), - } - reader = readers.get(meta.file_type) - if reader is None: - raise ValueError( - f"Unsupported file type '{meta.file_type}' for table " - f"'{table_name}'. Supported: {', '.join(readers)}." - ) - return reader(cache_file) - - def run_parquet_sql(self, table_name: str, sql: str) -> pd.DataFrame: - """Run DuckDB SQL against local cache file (no temp copy needed).""" - import duckdb - - meta = self.get_table_metadata(table_name) - if meta is None: - raise FileNotFoundError(f"Table not found: {table_name}") - if meta.file_type != "parquet": - raise ValueError(f"Table {table_name} is not a parquet file") - if "{parquet}" not in sql: - raise ValueError("SQL must contain {parquet} placeholder") - - cache_file = self._cache_path(meta.filename) - if not cache_file.exists(): - self._download_bytes(meta.filename) - - # Update mtime for LRU - try: - os.utime(cache_file, None) - except OSError: - pass - - escaped = str(cache_file).replace("\\", "\\\\").replace("'", "''") - full_sql = sql.format(parquet=f"read_parquet('{escaped}')") - conn = duckdb.connect(":memory:") - try: - return conn.execute(full_sql).fetchdf() - finally: - conn.close() - - # ------------------------------------------------------------------ - # local_dir — the biggest win - # ------------------------------------------------------------------ - - @contextmanager - def local_dir(self): - """Yield the cache directory — no temp dir, no mass downloads. - - Verifies that all workspace data files are present in the local - cache before yielding. Any missing files (evicted or written by - another instance) are downloaded on demand. - """ - self._ensure_all_cached() - yield self._cache_dir - - def _ensure_all_cached(self) -> None: - """Download any workspace files not present in local cache.""" - for blob in self._container.list_blobs(name_starts_with=self._prefix): - rel = blob.name[len(self._prefix) :] - if not rel or rel == METADATA_FILENAME: - continue - cache_file = self._cache_path(rel) - if not cache_file.exists(): - data = self._container.download_blob(blob.name).readall() - cache_file.parent.mkdir(parents=True, exist_ok=True) - cache_file.write_bytes(data) - logger.debug("local_dir: downloaded missing file %s", rel) - - # ------------------------------------------------------------------ - # Cache eviction (LRU by mtime) - # ------------------------------------------------------------------ - - def _get_cache_size(self) -> int: - """Total bytes of all files in the cache directory.""" - total = 0 - for f in self._cache_dir.iterdir(): - if f.is_file(): - try: - total += f.stat().st_size - except OSError: - pass - return total - - def _maybe_evict(self) -> None: - """Evict LRU files if cache exceeds ``max_cache_bytes``. - - Evicts down to 80 % of max. Never evicts ``workspace.yaml`` - or files with pending background uploads. - """ - total = self._get_cache_size() - if total <= self._max_cache_bytes: - return - - target = int(self._max_cache_bytes * 0.8) - - # Collect eviction candidates (sorted oldest mtime first) - candidates: list[tuple[Path, float, int]] = [] - with self._upload_lock: - pending = set(self._pending_uploads) - - for f in self._cache_dir.iterdir(): - if not f.is_file(): - continue - if f.name == METADATA_FILENAME: - continue # never evict metadata - if f.name in pending: - continue # never evict files being uploaded - try: - st = f.stat() - candidates.append((f, st.st_mtime, st.st_size)) - except OSError: - pass - - # Sort by mtime ascending (oldest first = evict first) - candidates.sort(key=lambda x: x[1]) - - evicted = 0 - freed = 0 - for path, mtime, size in candidates: - if total <= target: - break - try: - path.unlink(missing_ok=True) - total -= size - freed += size - evicted += 1 - except OSError: - pass - - if evicted: - logger.info( - "Cache eviction: removed %d file(s), freed %.1f MB, " - "remaining %.1f / %.1f MB", - evicted, - freed / (1024 * 1024), - total / (1024 * 1024), - self._max_cache_bytes / (1024 * 1024), - ) - - # Also run global cross-user eviction if needed - self._global_cache.maybe_evict_global() - - def get_cache_stats(self) -> dict[str, Any]: - """Return cache statistics for monitoring / debugging.""" - files = [] - total_size = 0 - for f in self._cache_dir.iterdir(): - if f.is_file(): - try: - st = f.stat() - files.append({"name": f.name, "size": st.st_size, "mtime": st.st_mtime}) - total_size += st.st_size - except OSError: - pass - - with self._upload_lock: - pending = list(self._pending_uploads) - - return { - "cache_dir": str(self._cache_dir), - "file_count": len(files), - "total_size_bytes": total_size, - "total_size_mb": round(total_size / (1024 * 1024), 2), - "max_size_mb": round(self._max_cache_bytes / (1024 * 1024), 2), - "utilization_pct": round(total_size / self._max_cache_bytes * 100, 1) - if self._max_cache_bytes > 0 - else 0, - "pending_uploads": pending, - "files": sorted(files, key=lambda f: f["mtime"], reverse=True), - } - - # ------------------------------------------------------------------ - # Lifecycle - # ------------------------------------------------------------------ - - def wait_for_uploads(self, timeout: float | None = 30) -> bool: - """Block until all pending background uploads complete. - - Args: - timeout: Max seconds to wait. ``None`` = wait forever. - - Returns: - ``True`` if all uploads finished, ``False`` if timeout was hit. - """ - # Collect outstanding futures - futures = [f for f in self._upload_futures if not f.done()] - if not futures: - return True - - from concurrent.futures import wait, FIRST_EXCEPTION - - done, not_done = wait(futures, timeout=timeout) - # Clean up completed futures - self._upload_futures = [f for f in self._upload_futures if not f.done()] - - if not_done: - logger.warning( - "%d background upload(s) did not complete within %.1fs", - len(not_done), - timeout or 0, - ) - return False - return True - - def _atexit_flush(self) -> None: - """Best-effort flush of pending uploads on interpreter shutdown.""" - with self._upload_lock: - if not self._pending_uploads: - return - pending_count = len(self._pending_uploads) - - logger.info("Flushing %d pending upload(s) on shutdown...", pending_count) - self.wait_for_uploads(timeout=30) - - def cleanup(self) -> None: - """Remove local cache immediately, delete Azure blobs in background. - - The local cache is cleared **synchronously** so the workspace is - immediately reusable. Azure blob deletion is submitted to the - background thread pool so the caller isn't blocked by many - sequential Azure API calls. - """ - # 1. Wait for any in-flight uploads (so we don't race with them) - self.wait_for_uploads(timeout=60) - - # 2. Clear local caches immediately (non-blocking) - if self._cache_dir.exists(): - shutil.rmtree(self._cache_dir, ignore_errors=True) - self._cache_dir.mkdir(parents=True, exist_ok=True) - self._metadata_cache = None - self._blob_data_cache.clear() - self._cleanup_temp_files() - self._cleanup_scratch() - - # 3. Delete Azure blobs in background - prefix = self._prefix - container = self._container - - def _bg_cleanup() -> None: - try: - for blob in container.list_blobs(name_starts_with=prefix): - try: - container.delete_blob(blob.name) - except Exception: - logger.debug("Failed to delete blob %s", blob.name) - logger.info("Background cleanup finished for %s", prefix) - except Exception: - logger.warning( - "Background Azure cleanup failed for %s", - prefix, - exc_info=True, - ) - - self._upload_executor.submit(_bg_cleanup) - logger.info("Cleanup: local cache cleared, Azure deletion queued for %s", self._safe_id) - - # ------------------------------------------------------------------ - # snapshot / session overrides: ensure cache consistency - # ------------------------------------------------------------------ - - def restore_workspace_snapshot(self, src: Path) -> None: - """Restore snapshot and repopulate local cache.""" - # Clear local cache first - if self._cache_dir.exists(): - shutil.rmtree(self._cache_dir, ignore_errors=True) - self._cache_dir.mkdir(parents=True, exist_ok=True) - - # Delegate to parent (uploads blobs to Azure) - super().restore_workspace_snapshot(src) - - # Repopulate cache from the source snapshot - if src.exists(): - for f in src.rglob("*"): - if f.is_file(): - rel = str(f.relative_to(src)) - cache_file = self._cache_path(rel) - cache_file.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(f, cache_file) - - def invalidate_metadata_cache(self) -> None: - """Force re-read of metadata from Azure (clears local + in-memory).""" - self._cache_path(METADATA_FILENAME).unlink(missing_ok=True) - super().invalidate_metadata_cache() - - # ------------------------------------------------------------------ - # Representation - # ------------------------------------------------------------------ - - def __repr__(self) -> str: - return ( - f"CachedAzureBlobWorkspace(identity_id={self._identity_id!r}, " - f"prefix={self._prefix!r}, cache={self._cache_dir!r})" - ) diff --git a/pyproject.toml b/pyproject.toml index 780e8f98..87254ce2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,14 @@ Repository = "https://github.com/microsoft/data-formulator.git" package-dir = {"" = "py-src"} include-package-data = true +# Non-Python resources that must ship inside the installed package. The Azure +# build runs `pip install .` from a zip (no VCS), so include-package-data alone +# does not pick these up — declare them explicitly. Analyst skills load their +# `SKILL.md` body and `tools.json` schemas at runtime from the package dir. +# Scoped to the exact skill filenames so unrelated data JSON is not bundled. +[tool.setuptools.package-data] +"*" = ["SKILL.md", "tools.json"] + [project.scripts] data_formulator = "data_formulator:run_app" diff --git a/tests/backend/benchmarks/benchmark_workspace.py b/tests/backend/benchmarks/benchmark_workspace.py index ed271deb..83c26936 100644 --- a/tests/backend/benchmarks/benchmark_workspace.py +++ b/tests/backend/benchmarks/benchmark_workspace.py @@ -308,7 +308,7 @@ def print_report(all_results, latency_ms=None): print(" * full_derive_data -- the two above combined (dominates latency)") print(" * warm cache -- blob_data_cache avoids re-downloads for reads") print(" * local_dir always bypasses the blob_data_cache") - print(" * CachedAzureBlobWorkspace keeps a local mirror => local_dir is free") + print(" * blob_disk_cache keeps an ETag-validated local copy => reads avoid re-download") print() @@ -418,11 +418,11 @@ def _run_simulated(args, all_results): except Exception: pass - # -- 5. CachedAzureBlobWorkspace simulation ------------------------------ - # The CachedAzureBlobWorkspace uses a LOCAL file mirror so reads - # are at filesystem speed. We simulate it here by wrapping the + # -- 5. Local-mirror cache simulation ------------------------------------ + # AzureBlobWorkspace + blob_disk_cache keeps an ETag-validated local copy + # so reads are at filesystem speed. We simulate it here by wrapping the # SimulatedBlobWorkspace with the same write-through-to-cache pattern. - print(f"\n[5/5] CachedAzureBlobWorkspace pattern (local mirror)") + print(f"\n[5/5] Local-mirror cache pattern (blob_disk_cache)") with tempfile.TemporaryDirectory(prefix="df_bench_cached_") as tmpdir: ws_cached = Workspace("bench_cached", root_dir=tmpdir) all_results["Cached Azure"] = run_benchmark( diff --git a/tests/backend/security/test_confined_dir_migration.py b/tests/backend/security/test_confined_dir_migration.py index 8a620669..0ce69d61 100644 --- a/tests/backend/security/test_confined_dir_migration.py +++ b/tests/backend/security/test_confined_dir_migration.py @@ -4,7 +4,7 @@ """Regression tests for Phase 5 ConfinedDir migration. Verifies that Workspace.confined_* properties, agent tool path safety, -scratch route path safety, and CachedAzureBlobWorkspace._cache_path all +and scratch route path safety all correctly delegate to ConfinedDir after the migration from hand-written resolve+relative_to checks. """ @@ -162,52 +162,6 @@ def test_preview_scratch_traversal_blocked(self, agent, workspace_path): assert "error" in actions[0] -# ── CachedAzureBlobWorkspace._cache_path uses ConfinedDir ────────────── - - -class TestCachedAzureBlobCachePath: - - def test_cache_path_rejects_traversal(self, tmp_path): - from data_formulator.datalake.cached_azure_blob_workspace import ( - CachedAzureBlobWorkspace, - ) - jail = ConfinedDir(tmp_path / "cache", mkdir=True) - - instance = CachedAzureBlobWorkspace.__new__(CachedAzureBlobWorkspace) - instance._cache_dir = jail.root - instance._cache_jail = jail - - with pytest.raises(ValueError): - instance._cache_path("../../etc/passwd") - - def test_cache_path_normal_file(self, tmp_path): - from data_formulator.datalake.cached_azure_blob_workspace import ( - CachedAzureBlobWorkspace, - ) - cache_dir = tmp_path / "cache" - jail = ConfinedDir(cache_dir, mkdir=True) - - instance = CachedAzureBlobWorkspace.__new__(CachedAzureBlobWorkspace) - instance._cache_dir = jail.root - instance._cache_jail = jail - - result = instance._cache_path("data.parquet") - assert result == (cache_dir / "data.parquet").resolve() - - def test_cache_path_absolute_path_rejected(self, tmp_path): - from data_formulator.datalake.cached_azure_blob_workspace import ( - CachedAzureBlobWorkspace, - ) - jail = ConfinedDir(tmp_path / "cache", mkdir=True) - - instance = CachedAzureBlobWorkspace.__new__(CachedAzureBlobWorkspace) - instance._cache_dir = jail.root - instance._cache_jail = jail - - with pytest.raises(ValueError): - instance._cache_path("/etc/passwd") - - # ── Scratch routes use workspace.confined_scratch ───────────────────────