diff --git a/src/backend/facade.ts b/src/backend/facade.ts index 9518148..2ed6366 100644 --- a/src/backend/facade.ts +++ b/src/backend/facade.ts @@ -36,6 +36,11 @@ export interface BackendFacadeDeps { getModelFromRequest: (request: ChatRequest) => string; // Cache integration for performance getSessionFileDataCached?: (sessionFilePath: string, mtime: number, fileSize: number) => Promise; + // Stat helper for OpenCode DB virtual paths + statSessionFile: (sessionFile: string) => Promise; + // OpenCode session handling + isOpenCodeSession?: (sessionFile: string) => boolean; + getOpenCodeSessionData?: (sessionFile: string) => Promise<{ tokens: number; interactions: number; modelUsage: ModelUsage; timestamp: number }>; } export class BackendFacade { @@ -84,7 +89,10 @@ export class BackendFacade { estimateTokensFromText: deps.estimateTokensFromText, getModelFromRequest: deps.getModelFromRequest, getSessionFileDataCached: deps.getSessionFileDataCached, - updateTokenStats: deps.updateTokenStats + updateTokenStats: deps.updateTokenStats, + statSessionFile: deps.statSessionFile, + isOpenCodeSession: deps.isOpenCodeSession, + getOpenCodeSessionData: deps.getOpenCodeSessionData }, this.credentialService, this.dataPlaneService, diff --git a/src/backend/services/syncService.ts b/src/backend/services/syncService.ts index a55343b..de3e577 100644 --- a/src/backend/services/syncService.ts +++ b/src/backend/services/syncService.ts @@ -78,6 +78,11 @@ export interface SyncServiceDeps { getSessionFileDataCached?: (sessionFilePath: string, mtime: number, fileSize: number) => Promise; // UI refresh callback after successful sync updateTokenStats?: () => Promise; + // Stat helper for OpenCode DB virtual paths + statSessionFile: (sessionFile: string) => Promise; + // OpenCode session handling + isOpenCodeSession?: (sessionFile: string) => boolean; + getOpenCodeSessionData?: (sessionFile: string) => Promise<{ tokens: number; interactions: number; modelUsage: any; timestamp: number }>; } /** @@ -468,7 +473,7 @@ export class SyncService { let fileMtimeMs: number | undefined; try { - const fileStat = await fs.promises.stat(sessionFile); + const fileStat = await this.deps.statSessionFile(sessionFile); fileMtimeMs = fileStat.mtimeMs; @@ -483,6 +488,44 @@ export class SyncService { continue; } + // Handle OpenCode sessions separately (different data format) + if (this.deps.isOpenCodeSession && this.deps.isOpenCodeSession(sessionFile)) { + if (!this.deps.getOpenCodeSessionData) { + filesSkipped++; + continue; + } + + try { + const data = await this.deps.getOpenCodeSessionData(sessionFile); + const eventMs = data.timestamp || fileMtimeMs; + + if (!eventMs || eventMs < startMs) { + filesSkipped++; + continue; + } + + const dayKey = this.utility.toUtcDayKey(new Date(eventMs)); + const workspaceId = this.utility.extractWorkspaceIdFromSessionPath(sessionFile); + await this.ensureWorkspaceNameResolved(workspaceId, sessionFile, workspaceNamesById); + + // Process each model's usage with per-model interaction counts + for (const [model, usage] of Object.entries(data.modelUsage)) { + const key: DailyRollupKey = { day: dayKey, model, workspaceId, machineId, userId }; + upsertDailyRollup(rollups as any, key, { + inputTokens: (usage as any).inputTokens || 0, + outputTokens: (usage as any).outputTokens || 0, + interactions: (usage as any).interactions || 0 + }); + } + + filesProcessed++; + continue; + } catch (e) { + this.deps.warn(`Backend sync: failed to process OpenCode session ${sessionFile}: ${e}`); + continue; + } + } + const workspaceId = this.utility.extractWorkspaceIdFromSessionPath(sessionFile); await this.ensureWorkspaceNameResolved(workspaceId, sessionFile, workspaceNamesById); @@ -490,7 +533,7 @@ export class SyncService { // Note: We still parse the file to get accurate day keys from timestamps, // but use cached token counts for performance if (useCachedData) { - const fileStat = await fs.promises.stat(sessionFile); + const fileStat = await this.deps.statSessionFile(sessionFile); const cacheSuccess = await this.processCachedSessionFile( sessionFile, fileMtimeMs, @@ -682,6 +725,26 @@ export class SyncService { await this.dataPlaneService.ensureTableExists(settings, creds.tableCredential); await this.dataPlaneService.validateAccess(settings, creds.tableCredential); + // Check blob upload status upfront (before expensive file scanning) + let blobUploadNeeded = false; + if (settings.blobUploadEnabled && this.blobUploadService) { + const machineId = vscode.env.machineId; + const uploadSettings = { + enabled: settings.blobUploadEnabled, + containerName: settings.blobContainerName, + uploadFrequencyHours: settings.blobUploadFrequencyHours, + compressFiles: settings.blobCompressFiles + }; + blobUploadNeeded = this.blobUploadService.shouldUpload(machineId, uploadSettings); + if (blobUploadNeeded) { + this.deps.log('Blob upload: will upload session files after table sync'); + } else { + const status = this.blobUploadService.getUploadStatus(machineId); + const hoursSince = status ? Math.round((Date.now() - status.lastUploadTime) / (1000 * 60 * 60)) : 0; + this.deps.log(`Blob upload: not needed (last upload ${hoursSince}h ago, frequency: ${settings.blobUploadFrequencyHours}h)`); + } + } + // Fetch session files once and reuse for both rollups and blob upload const sessionFiles = await this.deps.getCopilotSessionFiles(); @@ -755,9 +818,8 @@ export class SyncService { this.deps.log('Backend sync: completed'); - // Upload session files to Blob Storage if enabled - // Check if upload is needed BEFORE processing files to avoid redundant work - if (settings.blobUploadEnabled && this.blobUploadService) { + // Upload session files to Blob Storage if needed (check was done earlier) + if (blobUploadNeeded && this.blobUploadService) { try { const machineId = vscode.env.machineId; const uploadSettings = { @@ -767,29 +829,21 @@ export class SyncService { compressFiles: settings.blobCompressFiles }; - // Only fetch and upload if it's time to upload - if (this.blobUploadService.shouldUpload(machineId, uploadSettings)) { - this.deps.log('Blob upload: starting'); - - const uploadResult = await this.blobUploadService.uploadSessionFiles( - settings.storageAccount, - uploadSettings, - creds.blobCredential, - sessionFiles, // Reuse session files from rollup computation - machineId, - settings.datasetId - ); - - if (uploadResult.success) { - this.deps.log(`Blob upload: ${uploadResult.message}`); - } else { - this.deps.warn(`Blob upload: ${uploadResult.message}`); - } + this.deps.log('Blob upload: starting'); + + const uploadResult = await this.blobUploadService.uploadSessionFiles( + settings.storageAccount, + uploadSettings, + creds.blobCredential, + sessionFiles, // Reuse session files from rollup computation + machineId, + settings.datasetId + ); + + if (uploadResult.success) { + this.deps.log(`Blob upload: ${uploadResult.message}`); } else { - // Log the skip reason without fetching session files - const status = this.blobUploadService.getUploadStatus(machineId); - const hoursSince = status ? Math.round((Date.now() - status.lastUploadTime) / (1000 * 60 * 60)) : 0; - this.deps.log(`Blob upload: skipped (last upload ${hoursSince}h ago, frequency: ${settings.blobUploadFrequencyHours}h)`); + this.deps.warn(`Blob upload: ${uploadResult.message}`); } } catch (blobError: any) { this.deps.warn(`Blob upload: failed - ${blobError?.message ?? blobError}`); diff --git a/src/extension.ts b/src/extension.ts index bd40fe4..0ae8562 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -5473,12 +5473,15 @@ class CopilotTokenTracker implements vscode.Disposable { */ private isNonSessionFile(filename: string): boolean { const nonSessionFilePatterns = [ - 'embeddings', // commandEmbeddings.json, settingEmbeddings.json - 'index', // index files - 'cache', // cache files + 'embeddings', // commandEmbeddings.json, settingEmbeddings.json + 'index', // index files + 'cache', // cache files 'preferences', 'settings', - 'config' + 'config', + 'workspacesessions', // copilot.cli.workspaceSessions.*.json (index files with session ID lists) + 'globalsessions', // copilot.cli.oldGlobalSessions.json (index files) + 'api.json' // api.json (API configuration) ]; const lowerFilename = filename.toLowerCase(); return nonSessionFilePatterns.some(pattern => lowerFilename.includes(pattern)); @@ -5755,6 +5758,54 @@ class CopilotTokenTracker implements vscode.Disposable { return modelUsage; } + /** + * Get all session data from an OpenCode session in one call (for backend sync). + * Returns tokens, interactions, model usage, and timestamp. + * Includes per-model interaction counts in modelUsage. + */ + private async getOpenCodeSessionData(sessionFilePath: string): Promise<{ tokens: number; interactions: number; modelUsage: ModelUsage & { [key: string]: { inputTokens: number; outputTokens: number; interactions?: number } }; timestamp: number }> { + const messages = await this.getOpenCodeMessagesForSession(sessionFilePath); + + // Get timestamp from the first message + let timestamp = Date.now(); + if (messages.length > 0 && messages[0].time_created) { + timestamp = messages[0].time_created; + } + + // Get tokens + const { tokens } = await this.getTokensFromOpenCodeSession(sessionFilePath); + + // Get interactions (total count) + const interactions = await this.countOpenCodeInteractions(sessionFilePath); + + // Get model usage with per-model interaction counts + const baseModelUsage = await this.getOpenCodeModelUsage(sessionFilePath); + + // Count interactions per model (each user turn -> 1 interaction for the model that responded) + const modelInteractions: { [model: string]: number } = {}; + let prevTotal = 0; + for (let i = 0; i < messages.length; i++) { + const msg = messages[i]; + if (msg.role !== 'user') { continue; } + const turnAssistantMsgs = messages.filter((m, idx) => idx > i && m.role === 'assistant' && m.parentID === msg.id); + if (turnAssistantMsgs.length === 0) { continue; } + + const model = turnAssistantMsgs[0].modelID || turnAssistantMsgs[0].model?.modelID || 'unknown'; + modelInteractions[model] = (modelInteractions[model] || 0) + 1; + } + + // Merge interaction counts into model usage + const modelUsage: any = {}; + for (const [model, usage] of Object.entries(baseModelUsage)) { + modelUsage[model] = { + ...usage, + interactions: modelInteractions[model] || 0 + }; + } + + return { tokens, interactions, modelUsage, timestamp }; + } + private getModelFromRequest(request: any): string { // Try to determine model from request metadata (most reliable source) // First check the top-level modelId field (VS Code format) @@ -9218,7 +9269,10 @@ export function activate(context: vscode.ExtensionContext) { getCopilotSessionFiles: () => (tokenTracker as any).getCopilotSessionFiles(), estimateTokensFromText: (text: string, model?: string) => (tokenTracker as any).estimateTokensFromText(text, model), getModelFromRequest: (req: any) => (tokenTracker as any).getModelFromRequest(req), - getSessionFileDataCached: (p: string, m: number, s: number) => (tokenTracker as any).getSessionFileDataCached(p, m, s) + getSessionFileDataCached: (p: string, m: number, s: number) => (tokenTracker as any).getSessionFileDataCached(p, m, s), + statSessionFile: (sessionFile: string) => (tokenTracker as any).statSessionFile(sessionFile), + isOpenCodeSession: (sessionFile: string) => (tokenTracker as any).isOpenCodeSessionFile(sessionFile), + getOpenCodeSessionData: (sessionFile: string) => (tokenTracker as any).getOpenCodeSessionData(sessionFile) }); const backendHandler = new BackendCommandHandler({ diff --git a/src/test-node/backend-cache-integration.test.ts b/src/test-node/backend-cache-integration.test.ts index 26322bd..b39497e 100644 --- a/src/test-node/backend-cache-integration.test.ts +++ b/src/test-node/backend-cache-integration.test.ts @@ -62,7 +62,8 @@ test('Backend cache integration: uses cached data when available', async () => { }, mtime }; - } + }, + statSessionFile: async (f: string) => fs.promises.stat(f) }); const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' }); @@ -120,7 +121,8 @@ test('Backend cache integration: falls back to parsing on cache miss', async () getSessionFileDataCached: async (filePath: string, mtime: number): Promise => { cacheMissCount++; throw new Error('ENOENT: file not found'); // Simulate cache miss - } + }, + statSessionFile: async (f: string) => fs.promises.stat(f) }); const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' }); @@ -197,10 +199,9 @@ test('Backend cache integration: validates cached data and rejects invalid struc getSessionFileDataCached: async (): Promise => { return invalidCache as any; // Return invalid cache data } - }); - - const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' }); - + , + statSessionFile: async (f: string) => fs.promises.stat(f) + }); // Should fall back to parsing when cache validation fails // Empty requests array means no rollups from fallback, but validation warning should be logged assert.ok( @@ -248,6 +249,7 @@ test('Backend cache integration: counts interactions only once for multi-model f getCopilotSessionFiles: async () => [sessionFile], estimateTokensFromText: (text: string) => (text ?? '').length, getModelFromRequest: (request: any) => (request?.model ?? 'gpt-4o').toString(), + statSessionFile: async (f: string) => fs.promises.stat(f), getSessionFileDataCached: async (): Promise => { // Simulate cached token data for the 3 models return { @@ -325,6 +327,8 @@ test('Backend cache integration: handles cache errors gracefully', async () => { getSessionFileDataCached: async (): Promise => { throw new Error('Network timeout'); // Unexpected error } + , + statSessionFile: async (f: string) => fs.promises.stat(f) }); const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' }); diff --git a/src/test-node/backend-configurator.test.ts b/src/test-node/backend-configurator.test.ts index 67404bb..362db29 100644 --- a/src/test-node/backend-configurator.test.ts +++ b/src/test-node/backend-configurator.test.ts @@ -2,6 +2,7 @@ import './vscode-shim-register'; import test from 'node:test'; import * as assert from 'node:assert/strict'; +import * as fs from 'node:fs'; import * as vscode from 'vscode'; import { JSDOM } from 'jsdom'; @@ -107,7 +108,8 @@ test('saveDraft persists settings, records consent, and clamps values', async () co2AbsorptionPerTreePerYear: 0, getCopilotSessionFiles: async () => [], estimateTokensFromText: () => 0, - getModelFromRequest: () => 'gpt-4o' + getModelFromRequest: () => 'gpt-4o', + statSessionFile: async (f: string) => fs.promises.stat(f) }); facade.getSettings = () => current; @@ -152,7 +154,8 @@ test('saveDraft blocks when consent is withheld', async () => { co2AbsorptionPerTreePerYear: 0, getCopilotSessionFiles: async () => [], estimateTokensFromText: () => 0, - getModelFromRequest: () => 'gpt-4o' + getModelFromRequest: () => 'gpt-4o', + statSessionFile: async (f: string) => fs.promises.stat(f) }); facade.getSettings = () => current; @@ -187,7 +190,8 @@ test('updateSharedKey stores secret and returns updated panel state', async () = co2AbsorptionPerTreePerYear: 0, getCopilotSessionFiles: async () => [], estimateTokensFromText: () => 0, - getModelFromRequest: () => 'gpt-4o' + getModelFromRequest: () => 'gpt-4o', + statSessionFile: async (f: string) => fs.promises.stat(f) }); facade.getSettings = () => current; @@ -218,7 +222,8 @@ test('testConnectionFromDraft surfaces success, errors, and shared-key requireme co2AbsorptionPerTreePerYear: 0, getCopilotSessionFiles: async () => [], estimateTokensFromText: () => 0, - getModelFromRequest: () => 'gpt-4o' + getModelFromRequest: () => 'gpt-4o', + statSessionFile: async (f: string) => fs.promises.stat(f) }); let validated = 0; @@ -477,7 +482,8 @@ test('launchConfigureWizardFromPanel triggers wizard, timers, cache clear, and s co2AbsorptionPerTreePerYear: 0, getCopilotSessionFiles: async () => [], estimateTokensFromText: () => 0, - getModelFromRequest: () => 'gpt-4o' + getModelFromRequest: () => 'gpt-4o', + statSessionFile: async (f: string) => fs.promises.stat(f) }); facade.azureResourceService = { configureBackendWizard: async () => { wizardCalled++; } } as any; diff --git a/src/test-node/backend-facade-helpers.test.ts b/src/test-node/backend-facade-helpers.test.ts index 01509f0..94a0c8e 100644 --- a/src/test-node/backend-facade-helpers.test.ts +++ b/src/test-node/backend-facade-helpers.test.ts @@ -14,7 +14,11 @@ function createFacade(): BackendFacade { co2AbsorptionPerTreePerYear: 21000, getCopilotSessionFiles: async () => [], estimateTokensFromText: () => 0, - getModelFromRequest: () => 'gpt-4o' + getModelFromRequest: () => 'gpt-4o', + statSessionFile: async (f: string) => { + const fs = await import('fs'); + return fs.promises.stat(f); + } }); } diff --git a/src/test-node/backend-facade-query.test.ts b/src/test-node/backend-facade-query.test.ts index efb83cb..06061ae 100644 --- a/src/test-node/backend-facade-query.test.ts +++ b/src/test-node/backend-facade-query.test.ts @@ -1,5 +1,6 @@ import test from 'node:test'; import * as assert from 'node:assert/strict'; +import * as fs from 'node:fs'; import { BackendFacade } from '../backend/facade'; @@ -17,7 +18,8 @@ test('BackendFacade queryBackendRollups aggregates, filters, and caches results' co2AbsorptionPerTreePerYear: 21000, getCopilotSessionFiles: async () => [], estimateTokensFromText: () => 0, - getModelFromRequest: () => 'gpt-4o' + getModelFromRequest: () => 'gpt-4o', + statSessionFile: async (f: string) => fs.promises.stat(f) }); let listCalls = 0; diff --git a/src/test-node/backend-facade-rollups.test.ts b/src/test-node/backend-facade-rollups.test.ts index cf13871..1b5d717 100644 --- a/src/test-node/backend-facade-rollups.test.ts +++ b/src/test-node/backend-facade-rollups.test.ts @@ -93,7 +93,8 @@ test('BackendFacade computes daily rollups from JSONL and JSON sessions (and ski co2AbsorptionPerTreePerYear: 21000, getCopilotSessionFiles: async () => [jsonlPath, jsonPath, invalidJsonPath, missingPath], estimateTokensFromText: (text: string) => (text ?? '').length, - getModelFromRequest: (request: any) => (request?.model ?? 'gpt-4o').toString() + getModelFromRequest: (request: any) => (request?.model ?? 'gpt-4o').toString(), + statSessionFile: async (f: string) => fs.promises.stat(f) }); const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' });