Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion src/backend/facade.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ export interface BackendFacadeDeps {
getModelFromRequest: (request: ChatRequest) => string;
// Cache integration for performance
getSessionFileDataCached?: (sessionFilePath: string, mtime: number, fileSize: number) => Promise<SessionFileCache>;
// Stat helper for OpenCode DB virtual paths
statSessionFile: (sessionFile: string) => Promise<any>;
// OpenCode session handling
isOpenCodeSession?: (sessionFile: string) => boolean;
getOpenCodeSessionData?: (sessionFile: string) => Promise<{ tokens: number; interactions: number; modelUsage: ModelUsage; timestamp: number }>;
}

export class BackendFacade {
Expand Down Expand Up @@ -84,7 +89,10 @@ export class BackendFacade {
estimateTokensFromText: deps.estimateTokensFromText,
getModelFromRequest: deps.getModelFromRequest,
getSessionFileDataCached: deps.getSessionFileDataCached,
updateTokenStats: deps.updateTokenStats
updateTokenStats: deps.updateTokenStats,
statSessionFile: deps.statSessionFile,
isOpenCodeSession: deps.isOpenCodeSession,
getOpenCodeSessionData: deps.getOpenCodeSessionData
},
this.credentialService,
this.dataPlaneService,
Expand Down
108 changes: 81 additions & 27 deletions src/backend/services/syncService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,11 @@ export interface SyncServiceDeps {
getSessionFileDataCached?: (sessionFilePath: string, mtime: number, fileSize: number) => Promise<SessionFileCache>;
// UI refresh callback after successful sync
updateTokenStats?: () => Promise<void>;
// Stat helper for OpenCode DB virtual paths
statSessionFile: (sessionFile: string) => Promise<fs.Stats>;
// OpenCode session handling
isOpenCodeSession?: (sessionFile: string) => boolean;
getOpenCodeSessionData?: (sessionFile: string) => Promise<{ tokens: number; interactions: number; modelUsage: any; timestamp: number }>;
}

/**
Expand Down Expand Up @@ -468,7 +473,7 @@ export class SyncService {
let fileMtimeMs: number | undefined;

try {
const fileStat = await fs.promises.stat(sessionFile);
const fileStat = await this.deps.statSessionFile(sessionFile);
fileMtimeMs = fileStat.mtimeMs;


Expand All @@ -483,14 +488,52 @@ export class SyncService {
continue;
}

// Handle OpenCode sessions separately (different data format)
if (this.deps.isOpenCodeSession && this.deps.isOpenCodeSession(sessionFile)) {
if (!this.deps.getOpenCodeSessionData) {
filesSkipped++;
continue;
}

try {
const data = await this.deps.getOpenCodeSessionData(sessionFile);
const eventMs = data.timestamp || fileMtimeMs;

if (!eventMs || eventMs < startMs) {
filesSkipped++;
continue;
}

const dayKey = this.utility.toUtcDayKey(new Date(eventMs));
const workspaceId = this.utility.extractWorkspaceIdFromSessionPath(sessionFile);
await this.ensureWorkspaceNameResolved(workspaceId, sessionFile, workspaceNamesById);

// Process each model's usage with per-model interaction counts
for (const [model, usage] of Object.entries(data.modelUsage)) {
const key: DailyRollupKey = { day: dayKey, model, workspaceId, machineId, userId };
upsertDailyRollup(rollups as any, key, {
inputTokens: (usage as any).inputTokens || 0,
outputTokens: (usage as any).outputTokens || 0,
interactions: (usage as any).interactions || 0
});
}

filesProcessed++;
continue;
} catch (e) {
this.deps.warn(`Backend sync: failed to process OpenCode session ${sessionFile}: ${e}`);
continue;
}
}

const workspaceId = this.utility.extractWorkspaceIdFromSessionPath(sessionFile);
await this.ensureWorkspaceNameResolved(workspaceId, sessionFile, workspaceNamesById);

// Try to use cached data first (faster than full recomputation)
// Note: We still parse the file to get accurate day keys from timestamps,
// but use cached token counts for performance
if (useCachedData) {
const fileStat = await fs.promises.stat(sessionFile);
const fileStat = await this.deps.statSessionFile(sessionFile);
const cacheSuccess = await this.processCachedSessionFile(
sessionFile,
fileMtimeMs,
Expand Down Expand Up @@ -682,6 +725,26 @@ export class SyncService {
await this.dataPlaneService.ensureTableExists(settings, creds.tableCredential);
await this.dataPlaneService.validateAccess(settings, creds.tableCredential);

// Check blob upload status upfront (before expensive file scanning)
let blobUploadNeeded = false;
if (settings.blobUploadEnabled && this.blobUploadService) {
const machineId = vscode.env.machineId;
const uploadSettings = {
enabled: settings.blobUploadEnabled,
containerName: settings.blobContainerName,
uploadFrequencyHours: settings.blobUploadFrequencyHours,
compressFiles: settings.blobCompressFiles
};
blobUploadNeeded = this.blobUploadService.shouldUpload(machineId, uploadSettings);
if (blobUploadNeeded) {
this.deps.log('Blob upload: will upload session files after table sync');
} else {
const status = this.blobUploadService.getUploadStatus(machineId);
const hoursSince = status ? Math.round((Date.now() - status.lastUploadTime) / (1000 * 60 * 60)) : 0;
this.deps.log(`Blob upload: not needed (last upload ${hoursSince}h ago, frequency: ${settings.blobUploadFrequencyHours}h)`);
}
}

// Fetch session files once and reuse for both rollups and blob upload
const sessionFiles = await this.deps.getCopilotSessionFiles();

Expand Down Expand Up @@ -755,9 +818,8 @@ export class SyncService {

this.deps.log('Backend sync: completed');

// Upload session files to Blob Storage if enabled
// Check if upload is needed BEFORE processing files to avoid redundant work
if (settings.blobUploadEnabled && this.blobUploadService) {
// Upload session files to Blob Storage if needed (check was done earlier)
if (blobUploadNeeded && this.blobUploadService) {
try {
const machineId = vscode.env.machineId;
const uploadSettings = {
Expand All @@ -767,29 +829,21 @@ export class SyncService {
compressFiles: settings.blobCompressFiles
};

// Only fetch and upload if it's time to upload
if (this.blobUploadService.shouldUpload(machineId, uploadSettings)) {
this.deps.log('Blob upload: starting');

const uploadResult = await this.blobUploadService.uploadSessionFiles(
settings.storageAccount,
uploadSettings,
creds.blobCredential,
sessionFiles, // Reuse session files from rollup computation
machineId,
settings.datasetId
);

if (uploadResult.success) {
this.deps.log(`Blob upload: ${uploadResult.message}`);
} else {
this.deps.warn(`Blob upload: ${uploadResult.message}`);
}
this.deps.log('Blob upload: starting');

const uploadResult = await this.blobUploadService.uploadSessionFiles(
settings.storageAccount,
uploadSettings,
creds.blobCredential,
sessionFiles, // Reuse session files from rollup computation
machineId,
settings.datasetId
);

if (uploadResult.success) {
this.deps.log(`Blob upload: ${uploadResult.message}`);
} else {
// Log the skip reason without fetching session files
const status = this.blobUploadService.getUploadStatus(machineId);
const hoursSince = status ? Math.round((Date.now() - status.lastUploadTime) / (1000 * 60 * 60)) : 0;
this.deps.log(`Blob upload: skipped (last upload ${hoursSince}h ago, frequency: ${settings.blobUploadFrequencyHours}h)`);
this.deps.warn(`Blob upload: ${uploadResult.message}`);
}
} catch (blobError: any) {
this.deps.warn(`Blob upload: failed - ${blobError?.message ?? blobError}`);
Expand Down
64 changes: 59 additions & 5 deletions src/extension.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5473,12 +5473,15 @@ class CopilotTokenTracker implements vscode.Disposable {
*/
private isNonSessionFile(filename: string): boolean {
const nonSessionFilePatterns = [
'embeddings', // commandEmbeddings.json, settingEmbeddings.json
'index', // index files
'cache', // cache files
'embeddings', // commandEmbeddings.json, settingEmbeddings.json
'index', // index files
'cache', // cache files
'preferences',
'settings',
'config'
'config',
'workspacesessions', // copilot.cli.workspaceSessions.*.json (index files with session ID lists)
'globalsessions', // copilot.cli.oldGlobalSessions.json (index files)
'api.json' // api.json (API configuration)
];
const lowerFilename = filename.toLowerCase();
return nonSessionFilePatterns.some(pattern => lowerFilename.includes(pattern));
Expand Down Expand Up @@ -5755,6 +5758,54 @@ class CopilotTokenTracker implements vscode.Disposable {
return modelUsage;
}

/**
* Get all session data from an OpenCode session in one call (for backend sync).
* Returns tokens, interactions, model usage, and timestamp.
* Includes per-model interaction counts in modelUsage.
*/
private async getOpenCodeSessionData(sessionFilePath: string): Promise<{ tokens: number; interactions: number; modelUsage: ModelUsage & { [key: string]: { inputTokens: number; outputTokens: number; interactions?: number } }; timestamp: number }> {
const messages = await this.getOpenCodeMessagesForSession(sessionFilePath);

// Get timestamp from the first message
let timestamp = Date.now();
if (messages.length > 0 && messages[0].time_created) {
timestamp = messages[0].time_created;
}

// Get tokens
const { tokens } = await this.getTokensFromOpenCodeSession(sessionFilePath);

// Get interactions (total count)
const interactions = await this.countOpenCodeInteractions(sessionFilePath);

// Get model usage with per-model interaction counts
const baseModelUsage = await this.getOpenCodeModelUsage(sessionFilePath);

// Count interactions per model (each user turn -> 1 interaction for the model that responded)
const modelInteractions: { [model: string]: number } = {};
let prevTotal = 0;
for (let i = 0; i < messages.length; i++) {
const msg = messages[i];
if (msg.role !== 'user') { continue; }
const turnAssistantMsgs = messages.filter((m, idx) => idx > i && m.role === 'assistant' && m.parentID === msg.id);
if (turnAssistantMsgs.length === 0) { continue; }

const model = turnAssistantMsgs[0].modelID || turnAssistantMsgs[0].model?.modelID || 'unknown';
modelInteractions[model] = (modelInteractions[model] || 0) + 1;
}

// Merge interaction counts into model usage
const modelUsage: any = {};
for (const [model, usage] of Object.entries(baseModelUsage)) {
modelUsage[model] = {
...usage,
interactions: modelInteractions[model] || 0
};
}

return { tokens, interactions, modelUsage, timestamp };
}

private getModelFromRequest(request: any): string {
// Try to determine model from request metadata (most reliable source)
// First check the top-level modelId field (VS Code format)
Expand Down Expand Up @@ -9218,7 +9269,10 @@ export function activate(context: vscode.ExtensionContext) {
getCopilotSessionFiles: () => (tokenTracker as any).getCopilotSessionFiles(),
estimateTokensFromText: (text: string, model?: string) => (tokenTracker as any).estimateTokensFromText(text, model),
getModelFromRequest: (req: any) => (tokenTracker as any).getModelFromRequest(req),
getSessionFileDataCached: (p: string, m: number, s: number) => (tokenTracker as any).getSessionFileDataCached(p, m, s)
getSessionFileDataCached: (p: string, m: number, s: number) => (tokenTracker as any).getSessionFileDataCached(p, m, s),
statSessionFile: (sessionFile: string) => (tokenTracker as any).statSessionFile(sessionFile),
isOpenCodeSession: (sessionFile: string) => (tokenTracker as any).isOpenCodeSessionFile(sessionFile),
getOpenCodeSessionData: (sessionFile: string) => (tokenTracker as any).getOpenCodeSessionData(sessionFile)
});

const backendHandler = new BackendCommandHandler({
Expand Down
16 changes: 10 additions & 6 deletions src/test-node/backend-cache-integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ test('Backend cache integration: uses cached data when available', async () => {
},
mtime
};
}
},
statSessionFile: async (f: string) => fs.promises.stat(f)
});

const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' });
Expand Down Expand Up @@ -120,7 +121,8 @@ test('Backend cache integration: falls back to parsing on cache miss', async ()
getSessionFileDataCached: async (filePath: string, mtime: number): Promise<SessionFileCache> => {
cacheMissCount++;
throw new Error('ENOENT: file not found'); // Simulate cache miss
}
},
statSessionFile: async (f: string) => fs.promises.stat(f)
});

const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' });
Expand Down Expand Up @@ -197,10 +199,9 @@ test('Backend cache integration: validates cached data and rejects invalid struc
getSessionFileDataCached: async (): Promise<SessionFileCache> => {
return invalidCache as any; // Return invalid cache data
}
});

const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' });

,
statSessionFile: async (f: string) => fs.promises.stat(f)
});
// Should fall back to parsing when cache validation fails
// Empty requests array means no rollups from fallback, but validation warning should be logged
assert.ok(
Expand Down Expand Up @@ -248,6 +249,7 @@ test('Backend cache integration: counts interactions only once for multi-model f
getCopilotSessionFiles: async () => [sessionFile],
estimateTokensFromText: (text: string) => (text ?? '').length,
getModelFromRequest: (request: any) => (request?.model ?? 'gpt-4o').toString(),
statSessionFile: async (f: string) => fs.promises.stat(f),
getSessionFileDataCached: async (): Promise<SessionFileCache> => {
// Simulate cached token data for the 3 models
return {
Expand Down Expand Up @@ -325,6 +327,8 @@ test('Backend cache integration: handles cache errors gracefully', async () => {
getSessionFileDataCached: async (): Promise<SessionFileCache> => {
throw new Error('Network timeout'); // Unexpected error
}
,
statSessionFile: async (f: string) => fs.promises.stat(f)
});

const { rollups } = await facade.computeDailyRollupsFromLocalSessions({ lookbackDays: 1, userId: 'u1' });
Expand Down
16 changes: 11 additions & 5 deletions src/test-node/backend-configurator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import './vscode-shim-register';
import test from 'node:test';
import * as assert from 'node:assert/strict';
import * as fs from 'node:fs';

import * as vscode from 'vscode';
import { JSDOM } from 'jsdom';
Expand Down Expand Up @@ -107,7 +108,8 @@ test('saveDraft persists settings, records consent, and clamps values', async ()
co2AbsorptionPerTreePerYear: 0,
getCopilotSessionFiles: async () => [],
estimateTokensFromText: () => 0,
getModelFromRequest: () => 'gpt-4o'
getModelFromRequest: () => 'gpt-4o',
statSessionFile: async (f: string) => fs.promises.stat(f)
});

facade.getSettings = () => current;
Expand Down Expand Up @@ -152,7 +154,8 @@ test('saveDraft blocks when consent is withheld', async () => {
co2AbsorptionPerTreePerYear: 0,
getCopilotSessionFiles: async () => [],
estimateTokensFromText: () => 0,
getModelFromRequest: () => 'gpt-4o'
getModelFromRequest: () => 'gpt-4o',
statSessionFile: async (f: string) => fs.promises.stat(f)
});

facade.getSettings = () => current;
Expand Down Expand Up @@ -187,7 +190,8 @@ test('updateSharedKey stores secret and returns updated panel state', async () =
co2AbsorptionPerTreePerYear: 0,
getCopilotSessionFiles: async () => [],
estimateTokensFromText: () => 0,
getModelFromRequest: () => 'gpt-4o'
getModelFromRequest: () => 'gpt-4o',
statSessionFile: async (f: string) => fs.promises.stat(f)
});

facade.getSettings = () => current;
Expand Down Expand Up @@ -218,7 +222,8 @@ test('testConnectionFromDraft surfaces success, errors, and shared-key requireme
co2AbsorptionPerTreePerYear: 0,
getCopilotSessionFiles: async () => [],
estimateTokensFromText: () => 0,
getModelFromRequest: () => 'gpt-4o'
getModelFromRequest: () => 'gpt-4o',
statSessionFile: async (f: string) => fs.promises.stat(f)
});

let validated = 0;
Expand Down Expand Up @@ -477,7 +482,8 @@ test('launchConfigureWizardFromPanel triggers wizard, timers, cache clear, and s
co2AbsorptionPerTreePerYear: 0,
getCopilotSessionFiles: async () => [],
estimateTokensFromText: () => 0,
getModelFromRequest: () => 'gpt-4o'
getModelFromRequest: () => 'gpt-4o',
statSessionFile: async (f: string) => fs.promises.stat(f)
});

facade.azureResourceService = { configureBackendWizard: async () => { wizardCalled++; } } as any;
Expand Down
6 changes: 5 additions & 1 deletion src/test-node/backend-facade-helpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ function createFacade(): BackendFacade {
co2AbsorptionPerTreePerYear: 21000,
getCopilotSessionFiles: async () => [],
estimateTokensFromText: () => 0,
getModelFromRequest: () => 'gpt-4o'
getModelFromRequest: () => 'gpt-4o',
statSessionFile: async (f: string) => {
const fs = await import('fs');
return fs.promises.stat(f);
}
});
}

Expand Down
Loading