Skip to content
235 changes: 118 additions & 117 deletions packages/host/tests/acceptance/ai-assistant-test.gts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ import type MonacoService from '@cardstack/host/services/monaco-service';
import { AiAssistantMessageDrafts } from '@cardstack/host/utils/local-storage-keys';

import {
withCachedRealmSetup,
setupRealmCacheTeardown,
setupLocalIndexing,
addSkillToAiAssistant,
setupOnSave,
Expand Down Expand Up @@ -142,13 +144,8 @@ function modelNameFor(llmId: string): string {

module('Acceptance | AI Assistant tests', function (hooks) {
setupApplicationTest(hooks);
// Every test in this module builds the same realm fixtures in the beforeEach
// below, so the realm is indexed once and that index restored for each
// subsequent test. What a test writes afterwards stays with that test — the
// snapshot is restored, not carried forward.
setupLocalIndexing(hooks, {
reuseIndexAcrossTests: 'aiAssistant',
});
setupLocalIndexing(hooks);
setupRealmCacheTeardown(hooks);
setupOnSave(hooks);

let mockMatrixUtils = setupMockMatrix(hooks, {
Expand Down Expand Up @@ -431,13 +428,16 @@ module('Acceptance | AI Assistant tests', function (hooks) {
modelConfigurations: [deepseekModel, geminiFlashModel, openAiGpt5Model],
});

await setupAcceptanceTestRealm({
mockMatrixUtils,
contents: {
...SYSTEM_CARD_FIXTURE_CONTENTS,
'person.gts': { Person },
'pet.gts': { Pet },
'broken-card.gts': `
// Every test builds these fixtures identically, so the indexed result is
// cached for the module and restored rather than rebuilt.
await withCachedRealmSetup(async () => {
await setupAcceptanceTestRealm({
mockMatrixUtils,
contents: {
...SYSTEM_CARD_FIXTURE_CONTENTS,
'person.gts': { Person },
'pet.gts': { Pet },
'broken-card.gts': `
import { CardDef, field, contains } from '@cardstack/base/card-api';
import StringField from '@cardstack/base/string';
import { BrokenField } from './does-not-exist';
Expand All @@ -447,133 +447,134 @@ module('Acceptance | AI Assistant tests', function (hooks) {
@field broken = contains(BrokenField);
}
`,
'BrokenCard/errored.json': {
data: {
attributes: {
name: 'Errored Instance',
},
meta: {
adoptsFrom: {
module: '../broken-card',
name: 'BrokenCard',
'BrokenCard/errored.json': {
data: {
attributes: {
name: 'Errored Instance',
},
meta: {
adoptsFrom: {
module: '../broken-card',
name: 'BrokenCard',
},
},
},
},
},
'country.gts': countryDefinition,
'Country/indonesia.json': {
data: {
attributes: {
name: 'Indonesia',
},
meta: {
adoptsFrom: {
module: `${testRealmURL}country`,
name: 'Country',
'country.gts': countryDefinition,
'Country/indonesia.json': {
data: {
attributes: {
name: 'Indonesia',
},
meta: {
adoptsFrom: {
module: `${testRealmURL}country`,
name: 'Country',
},
},
},
},
},
'Pet/ringo.json': new Pet({ name: 'Ringo' }),
'Person/hassan.json': new Person({
firstName: 'Hassan',
lastName: 'Abdel-Rahman',
pet: mangoPet,
friends: [mangoPet],
}),
'Pet/mango.json': mangoPet,
'Pet/vangogh.json': new Pet({ name: 'Van Gogh' }),
'Person/fadhlan.json': new Person({
firstName: 'Fadhlan',
pet: mangoPet,
friends: [mangoPet],
}),
'plant.gts': `
'Pet/ringo.json': new Pet({ name: 'Ringo' }),
'Person/hassan.json': new Person({
firstName: 'Hassan',
lastName: 'Abdel-Rahman',
pet: mangoPet,
friends: [mangoPet],
}),
'Pet/mango.json': mangoPet,
'Pet/vangogh.json': new Pet({ name: 'Van Gogh' }),
'Person/fadhlan.json': new Person({
firstName: 'Fadhlan',
pet: mangoPet,
friends: [mangoPet],
}),
'plant.gts': `
import { CardDef, field, contains, StringField } from '@cardstack/base/card-api';
export class Plant extends CardDef {
static displayName = "Plant";
@field commonName = contains(StringField);
}
`,
'Plant/highbush-blueberry.json': {
data: {
attributes: {
commonName: 'Highbush Blueberry',
},
meta: {
adoptsFrom: {
module: `../plant`,
name: 'Plant',
'Plant/highbush-blueberry.json': {
data: {
attributes: {
commonName: 'Highbush Blueberry',
},
meta: {
adoptsFrom: {
module: `../plant`,
name: 'Plant',
},
},
},
},
},
'Spec/plant-spec.json': {
data: {
type: 'card',
attributes: {
ref: {
name: 'Plant',
module: `${testRealmURL}plant`,
'Spec/plant-spec.json': {
data: {
type: 'card',
attributes: {
ref: {
name: 'Plant',
module: `${testRealmURL}plant`,
},
specType: 'card',
cardTitle: 'Plant spec',
},
specType: 'card',
cardTitle: 'Plant spec',
},
meta: {
adoptsFrom: {
module: '@cardstack/base/spec',
name: 'Spec',
meta: {
adoptsFrom: {
module: '@cardstack/base/spec',
name: 'Spec',
},
},
},
},
},
'Skill/example.json': {
data: {
attributes: {
cardTitle: 'Example Skill',
cardDescription: 'This skill card is for testing purposes',
instructions: 'This is an example skill card',
commands: [],
},
meta: {
adoptsFrom: skillCardRef,
'Skill/example.json': {
data: {
attributes: {
cardTitle: 'Example Skill',
cardDescription: 'This skill card is for testing purposes',
instructions: 'This is an example skill card',
commands: [],
},
meta: {
adoptsFrom: skillCardRef,
},
},
},
},
'Skill/example2.json': {
data: {
attributes: {
cardTitle: 'Example 2 Skill',
cardDescription: 'This skill card is also for testing purposes',
instructions: 'This is a second example skill card',
commands: [],
},
meta: {
adoptsFrom: skillCardRef,
'Skill/example2.json': {
data: {
attributes: {
cardTitle: 'Example 2 Skill',
cardDescription: 'This skill card is also for testing purposes',
instructions: 'This is a second example skill card',
commands: [],
},
meta: {
adoptsFrom: skillCardRef,
},
},
},
'ModelConfiguration/gpt-4o-mini.json': openAiGpt4oMiniModel,
'ModelConfiguration/gpt-5.json': openAiGpt5Model,
'ModelConfiguration/gpt-5-extended.json': openAiGpt5ExtendedModel,
'ModelConfiguration/claude-sonnet-4.6.json':
anthropicClaudeSonnet46Model,
'ModelConfiguration/claude-sonnet-4.5.json':
anthropicClaudeSonnet45Model,
'ModelConfiguration/claude-sonnet-3.7.json':
anthropicClaudeSonnet37Model,
'SystemCard/default.json': defaultSystemCard,
'ModelConfiguration/deepseek-chat-v3-0324.json': deepseekModel,
'ModelConfiguration/gemini-2.5-flash.json': geminiFlashModel,
'SystemCard/productivity.json': alternateSystemCard,
'index.json': new CardsGrid(),
'realm.json': realmConfigCardJSON({
name: 'Test Workspace B',
backgroundURL:
'https://i.postimg.cc/VNvHH93M/pawel-czerwinski-Ly-ZLa-A5jti-Y-unsplash.jpg',
iconURL: 'https://i.postimg.cc/L8yXRvws/icon.png',
}),
},
'ModelConfiguration/gpt-4o-mini.json': openAiGpt4oMiniModel,
'ModelConfiguration/gpt-5.json': openAiGpt5Model,
'ModelConfiguration/gpt-5-extended.json': openAiGpt5ExtendedModel,
'ModelConfiguration/claude-sonnet-4.6.json':
anthropicClaudeSonnet46Model,
'ModelConfiguration/claude-sonnet-4.5.json':
anthropicClaudeSonnet45Model,
'ModelConfiguration/claude-sonnet-3.7.json':
anthropicClaudeSonnet37Model,
'SystemCard/default.json': defaultSystemCard,
'ModelConfiguration/deepseek-chat-v3-0324.json': deepseekModel,
'ModelConfiguration/gemini-2.5-flash.json': geminiFlashModel,
'SystemCard/productivity.json': alternateSystemCard,
'index.json': new CardsGrid(),
'realm.json': realmConfigCardJSON({
name: 'Test Workspace B',
backgroundURL:
'https://i.postimg.cc/VNvHH93M/pawel-czerwinski-Ly-ZLa-A5jti-Y-unsplash.jpg',
iconURL: 'https://i.postimg.cc/L8yXRvws/icon.png',
}),
},
});
});

getService('matrix-service').fetchMatrixHostedFile = async (_url) => {
Expand Down
26 changes: 25 additions & 1 deletion packages/host/tests/helpers/index.gts
Original file line number Diff line number Diff line change
Expand Up @@ -247,13 +247,37 @@ export async function withCachedRealmSetup<T>(
let dbAdapter = await getDbAdapter();
if (dbAdapter.hasSnapshot(snapshotName)) {
await dbAdapter.importSnapshot(snapshotName);
return setup();
let result = await setup();
// A restored index does not stop the realm indexing. Every fixture carries
// the mtime the *current* adapter stamped it with, while the restored rows
// carry the first test's, and `discover-invalidations` compares those for
// equality — so `realm.start()` invalidates the lot and re-indexes. The work
// is cheap, because the snapshot also restored the transpile and prerender
// caches, but it is asynchronous: without draining it here, a test that
// reads index-derived state races indexing still in flight. That is not
// hypothetical — it is how `ai-assistant`'s code-mode test lost the cursor
// position it clicks a definition to reach.
//
// Deliberately not solved by making the mtimes match. The mtime comparison
// is what rebuilds the index when a module's fixtures diverge from the
// snapshot under an unchanged cache key, which is this helper's only
// protection against silently serving the wrong fixtures.
await drainRealmIndexing();
return result;
}
let result = await setup();
await dbAdapter.exportSnapshot(snapshotName);
return result;
}

// Settle any indexing the realms have in flight. Mirrors what
// `setupLocalIndexing`'s afterEach does between tests.
async function drainRealmIndexing(): Promise<void> {
for (let { realm } of getTestRealmRegistry().values()) {
await realm.incrementalIndexing();
}
}

function getCurrentModuleCacheKey(): string {
let config = QUnit.config as QUnit['config'] & {
currentModule?: { name?: string };
Expand Down
Loading