Skip to content

Commit bca7836

Browse files
authored
feat(project-files): add core storage and realtime operations (#9016)
* refactor(files): unify mutation accounting and retention * feat(project-files): add core storage and realtime operations * feat(files): unify document read subscriptions and write capabilities * fix(files): resolve capability type and artifact checks * fix(docs): hide unreleased Project file references * fix(files): address review admission and collaboration gaps * test(cli): load canonical documentation in suite setup * fix(files): bound seed reads and preserve browser snapshots * fix(files): keep read provenance private unless requested * fix(files): classify raw Project downloads as opaque
1 parent a754780 commit bca7836

228 files changed

Lines changed: 20137 additions & 1663 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/scripts/http-e2e.sh‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# mobile, which serves the production build the `e2e-build` job uploads with `next start` (the
44
# job extracts it into apps/sim/.next first).
55
#
6-
# Usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|mobile> (run from apps/sim)
6+
# Usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|project-files|mobile> (run from apps/sim)
77
#
88
# The job provides DATABASE_URL, BETTER_AUTH_SECRET and ENCRYPTION_KEY; each group sets the rest of
99
# its app's environment here. Reports and server logs land in $RUNNER_TEMP/e2e.
@@ -20,7 +20,7 @@
2020
# telemetry flush runs detached and still writes .next/dev).
2121
set -euo pipefail
2222

23-
group=${1:?usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|mobile>}
23+
group=${1:?usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|project-files|mobile>}
2424
report_dir="$RUNNER_TEMP/e2e"
2525
ready_timeout_seconds=600
2626
mkdir -p "$report_dir"
@@ -212,6 +212,15 @@ case "$group" in
212212
bun run test:desktop-inbox:e2e
213213
;;
214214

215+
project-files)
216+
PROJECT_FILES_E2E_ADMIN_DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \
217+
PROJECT_FILES_E2E_REDIS_URL=redis://127.0.0.1:6379 \
218+
PROJECT_FILES_E2E_BASE_URL=http://127.0.0.1:3050 \
219+
PROJECT_FILES_E2E_RELAY_URL=http://127.0.0.1:3052 \
220+
PROJECT_FILES_E2E_REPORT_PATH="$report_dir/project-files/orchestration.json" \
221+
bun --no-env-file scripts/test-project-files-e2e.ts
222+
;;
223+
215224
# MOBILE_E2E_BROWSER (chromium or webkit) narrows the job to one browser; unset runs both.
216225
mobile)
217226
app_server=start

‎.github/workflows/checks.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,8 @@ jobs:
196196
runner: blacksmith-4vcpu-ubuntu-2404
197197
- group: stop-after
198198
runner: blacksmith-4vcpu-ubuntu-2404
199+
- group: project-files
200+
runner: blacksmith-8vcpu-ubuntu-2404
199201
- group: desktop-inbox
200202
runner: blacksmith-4vcpu-ubuntu-2404
201203
services:
@@ -212,7 +214,7 @@ jobs:
212214
--health-interval 5s
213215
--health-timeout 5s
214216
--health-retries 10
215-
# Only the desktop executor's app is given REDIS_URL: its doorbell and presence live there.
217+
# Project file subscriptions and the desktop executor use Redis.
216218
redis:
217219
image: redis:8.2-alpine
218220
ports:

‎apps/docs/openapi-v2-files-audit.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6243,7 +6243,7 @@
62436243
],
62446244
"additionalProperties": false,
62456245
"title": "File share",
6246-
"description": "Public-safe share configuration for a workspace file."
6246+
"description": "Public-safe share configuration for a file."
62476247
},
62486248
"V2FileMetadata": {
62496249
"type": "object",

‎apps/realtime/src/access-revalidation.test.ts‎

Lines changed: 10 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -130,43 +130,29 @@ describe('access-revalidation sweep', () => {
130130
expect(tableSocket.leave).not.toHaveBeenCalled()
131131
})
132132

133-
it('evicts a file-doc socket downgraded to read, and keeps its table room', async () => {
134-
// A file-doc room IS the editor and requires `write`; a table room requires only
135-
// `read`. One downgraded user in both rooms must lose exactly the document.
133+
it('keeps a downgraded reader subscribed and reports the loss of write access', async () => {
136134
const socket = makeSocket('sock-1', 'user-1', 'workspace-file-doc:file-1')
137135
socket.rooms.add('table:t-1')
138136
const manager = makeManager([socket])
139137
mockResolveRole.mockResolvedValue('read')
140-
141138
const sweep = startAccessRevalidationSweep(manager)
142139
await sweep.runOnce()
143140
sweep.stop()
144-
145-
expect(socket.leave).toHaveBeenCalledWith('workspace-file-doc:file-1')
146-
expect(socket.leave).not.toHaveBeenCalledWith('table:t-1')
147-
expect(socket.emit).toHaveBeenCalledWith(
148-
'room-access-revoked',
149-
expect.objectContaining({ room: { type: 'workspace-file-doc', id: 'file-1' } })
150-
)
141+
expect(socket.leave).not.toHaveBeenCalled()
142+
expect(socket.emit).toHaveBeenCalledWith('file-doc-permission', {
143+
fileId: 'file-1',
144+
canWrite: false,
145+
})
151146
})
152147

153-
it('falls back to the room type own membership level on a cold-cache failure', async () => {
154-
// A static 'read' fallback would have evicted every file-doc socket (which needs
155-
// `write`) the first time the DB blipped with a cold cache.
148+
it('retains a document subscription for retry when authorization is unavailable', async () => {
156149
const socket = makeSocket('sock-1', 'user-1', 'workspace-file-doc:file-1')
157-
const manager = makeManager([socket])
158-
mockResolveRole.mockResolvedValue('write')
159-
160-
const sweep = startAccessRevalidationSweep(manager)
150+
mockResolveRole.mockRejectedValue(new Error('database unavailable'))
151+
const sweep = startAccessRevalidationSweep(makeManager([socket]))
161152
await sweep.runOnce()
162153
sweep.stop()
163-
164-
expect(mockResolveRole).toHaveBeenCalledWith(
165-
'user-1',
166-
{ type: 'workspace-file-doc', id: 'file-1' },
167-
'write'
168-
)
169154
expect(socket.leave).not.toHaveBeenCalled()
155+
expect(socket.emit).not.toHaveBeenCalledWith('file-doc-permission', expect.anything())
170156
})
171157

172158
it('evicts only the revoked socket, not co-members of the room', async () => {

‎apps/realtime/src/access-revalidation.ts‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { ROOM_MEMBERSHIP_ACTIONS, satisfiesRoomMembership } from '@sim/platform-
33
import type { AccessRevokedBroadcast } from '@sim/realtime-protocol/events'
44
import { FILE_DOC_EVENTS, type FileDocPermission } from '@sim/realtime-protocol/file-doc'
55
import {
6+
isProjectRoom,
67
parseRoomName,
78
projectFileDocTarget,
89
ROOM_TYPES,
@@ -35,14 +36,8 @@ const logger = createLogger('AccessRevalidation')
3536
export const ACCESS_REVALIDATION_SWEEP_INTERVAL_MS = ROLE_REVALIDATION_TTL_MS
3637

3738
/**
38-
* Fallback consumed by {@link resolveCurrentRoomPermission} only on a transient DB
39-
* failure with a cold cache, where resolving to a permission that KEEPS the socket
40-
* (never eviction) is the safe outcome during an outage. It is the room's own
41-
* membership level, so it satisfies that room's requirement exactly — a static
42-
* `'read'` would have evicted every file-doc socket (which needs `write`) on a cold
43-
* cache blip. The scan deliberately does not read presence for a per-socket
44-
* join-time role: that would put a Redis dependency inside the security-critical
45-
* scan lane, and the fallback's only job is to be permissive.
39+
* Preserves membership for room types that allow a cold-cache outage fallback.
40+
* Document rooms throw instead; the sweep retains their subscriptions for retry.
4641
*/
4742
function fallbackRoleFor(type: RoomType): string {
4843
return ROOM_MEMBERSHIP_ACTIONS[type]
@@ -329,15 +324,21 @@ export function startAccessRevalidationSweep(roomManager: IRoomManager): AccessR
329324
// resolution keeps running in the background and is re-raced when the
330325
// rotation returns to this socket, so it is acted on once it settles.
331326
const role = await Promise.race([
332-
room.type === ROOM_TYPES.PROJECT_FILE_DOC
327+
isProjectRoom(room)
333328
? resolveCurrentRoomPermission(userId, room, fallbackRoleFor(room.type), socket.id)
334329
: resolveCurrentRoomPermission(userId, room, fallbackRoleFor(room.type)),
335330
sleep(Math.min(SCAN_SOCKET_TIMEOUT_MS, remainingBudget)).then(() => SCAN_TIMED_OUT),
336331
])
337332
// {@link SCAN_TIMED_OUT} is the only symbol this race can yield; matching on
338333
// the type narrows it out of the permission comparison below.
339-
if (typeof role !== 'symbol' && room.type === ROOM_TYPES.PROJECT_FILE_DOC) {
340-
const target = projectFileDocTarget(room)
334+
if (
335+
typeof role !== 'symbol' &&
336+
(room.type === ROOM_TYPES.PROJECT_FILE_DOC || room.type === ROOM_TYPES.WORKSPACE_FILE_DOC)
337+
) {
338+
const target =
339+
room.type === ROOM_TYPES.PROJECT_FILE_DOC
340+
? projectFileDocTarget(room)
341+
: { fileId: room.id }
341342
if (target)
342343
socket.emit(FILE_DOC_EVENTS.PERMISSION, {
343344
...target,
@@ -349,9 +350,6 @@ export function startAccessRevalidationSweep(roomManager: IRoomManager): AccessR
349350
`Authorization check timed out for user ${userId} on ${name}; skipping this pass`
350351
)
351352
} else if (!satisfiesRoomMembership(role, room.type)) {
352-
// Covers both revocation (`null`) and downgrade below the level this room
353-
// requires — a member dropped to `read` may keep a table room but not the
354-
// collaborative document editor, exactly as at join time.
355353
revokeSocket(socket, room, name)
356354
}
357355
} catch (error) {

‎apps/realtime/src/handlers/file-doc-app.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,13 +195,14 @@ export async function fetchProjectFileDocAccess(
195195
/** Fetch a seed under the joining socket's current Project read access. */
196196
export async function fetchProjectFileDocSeed(
197197
target: ProjectDocumentRequest
198-
): Promise<{ update: Uint8Array; version: number }> {
198+
): Promise<{ update: Uint8Array; version: number } | null> {
199199
const response = await postToApp(
200200
projectDocumentPath(target, 'seed'),
201201
{},
202202
FILE_DOC_TIMEOUTS.seedRequestMs,
203203
target
204204
)
205+
if (response.status === 404) return null
205206
if (!response.ok) throw new Error(`Project document seed failed: ${response.status}`)
206207
const body = (await response.json()) as { update?: unknown; version?: unknown }
207208
if (typeof body.update !== 'string' || typeof body.version !== 'number')

0 commit comments

Comments
 (0)