Skip to content

Commit 1b99f59

Browse files
florianbgtflorianbgt
committed
implement bucket
1 parent aba13d2 commit 1b99f59

File tree

18 files changed

+249
-30
lines changed

18 files changed

+249
-30
lines changed

‎.github/workflows/test-lint.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,13 @@ jobs:
4040
ports:
4141
- 12111:12111
4242
- 12112:12112
43+
fake-gcs-server:
44+
image: fsouza/fake-gcs-server:latest
45+
ports:
46+
- 4443:4443
47+
options: >-
48+
--scheme http
49+
--public-host localhost:4443
4350
steps:
4451
- name: Checkout repository
4552
uses: actions/checkout@v4

‎deno.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@
1111
"dev:runner": "deno task --config ./packages/runner/deno.json dev",
1212
"build:app": "deno task --config ./packages/app/deno.json build",
1313
"build:core": "deno task --config ./packages/core/deno.json build",
14-
"test": "STRIPE_USE_MOCK=true deno test -A",
14+
"test": "STRIPE_USE_MOCK=true GCP_USE_FAKE_GCS_SERVER=true deno test -A",
1515
"db:docker": "docker container rm stackcore-pg --force || true && docker run --name stackcore-pg -e POSTGRES_PASSWORD=password -e POSTGRES_USER=user -e POSTGRES_DB=core -p 5432:5432 -d postgres:17",
1616
"db:migrate": "deno task --config ./packages/core/deno.json migrate -A",
1717
"stripe:create-products": "deno run -A --env-file=.env ./packages/core/src/stripe/scripts/createProducts.ts",
1818
"stripe:cli": "docker run --rm -it --env-file=.env stripe/stripe-cli:latest",
1919
"stripe:forward": "deno task stripe:cli listen --forward-to http://host.docker.internal:4000/billing/webhook",
20-
"stripe:mock": "docker run --rm -it -p 12111-12112:12111-12112 stripe/stripe-mock:latest"
20+
"stripe:mock": "docker run --rm -it -p 12111-12112:12111-12112 stripe/stripe-mock:latest",
21+
"bucket:mock": "docker run --rm -it -p 4443:4443 fsouza/fake-gcs-server:latest -scheme http -public-host localhost:4443"
2122
},
2223
"lint": {
2324
"exclude": [

‎packages/app/src/pages/projects/add.tsx‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ export default function AddProjectPage() {
4747

4848
const formSchema = z.object({
4949
name: z.string().min(1, "Project name is required").max(100),
50+
repoUrl: z.string().url("Invalid repository URL"),
5051
maxCodeCharPerSymbol: z.number().int().min(1, "Must be at least 1"),
5152
maxCodeCharPerFile: z.number().int().min(1, "Must be at least 1"),
5253
maxCharPerSymbol: z.number().int().min(1, "Must be at least 1"),
@@ -74,6 +75,7 @@ export default function AddProjectPage() {
7475
disabled: isBusy,
7576
defaultValues: {
7677
name: "",
78+
repoUrl: "",
7779
maxCodeCharPerSymbol: 1000,
7880
maxCodeCharPerFile: 50000,
7981
maxCharPerSymbol: 2000,
@@ -102,6 +104,7 @@ export default function AddProjectPage() {
102104
try {
103105
const { url, method, body } = ProjectApiTypes.prepareCreateProject({
104106
name: values.name,
107+
repoUrl: values.repoUrl,
105108
workspaceId: selectedWorkspaceId,
106109
maxCodeCharPerSymbol: values.maxCodeCharPerSymbol,
107110
maxCodeCharPerFile: values.maxCodeCharPerFile,
@@ -325,6 +328,28 @@ export default function AddProjectPage() {
325328
</FormItem>
326329
)}
327330
/>
331+
<FormField
332+
control={form.control}
333+
name="repoUrl"
334+
render={({ field }) => (
335+
<FormItem>
336+
<FormLabel>
337+
Repository URL{" "}
338+
<span className="text-destructive">*</span>
339+
</FormLabel>
340+
<FormControl>
341+
<Input
342+
{...field}
343+
placeholder="https://github.com/user/repo"
344+
/>
345+
</FormControl>
346+
<FormDescription>
347+
The URL of the repository for this project
348+
</FormDescription>
349+
<FormMessage />
350+
</FormItem>
351+
)}
352+
/>
328353
</div>
329354

330355
{/* Configuration Tabs */}

‎packages/app/src/pages/projects/project/manifests/manifest.tsx‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,25 @@ export default function ProjectManifest() {
2121
ManifestApiTypes.GetManifestDetailsResponse | undefined
2222
>(undefined);
2323

24-
const [_dependencyManifest, setDependencyManifest] = useState<
24+
const [dependencyManifest, setDependencyManifest] = useState<
2525
DependencyManifest | undefined
2626
>(undefined);
2727

2828
const [auditManifest, setAuditManifest] = useState<
2929
ManifestApiTypes.GetManifestAuditResponse | undefined
3030
>(undefined);
3131

32+
async function fetchManifest(url: string) {
33+
const response = await fetch(url);
34+
if (!response.ok) {
35+
throw new Error("Failed to fetch manifest");
36+
}
37+
const manifest = await response
38+
.json() as unknown as DependencyManifest;
39+
40+
return manifest;
41+
}
42+
3243
useEffect(() => {
3344
async function fetchData() {
3445
if (!manifestId) {
@@ -53,9 +64,12 @@ export default function ProjectManifest() {
5364
throw new Error("Failed to fetch manifest");
5465
}
5566

56-
const manifest = await manifestResponse.json();
67+
const manifest = await manifestResponse
68+
.json() as ManifestApiTypes.GetManifestDetailsResponse;
5769
setManifestData(manifest);
58-
setDependencyManifest(manifest.manifest);
70+
71+
const dependencyManifest = await fetchManifest(manifest.manifest);
72+
setDependencyManifest(dependencyManifest);
5973

6074
// Fetch audit manifest
6175
const { url: auditUrl, method: auditMethod } = ManifestApiTypes
@@ -93,7 +107,7 @@ export default function ProjectManifest() {
93107
return (
94108
<DependencyVisualizer
95109
manifestId={manifestData?.id || 0}
96-
dependencyManifest={manifestData?.manifest as DependencyManifest}
110+
dependencyManifest={dependencyManifest as DependencyManifest}
97111
auditManifest={auditManifest as AuditManifest}
98112
/>
99113
);

‎packages/app/src/pages/projects/project/settings.tsx‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ export default function ProjectSettings() {
4646

4747
const formSchema = z.object({
4848
name: z.string().min(1, "Project name is required").max(100),
49+
repoUrl: z.string().url("Invalid repository URL"),
4950
maxCodeCharPerSymbol: z.number().int().min(1, "Must be at least 1"),
5051
maxCodeCharPerFile: z.number().int().min(1, "Must be at least 1"),
5152
maxCharPerSymbol: z.number().int().min(1, "Must be at least 1"),
@@ -73,6 +74,7 @@ export default function ProjectSettings() {
7374
disabled: isBusy,
7475
defaultValues: {
7576
name: context.project.name,
77+
repoUrl: context.project.repo_url,
7678
maxCodeCharPerSymbol: context.project.max_code_char_per_symbol,
7779
maxCodeCharPerFile: context.project.max_code_char_per_file,
7880
maxCharPerSymbol: context.project.max_char_per_symbol,
@@ -100,6 +102,7 @@ export default function ProjectSettings() {
100102
context.project.id,
101103
{
102104
name: values.name,
105+
repoUrl: values.repoUrl,
103106
maxCodeCharPerSymbol: values.maxCodeCharPerSymbol,
104107
maxCodeCharPerFile: values.maxCodeCharPerFile,
105108
maxCharPerSymbol: values.maxCharPerSymbol,
@@ -298,6 +301,28 @@ export default function ProjectSettings() {
298301
</FormItem>
299302
)}
300303
/>
304+
<FormField
305+
control={form.control}
306+
name="repoUrl"
307+
render={({ field }) => (
308+
<FormItem>
309+
<FormLabel>
310+
Repository URL{" "}
311+
<span className="text-destructive">*</span>
312+
</FormLabel>
313+
<FormControl>
314+
<Input
315+
{...field}
316+
placeholder="https://github.com/user/repo"
317+
/>
318+
</FormControl>
319+
<FormDescription>
320+
The URL of the repository for this project
321+
</FormDescription>
322+
<FormMessage />
323+
</FormItem>
324+
)}
325+
/>
301326
</div>
302327

303328
{/* Configuration Tabs */}
@@ -380,6 +405,7 @@ export default function ProjectSettings() {
380405
</Button>
381406
<Button
382407
variant="outline"
408+
type="button"
383409
onClick={() =>
384410
navigate(`/projects/${context.project.id}/manifests`)}
385411
disabled={isBusy}

‎packages/core/deno.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"build": "deno compile --allow-all --include src/db/migrations --output ./dist/core ./src/index.ts"
88
},
99
"imports": {
10+
"@google-cloud/storage": "npm:@google-cloud/storage@^7.16.0",
1011
"@oak/oak": "jsr:@oak/oak@^17.1.4",
1112
"@sentry/deno": "npm:@sentry/deno@^9.28.1",
1213
"@std/assert": "jsr:@std/assert@1",

‎packages/core/src/api/manifest/router.test.ts‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ Deno.test("create a manifest", async () => {
5151
userId,
5252
{
5353
name: "Test Project",
54+
repoUrl: "https://github.com/test/test",
5455
workspaceId: workspace.id,
5556
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
5657
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -141,6 +142,7 @@ Deno.test("create a manifest - non-member of workspace", async () => {
141142
userId,
142143
{
143144
name: "Test Project",
145+
repoUrl: "https://github.com/test/test",
144146
workspaceId: workspace.id,
145147
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
146148
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -260,6 +262,7 @@ Deno.test("create manifest - workspace access disabled", async () => {
260262
userId,
261263
{
262264
name: "Test Project",
265+
repoUrl: "https://github.com/test/test",
263266
workspaceId: workspace.id,
264267
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
265268
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -338,6 +341,7 @@ Deno.test("get manifests", async () => {
338341
userId,
339342
{
340343
name: "Test Project",
344+
repoUrl: "https://github.com/test/test",
341345
workspaceId: workspace.id,
342346
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
343347
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -429,6 +433,7 @@ Deno.test("get manifests with workspace filter", async () => {
429433
userId,
430434
{
431435
name: "Test Project",
436+
repoUrl: "https://github.com/test/test",
432437
workspaceId: workspace.id,
433438
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
434439
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -513,6 +518,7 @@ Deno.test("get manifests with search", async () => {
513518
userId,
514519
{
515520
name: "Test Project",
521+
repoUrl: "https://github.com/test/test",
516522
workspaceId: workspace.id,
517523
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
518524
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -608,6 +614,7 @@ Deno.test("get manifests - pagination", async () => {
608614
userId,
609615
{
610616
name: "Test Project",
617+
repoUrl: "https://github.com/test/test",
611618
workspaceId: workspace.id,
612619
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
613620
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -736,6 +743,7 @@ Deno.test("get manifest details", async () => {
736743
userId,
737744
{
738745
name: "Test Project",
746+
repoUrl: "https://github.com/test/test",
739747
workspaceId: workspace.id,
740748
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
741749
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -776,6 +784,12 @@ Deno.test("get manifest details", async () => {
776784
throw new Error(createResponse.error);
777785
}
778786

787+
await db
788+
.selectFrom("manifest")
789+
.selectAll()
790+
.where("id", "=", createResponse.id)
791+
.executeTakeFirstOrThrow();
792+
779793
const { url, method } = ManifestApiTypes.prepareGetManifestDetails(
780794
createResponse.id,
781795
);
@@ -796,8 +810,15 @@ Deno.test("get manifest details", async () => {
796810
assertEquals(manifest.branch, "main");
797811
assertEquals(manifest.commitSha, "abc123");
798812
assertEquals(manifest.version, 1);
799-
assertEquals(manifest.manifest.test, "data");
800-
assertEquals(manifest.manifest.complex.nested, "value");
813+
assertEquals(manifest.manifest.length > 0, true);
814+
815+
// get the content of the manifest from the bucket
816+
const manifestContentResponse = await fetch(
817+
manifest.manifest,
818+
);
819+
const manifestContent = await manifestContentResponse.json();
820+
assertEquals(manifestContent.test, "data");
821+
assertEquals(manifestContent.complex.nested, "value");
801822
} finally {
802823
await resetTables();
803824
await destroyKyselyDb();
@@ -827,6 +848,7 @@ Deno.test("get manifest details - non-member", async () => {
827848
userId,
828849
{
829850
name: "Test Project",
851+
repoUrl: "https://github.com/test/test",
830852
workspaceId: workspace.id,
831853
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
832854
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -915,6 +937,7 @@ Deno.test("delete a manifest", async () => {
915937
userId,
916938
{
917939
name: "Test Project",
940+
repoUrl: "https://github.com/test/test",
918941
workspaceId: workspace.id,
919942
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
920943
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -1007,6 +1030,7 @@ Deno.test("delete a manifest - non-member", async () => {
10071030
userId,
10081031
{
10091032
name: "Test Project",
1033+
repoUrl: "https://github.com/test/test",
10101034
workspaceId: workspace.id,
10111035
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
10121036
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -1104,6 +1128,7 @@ Deno.test("get manifest audit", async () => {
11041128
userId,
11051129
{
11061130
name: "Test Project",
1131+
repoUrl: "https://github.com/test/test",
11071132
workspaceId: workspace.id,
11081133
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
11091134
maxCodeCharPerFile: config.maxCodeCharPerFile,
@@ -1255,6 +1280,7 @@ Deno.test("get manifest audit - non-member", async () => {
12551280
userId,
12561281
{
12571282
name: "Test Project",
1283+
repoUrl: "https://github.com/test/test",
12581284
workspaceId: workspace.id,
12591285
maxCodeCharPerSymbol: config.maxCodeCharPerSymbol,
12601286
maxCodeCharPerFile: config.maxCodeCharPerFile,

‎packages/core/src/api/manifest/service.ts‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ import type {
88
GetManifestDetailsResponse,
99
GetManifestsResponse,
1010
} from "./types.ts";
11+
import {
12+
downloadJsonFromBucket,
13+
getPublicLink,
14+
uploadJsonToBucket,
15+
} from "../../bucketStorage/index.ts";
1116

1217
export const manifestNotFoundError = "manifest_not_found";
1318
export const projectNotFoundError = "project_not_found";
@@ -66,6 +71,9 @@ export class ManifestService {
6671
return { error: accessDisabledError };
6772
}
6873

74+
const manifestFileName = `${projectId}-${Date.now()}.json`;
75+
await uploadJsonToBucket(manifest, manifestFileName);
76+
6977
// Create the manifest
7078
const newManifest = await db
7179
.insertInto("manifest")
@@ -75,7 +83,7 @@ export class ManifestService {
7583
commitSha,
7684
commitShaDate,
7785
version: settings.MANIFEST.DEFAULT_VERSION,
78-
manifest,
86+
manifest: manifestFileName,
7987
created_at: new Date(),
8088
})
8189
.returningAll()
@@ -216,7 +224,12 @@ export class ManifestService {
216224
return { error: manifestNotFoundError };
217225
}
218226

219-
return manifest;
227+
const publicLink = await getPublicLink(manifest.manifest);
228+
229+
return {
230+
...manifest,
231+
manifest: publicLink,
232+
};
220233
}
221234

222235
/**
@@ -284,10 +297,14 @@ export class ManifestService {
284297
return { error: projectNotFoundError };
285298
}
286299

300+
const manifestJson = await downloadJsonFromBucket(
301+
manifest.manifest,
302+
) as DependencyManifest;
303+
287304
try {
288305
const auditManifest = generateAuditManifest(
289306
manifest.version,
290-
manifest.manifest as DependencyManifest,
307+
manifestJson,
291308
{
292309
file: {
293310
maxCodeChar: project.max_char_per_file,

0 commit comments

Comments
 (0)