From 5fbd2312a9e59d31ff0dd297c9b43665872e1395 Mon Sep 17 00:00:00 2001 From: Dominika Zemanovicova Date: Fri, 19 Jun 2026 09:49:20 +0200 Subject: [PATCH 1/4] Introduce collectors contract Assisted-By: Cursor Desktop Signed-off-by: Dominika Zemanovicova --- .../scorecard/.changeset/fair-jeans-jump.md | 6 + .../plugins/scorecard-backend/src/plugin.ts | 16 +++ .../src/providers/CollectorRegistry.test.ts | 65 +++++++++ .../src/providers/CollectorRegistry.ts | 46 +++++++ .../plugins/scorecard-node/package.json | 3 +- .../scorecard-node/src/api/Collector.ts | 65 +++++++++ .../src/api/collectWithContract.test.ts | 129 ++++++++++++++++++ .../src/api/collectWithContract.ts | 108 +++++++++++++++ .../plugins/scorecard-node/src/api/index.ts | 3 + .../plugins/scorecard-node/src/extensions.ts | 21 ++- workspaces/scorecard/yarn.lock | 1 + 11 files changed, 461 insertions(+), 2 deletions(-) create mode 100644 workspaces/scorecard/.changeset/fair-jeans-jump.md create mode 100644 workspaces/scorecard/plugins/scorecard-backend/src/providers/CollectorRegistry.test.ts create mode 100644 workspaces/scorecard/plugins/scorecard-backend/src/providers/CollectorRegistry.ts create mode 100644 workspaces/scorecard/plugins/scorecard-node/src/api/Collector.ts create mode 100644 workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.test.ts create mode 100644 workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.ts diff --git a/workspaces/scorecard/.changeset/fair-jeans-jump.md b/workspaces/scorecard/.changeset/fair-jeans-jump.md new file mode 100644 index 0000000000..816752e7ec --- /dev/null +++ b/workspaces/scorecard/.changeset/fair-jeans-jump.md @@ -0,0 +1,6 @@ +--- +'@red-hat-developer-hub/backstage-plugin-scorecard-backend': minor +'@red-hat-developer-hub/backstage-plugin-scorecard-node': minor +--- + +Introduce collectors extension point to use by metric providers to collect data from different datasources. diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/plugin.ts b/workspaces/scorecard/plugins/scorecard-backend/src/plugin.ts index 7ed04a12d5..0e965fc27c 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/plugin.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/plugin.ts @@ -21,9 +21,11 @@ import { createRouter } from './service/router'; import { catalogServiceRef } from '@backstage/plugin-catalog-node'; import { MetricProvider, + scorecardCollectorsExtensionPoint, scorecardMetricsExtensionPoint, } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; import { MetricProvidersRegistry } from './providers/MetricProvidersRegistry'; +import { CollectorRegistry } from './providers/CollectorRegistry'; import { CatalogMetricService } from './service/CatalogMetricService'; import { ThresholdEvaluator } from './threshold/ThresholdEvaluator'; import { scorecardPermissions } from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; @@ -47,6 +49,7 @@ export const scorecardPlugin = createBackendPlugin({ pluginId: 'scorecard', register(env) { const metricProvidersRegistry = new MetricProvidersRegistry(); + const collectorRegistry = new CollectorRegistry(); env.registerExtensionPoint(scorecardMetricsExtensionPoint, { addMetricProvider(...newMetricProviders: MetricProvider[]) { @@ -55,6 +58,19 @@ export const scorecardPlugin = createBackendPlugin({ }); }, }); + env.registerExtensionPoint(scorecardCollectorsExtensionPoint, { + addCollector(...collectors) { + collectors.forEach(collector => { + collectorRegistry.register(collector); + }); + }, + getCollector(collectorId: string) { + return collectorRegistry.getCollector(collectorId); + }, + hasCollector(collectorId: string) { + return collectorRegistry.hasCollector(collectorId); + }, + }); env.registerInit({ deps: { diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/providers/CollectorRegistry.test.ts b/workspaces/scorecard/plugins/scorecard-backend/src/providers/CollectorRegistry.test.ts new file mode 100644 index 0000000000..07c9e84eca --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-backend/src/providers/CollectorRegistry.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConflictError, NotFoundError } from '@backstage/errors'; +import type { Collector } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; +import { z } from 'zod'; +import { CollectorRegistry } from './CollectorRegistry'; + +describe('CollectorRegistry', () => { + const collector: Collector = { + getCollectorId: () => 'github:deployments', + getCollectorDescription: () => 'Collect github deployments', + getInputSchema: () => + z.object({ + from: z.string().datetime(), + to: z.string().datetime(), + }), + getOutputSchema: () => + z.object({ + deployments: z.array(z.object({ sha: z.string() })), + }), + collect: jest.fn(), + }; + + it('registers and resolves collector by id', () => { + const registry = new CollectorRegistry(); + registry.register(collector); + + expect(registry.hasCollector(collector.getCollectorId())).toBe(true); + expect(registry.getCollector(collector.getCollectorId())).toBe(collector); + }); + + it('throws on duplicate collector id', () => { + const registry = new CollectorRegistry(); + registry.register(collector); + + expect(() => registry.register(collector)).toThrow( + new ConflictError( + "Collector with ID 'github:deployments' has already been registered", + ), + ); + }); + + it('throws on missing collector', () => { + const registry = new CollectorRegistry(); + expect(() => registry.getCollector('missing.collector')).toThrow( + new NotFoundError( + "No collector registered for collector ID 'missing.collector'.", + ), + ); + }); +}); diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/providers/CollectorRegistry.ts b/workspaces/scorecard/plugins/scorecard-backend/src/providers/CollectorRegistry.ts new file mode 100644 index 0000000000..3668ea7328 --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-backend/src/providers/CollectorRegistry.ts @@ -0,0 +1,46 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConflictError, NotFoundError } from '@backstage/errors'; +import type { Collector } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; + +export class CollectorRegistry { + private readonly collectors = new Map(); + + register(collector: Collector): void { + const collectorId = collector.getCollectorId(); + if (this.collectors.has(collectorId)) { + throw new ConflictError( + `Collector with ID '${collectorId}' has already been registered`, + ); + } + this.collectors.set(collectorId, collector); + } + + getCollector(collectorId: string): Collector { + const collector = this.collectors.get(collectorId); + if (!collector) { + throw new NotFoundError( + `No collector registered for collector ID '${collectorId}'.`, + ); + } + return collector; + } + + hasCollector(collectorId: string): boolean { + return this.collectors.has(collectorId); + } +} diff --git a/workspaces/scorecard/plugins/scorecard-node/package.json b/workspaces/scorecard/plugins/scorecard-node/package.json index 7828ab6905..93125743d4 100644 --- a/workspaces/scorecard/plugins/scorecard-node/package.json +++ b/workspaces/scorecard/plugins/scorecard-node/package.json @@ -46,7 +46,8 @@ "@backstage/catalog-model": "^1.7.7", "@backstage/errors": "^1.2.7", "@backstage/plugin-catalog-node": "^2.1.0", - "@red-hat-developer-hub/backstage-plugin-scorecard-common": "workspace:^" + "@red-hat-developer-hub/backstage-plugin-scorecard-common": "workspace:^", + "zod": "^3.22.4" }, "devDependencies": { "@backstage/backend-test-utils": "^1.11.1", diff --git a/workspaces/scorecard/plugins/scorecard-node/src/api/Collector.ts b/workspaces/scorecard/plugins/scorecard-node/src/api/Collector.ts new file mode 100644 index 0000000000..bc83fde8bb --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-node/src/api/Collector.ts @@ -0,0 +1,65 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Entity } from '@backstage/catalog-model'; +import type { z } from 'zod'; + +/** + * Collector used by metric providers to gather datasource-specific data. + * @public + */ +export interface Collector< + TInputSchema extends z.ZodTypeAny = z.ZodTypeAny, + TOutputSchema extends z.ZodTypeAny = z.ZodTypeAny, +> { + /** + * Get the collector unique ID. + * @public + */ + getCollectorId(): string; + /** + * Human-readable collector description. + * @public + */ + getCollectorDescription(): string; + /** + * Input schema accepted by collect(). + * @public + */ + getInputSchema(): TInputSchema; + /** + * Output schema returned by collect(). + * @public + */ + getOutputSchema(): TOutputSchema; + /** + * Collect data for an entity and a collector-specific input payload. + * @public + */ + collect(options: { + entity: Entity; + input: z.infer; + }): Promise>; +} + +/** + * Minimal collector registry to resolve collectors by ID. + * @public + */ +export interface CollectorRegistry { + getCollector(collectorId: string): Collector; + hasCollector(collectorId: string): boolean; +} diff --git a/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.test.ts b/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.test.ts new file mode 100644 index 0000000000..f86e6cad3c --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Entity } from '@backstage/catalog-model'; +import { z } from 'zod'; +import type { Collector, CollectorRegistry } from './Collector'; +import { collectWithContract } from './collectWithContract'; + +const entity: Entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { name: 'service-a', namespace: 'default' }, +}; + +describe('collectWithContract', () => { + const collectorId = 'test.collector'; + + const inputSchema = z.object({ + from: z.string().datetime(), + to: z.string().datetime(), + }); + const outputSchema = z.object({ + deployments: z.array(z.object({ sha: z.string() })), + }); + + const collector: Collector = { + getCollectorId: () => collectorId, + getCollectorDescription: () => 'test collector', + getInputSchema: () => inputSchema, + getOutputSchema: () => outputSchema, + collect: jest.fn(async () => ({ + deployments: [{ sha: 'abc123' }], + })), + }; + + const collectorRegistry: CollectorRegistry = { + getCollector: () => collector, + hasCollector: () => true, + }; + + it('collects successfully when provider and collector contracts are compatible', async () => { + const result = await collectWithContract({ + collectorRegistry, + collectorId, + contract: { + inputSchema, + outputSchema, + }, + entity, + input: { + from: '2026-06-01T00:00:00.000Z', + to: '2026-06-08T00:00:00.000Z', + }, + }); + + expect(result).toEqual({ + deployments: [{ sha: 'abc123' }], + }); + }); + + it('fails when provider input schema does not pass', async () => { + await expect( + collectWithContract({ + collectorRegistry, + collectorId, + contract: { + inputSchema, + outputSchema, + }, + entity, + input: { from: 'invalid', to: 'still-invalid' }, + }), + ).rejects.toThrow('Invalid input for collector'); + }); + + it('fails when collector output does not satisfy provider expected output', async () => { + const outputMismatchCollector: Collector = { + ...collector, + getOutputSchema: () => + z.object({ + deployments: z.array(z.object({ sha: z.string(), id: z.number() })), + }), + collect: jest.fn(async () => ({ + deployments: [{ sha: 'abc123', id: 1 }], + })), + }; + + const outputMismatchCollectorRegistry: CollectorRegistry = { + getCollector: () => outputMismatchCollector, + hasCollector: () => true, + }; + + await expect( + collectWithContract({ + collectorRegistry: outputMismatchCollectorRegistry, + collectorId, + contract: { + inputSchema, + outputSchema: z.object({ + deployments: z.array( + z.object({ + sha: z.string(), + mergedAt: z.string(), + }), + ), + }), + }, + entity, + input: { + from: '2026-06-01T00:00:00.000Z', + to: '2026-06-08T00:00:00.000Z', + }, + }), + ).rejects.toThrow('does not satisfy provider expected schema'); + }); +}); diff --git a/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.ts b/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.ts new file mode 100644 index 0000000000..fb76cecff7 --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.ts @@ -0,0 +1,108 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Entity } from '@backstage/catalog-model'; +import { InputError } from '@backstage/errors'; +import type { z } from 'zod'; +import type { CollectorRegistry } from './Collector'; + +/** + * Provider-side collector contract expected by a metric provider. + * @public + */ +export type ProviderCollectorContract< + TInputSchema extends z.ZodTypeAny, + TOutputSchema extends z.ZodTypeAny, +> = { + inputSchema: TInputSchema; + outputSchema: TOutputSchema; +}; + +/** + * Resolve collector by id and execute collect with bidirectional schema checks: + * - provider input schema + * - collector input schema + * - collector output schema + * - provider output schema + * @public + */ +export const collectWithContract = async < + TInputSchema extends z.ZodTypeAny, + TOutputSchema extends z.ZodTypeAny, +>(options: { + collectorRegistry: CollectorRegistry; + collectorId: string; + contract: ProviderCollectorContract; + entity: Entity; + input: unknown; +}): Promise> => { + const collector = options.collectorRegistry.getCollector(options.collectorId); + + const providerInput = options.contract.inputSchema.safeParse(options.input); + if (!providerInput.success) { + throw new InputError( + `Invalid input for collector "${ + options.collectorId + }" expected by provider: ${providerInput.error.issues + .map(issue => issue.message) + .join('; ')}`, + ); + } + + const collectorInput = collector + .getInputSchema() + .safeParse(providerInput.data); + if (!collectorInput.success) { + throw new InputError( + `Input does not satisfy collector "${ + options.collectorId + }" input schema: ${collectorInput.error.issues + .map(issue => issue.message) + .join('; ')}`, + ); + } + + const rawOutput = await collector.collect({ + entity: options.entity, + input: collectorInput.data, + }); + + const collectorOutput = collector.getOutputSchema().safeParse(rawOutput); + if (!collectorOutput.success) { + throw new InputError( + `Collector "${ + options.collectorId + }" returned output that does not satisfy collector schema: ${collectorOutput.error.issues + .map(issue => issue.message) + .join('; ')}`, + ); + } + + const providerOutput = options.contract.outputSchema.safeParse( + collectorOutput.data, + ); + if (!providerOutput.success) { + throw new InputError( + `Collector "${ + options.collectorId + }" output does not satisfy provider expected schema: ${providerOutput.error.issues + .map(issue => issue.message) + .join('; ')}`, + ); + } + + return providerOutput.data; +}; diff --git a/workspaces/scorecard/plugins/scorecard-node/src/api/index.ts b/workspaces/scorecard/plugins/scorecard-node/src/api/index.ts index 0d88143944..96abb33efd 100644 --- a/workspaces/scorecard/plugins/scorecard-node/src/api/index.ts +++ b/workspaces/scorecard/plugins/scorecard-node/src/api/index.ts @@ -14,4 +14,7 @@ * limitations under the License. */ +export type { Collector, CollectorRegistry } from './Collector'; +export { collectWithContract } from './collectWithContract'; +export type { ProviderCollectorContract } from './collectWithContract'; export type { MetricProvider } from './MetricProvider'; diff --git a/workspaces/scorecard/plugins/scorecard-node/src/extensions.ts b/workspaces/scorecard/plugins/scorecard-node/src/extensions.ts index 7bf5bdac91..0bf706cbec 100644 --- a/workspaces/scorecard/plugins/scorecard-node/src/extensions.ts +++ b/workspaces/scorecard/plugins/scorecard-node/src/extensions.ts @@ -15,7 +15,7 @@ */ import { createExtensionPoint } from '@backstage/backend-plugin-api'; -import { MetricProvider } from './api'; +import { Collector, MetricProvider } from './api'; /** * Interface for the Scorecard metrics extension point @@ -33,3 +33,22 @@ export const scorecardMetricsExtensionPoint = createExtensionPoint({ id: 'scorecard.metrics', }); + +/** + * Interface for the Scorecard collectors extension point + * @public + */ +export interface ScorecardCollectorsExtensionPoint { + addCollector(...collectors: Array): void; + getCollector(collectorId: string): Collector; + hasCollector(collectorId: string): boolean; +} + +/** + * Extension point for adding and consuming collectors in the Scorecard plugin + * @public + */ +export const scorecardCollectorsExtensionPoint = + createExtensionPoint({ + id: 'scorecard.collectors', + }); diff --git a/workspaces/scorecard/yarn.lock b/workspaces/scorecard/yarn.lock index 8f5b9e083f..c9220ee210 100644 --- a/workspaces/scorecard/yarn.lock +++ b/workspaces/scorecard/yarn.lock @@ -12350,6 +12350,7 @@ __metadata: "@backstage/plugin-catalog-node": "npm:^2.1.0" "@backstage/types": "npm:^1.2.2" "@red-hat-developer-hub/backstage-plugin-scorecard-common": "workspace:^" + zod: "npm:^3.22.4" languageName: unknown linkType: soft From 1d8a564f5d72e76e1eb7b6917ae5535e07e24034 Mon Sep 17 00:00:00 2001 From: Dominika Zemanovicova Date: Mon, 22 Jun 2026 10:52:32 +0200 Subject: [PATCH 2/4] Add tests and rename Assisted-By: Cursor Desktop Signed-off-by: Dominika Zemanovicova --- .../src/api/collectWithContract.test.ts | 206 ++++++++++++++---- .../src/api/collectWithContract.ts | 26 +-- .../plugins/scorecard-node/src/api/index.ts | 2 +- 3 files changed, 178 insertions(+), 56 deletions(-) diff --git a/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.test.ts b/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.test.ts index f86e6cad3c..b808fbb86f 100644 --- a/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.test.ts +++ b/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.test.ts @@ -14,50 +14,60 @@ * limitations under the License. */ -import type { Entity } from '@backstage/catalog-model'; import { z } from 'zod'; -import type { Collector, CollectorRegistry } from './Collector'; +import { Collector, CollectorRegistry } from './Collector'; import { collectWithContract } from './collectWithContract'; -const entity: Entity = { +const entity = { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { name: 'service-a', namespace: 'default' }, }; describe('collectWithContract', () => { - const collectorId = 'test.collector'; + const collectorId = 'test:collector'; - const inputSchema = z.object({ + const contractInputSchema = z.object({ from: z.string().datetime(), to: z.string().datetime(), }); - const outputSchema = z.object({ + const collectorInputSchema = z.object({ + from: z.string().datetime(), + to: z.string().datetime(), + }); + const collectorOutputSchema = z.object({ + deployments: z.array(z.object({ sha: z.string() })), + }); + const contractOutputSchema = z.object({ deployments: z.array(z.object({ sha: z.string() })), }); - const collector: Collector = { + const makeCollector = (overrides: Partial = {}): Collector => ({ getCollectorId: () => collectorId, - getCollectorDescription: () => 'test collector', - getInputSchema: () => inputSchema, - getOutputSchema: () => outputSchema, + getCollectorDescription: () => 'Test collector', + getInputSchema: () => collectorInputSchema, + getOutputSchema: () => collectorOutputSchema, collect: jest.fn(async () => ({ deployments: [{ sha: 'abc123' }], })), - }; + ...overrides, + }); - const collectorRegistry: CollectorRegistry = { - getCollector: () => collector, + const makeCollectorRegistry = (collector: Collector): CollectorRegistry => ({ + getCollector: jest.fn(() => collector), hasCollector: () => true, - }; + }); + + it('collects successfully when collector and contract schemas are compatible', async () => { + const collector = makeCollector(); + const collectorRegistry = makeCollectorRegistry(collector); - it('collects successfully when provider and collector contracts are compatible', async () => { const result = await collectWithContract({ collectorRegistry, collectorId, contract: { - inputSchema, - outputSchema, + inputSchema: contractInputSchema, + outputSchema: contractOutputSchema, }, entity, input: { @@ -71,52 +81,162 @@ describe('collectWithContract', () => { }); }); - it('fails when provider input schema does not pass', async () => { + it('forwards input and entity to collector.collect', async () => { + const collect = jest.fn(async () => ({ + deployments: [{ sha: 'abc123' }], + })); + const collector = makeCollector({ collect }); + const collectorRegistry = makeCollectorRegistry(collector); + + await collectWithContract({ + collectorRegistry, + collectorId, + contract: { + inputSchema: contractInputSchema, + outputSchema: contractOutputSchema, + }, + entity, + input: { + from: '2026-06-01T00:00:00.000Z', + to: '2026-06-08T00:00:00.000Z', + }, + }); + + expect(collect).toHaveBeenCalledWith({ + entity, + input: { + from: '2026-06-01T00:00:00.000Z', + to: '2026-06-08T00:00:00.000Z', + }, + }); + }); + + it('fails when input does not satisfy contract input schema', async () => { + const collector = makeCollector(); + const collectorRegistry = makeCollectorRegistry(collector); + await expect( collectWithContract({ collectorRegistry, collectorId, contract: { - inputSchema, - outputSchema, + inputSchema: contractInputSchema, + outputSchema: contractOutputSchema, }, entity, - input: { from: 'invalid', to: 'still-invalid' }, + input: { from: 'invalid', to: 5 }, }), - ).rejects.toThrow('Invalid input for collector'); + ).rejects.toThrow('input does not satisfy contract input schema'); }); - it('fails when collector output does not satisfy provider expected output', async () => { - const outputMismatchCollector: Collector = { - ...collector, + it('fails when contract input does not satisfy collector input schema', async () => { + const collector = makeCollector({ + getInputSchema: () => + z.object({ + from: z.string().datetime(), + to: z.string().datetime(), + environment: z.string().min(1), + }), + }); + const collectorRegistry = makeCollectorRegistry(collector); + + await expect( + collectWithContract({ + collectorRegistry, + collectorId, + contract: { + inputSchema: contractInputSchema, + outputSchema: contractOutputSchema, + }, + entity, + input: { + from: '2026-06-01T00:00:00.000Z', + to: '2026-06-08T00:00:00.000Z', + }, + }), + ).rejects.toThrow('Input does not satisfy collector'); + }); + + it('fails when collector output does not satisfy collector output schema', async () => { + const collector = makeCollector({ getOutputSchema: () => z.object({ - deployments: z.array(z.object({ sha: z.string(), id: z.number() })), + deployments: z.array( + z.object({ + sha: z.string(), + id: z.number(), + }), + ), }), collect: jest.fn(async () => ({ - deployments: [{ sha: 'abc123', id: 1 }], + deployments: [{ sha: 'abc123' }], })), - }; + }); + const collectorRegistry = makeCollectorRegistry(collector); + + await expect( + collectWithContract({ + collectorRegistry, + collectorId, + contract: { + inputSchema: contractInputSchema, + outputSchema: contractOutputSchema, + }, + entity, + input: { + from: '2026-06-01T00:00:00.000Z', + to: '2026-06-08T00:00:00.000Z', + }, + }), + ).rejects.toThrow('returned output that does not satisfy collector schema'); + }); - const outputMismatchCollectorRegistry: CollectorRegistry = { - getCollector: () => outputMismatchCollector, - hasCollector: () => true, + it('fails when collector output does not satisfy contract output schema', async () => { + const collector = makeCollector({ + getOutputSchema: () => + z.object({ + deployments: z.array(z.object({ id: z.number() })), + }), + collect: jest.fn(async () => ({ + deployments: [{ id: 1 }], + })), + }); + const collectorRegistry = makeCollectorRegistry(collector); + + await expect( + collectWithContract({ + collectorRegistry, + collectorId, + contract: { + inputSchema: contractInputSchema, + outputSchema: contractOutputSchema, + }, + entity, + input: { + from: '2026-06-01T00:00:00.000Z', + to: '2026-06-08T00:00:00.000Z', + }, + }), + ).rejects.toThrow('output does not satisfy contract output schema'); + }); + + it('propagates collector lookup errors', async () => { + const collectorRegistry = { + getCollector: () => { + throw new Error( + `No collector registered for collector ID '${collectorId}'`, + ); + }, + hasCollector: () => false, }; await expect( collectWithContract({ - collectorRegistry: outputMismatchCollectorRegistry, + collectorRegistry, collectorId, contract: { - inputSchema, - outputSchema: z.object({ - deployments: z.array( - z.object({ - sha: z.string(), - mergedAt: z.string(), - }), - ), - }), + inputSchema: contractInputSchema, + outputSchema: contractOutputSchema, }, entity, input: { @@ -124,6 +244,8 @@ describe('collectWithContract', () => { to: '2026-06-08T00:00:00.000Z', }, }), - ).rejects.toThrow('does not satisfy provider expected schema'); + ).rejects.toThrow( + "No collector registered for collector ID 'test:collector'", + ); }); }); diff --git a/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.ts b/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.ts index fb76cecff7..440aa098a8 100644 --- a/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.ts +++ b/workspaces/scorecard/plugins/scorecard-node/src/api/collectWithContract.ts @@ -20,10 +20,10 @@ import type { z } from 'zod'; import type { CollectorRegistry } from './Collector'; /** - * Provider-side collector contract expected by a metric provider. + * Collector contract expected by caller. * @public */ -export type ProviderCollectorContract< +export type CollectorContract< TInputSchema extends z.ZodTypeAny, TOutputSchema extends z.ZodTypeAny, > = { @@ -33,10 +33,10 @@ export type ProviderCollectorContract< /** * Resolve collector by id and execute collect with bidirectional schema checks: - * - provider input schema + * - contract input schema * - collector input schema * - collector output schema - * - provider output schema + * - contract output schema * @public */ export const collectWithContract = async < @@ -45,18 +45,18 @@ export const collectWithContract = async < >(options: { collectorRegistry: CollectorRegistry; collectorId: string; - contract: ProviderCollectorContract; + contract: CollectorContract; entity: Entity; input: unknown; }): Promise> => { const collector = options.collectorRegistry.getCollector(options.collectorId); - const providerInput = options.contract.inputSchema.safeParse(options.input); - if (!providerInput.success) { + const contractInput = options.contract.inputSchema.safeParse(options.input); + if (!contractInput.success) { throw new InputError( `Invalid input for collector "${ options.collectorId - }" expected by provider: ${providerInput.error.issues + }": input does not satisfy contract input schema: ${contractInput.error.issues .map(issue => issue.message) .join('; ')}`, ); @@ -64,7 +64,7 @@ export const collectWithContract = async < const collectorInput = collector .getInputSchema() - .safeParse(providerInput.data); + .safeParse(contractInput.data); if (!collectorInput.success) { throw new InputError( `Input does not satisfy collector "${ @@ -91,18 +91,18 @@ export const collectWithContract = async < ); } - const providerOutput = options.contract.outputSchema.safeParse( + const contractOutput = options.contract.outputSchema.safeParse( collectorOutput.data, ); - if (!providerOutput.success) { + if (!contractOutput.success) { throw new InputError( `Collector "${ options.collectorId - }" output does not satisfy provider expected schema: ${providerOutput.error.issues + }" output does not satisfy contract output schema: ${contractOutput.error.issues .map(issue => issue.message) .join('; ')}`, ); } - return providerOutput.data; + return contractOutput.data; }; diff --git a/workspaces/scorecard/plugins/scorecard-node/src/api/index.ts b/workspaces/scorecard/plugins/scorecard-node/src/api/index.ts index 96abb33efd..cd6e309793 100644 --- a/workspaces/scorecard/plugins/scorecard-node/src/api/index.ts +++ b/workspaces/scorecard/plugins/scorecard-node/src/api/index.ts @@ -16,5 +16,5 @@ export type { Collector, CollectorRegistry } from './Collector'; export { collectWithContract } from './collectWithContract'; -export type { ProviderCollectorContract } from './collectWithContract'; +export type { CollectorContract } from './collectWithContract'; export type { MetricProvider } from './MetricProvider'; From 8572a8d75fafbeff1e2f78888a793d1052ae8068 Mon Sep 17 00:00:00 2001 From: Dominika Zemanovicova Date: Mon, 22 Jun 2026 11:03:30 +0200 Subject: [PATCH 3/4] Add docs Assisted-By: Cursor Desktop Signed-off-by: Dominika Zemanovicova --- .../plugins/scorecard-backend/README.md | 6 + .../scorecard-backend/docs/collectors.md | 155 ++++++++++++++++++ .../scorecard-backend/docs/providers.md | 9 + 3 files changed, 170 insertions(+) create mode 100644 workspaces/scorecard/plugins/scorecard-backend/docs/collectors.md diff --git a/workspaces/scorecard/plugins/scorecard-backend/README.md b/workspaces/scorecard/plugins/scorecard-backend/README.md index 37ae4b2123..2cdb0fd97e 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/README.md +++ b/workspaces/scorecard/plugins/scorecard-backend/README.md @@ -71,6 +71,12 @@ This policy would allow users to read only the GitHub Open PRs metric, while res The Scorecard plugin collects metrics from third-party data sources using metric providers. The Scorecard node plugin provides `scorecardMetricsExtensionPoint` extension point that is used to connect your backend plugin module that exports custom metrics via metric providers to the Scorecard backend plugin. For detailed information on creating metric providers, see [providers.md](./docs/providers.md). +### Collectors + +Collectors are reusable data-fetching contracts used by metric providers. They are registered through `scorecardCollectorsExtensionPoint` and consumed via `collectWithContract`. + +For details and examples, see [collectors.md](./docs/collectors.md). + ### Metric Collection Scheduling The Scorecard plugin automatically collects metrics on a scheduled basis. You can customize the schedule for any metric provider in your `app-config.yaml`. If no schedule is configured, metric providers use the following default schedule: diff --git a/workspaces/scorecard/plugins/scorecard-backend/docs/collectors.md b/workspaces/scorecard/plugins/scorecard-backend/docs/collectors.md new file mode 100644 index 0000000000..cc85685155 --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-backend/docs/collectors.md @@ -0,0 +1,155 @@ +# Collectors + +Collectors are reusable data-fetching components used by metric providers. + +Use collectors when: + +- multiple metric providers share datasource logic (including auth/API client setup) +- you are building composite providers that combine data from one or more datasources +- you want to keep providers focused on metric calculation logic + +## Core APIs + +Collector APIs are provided by `@red-hat-developer-hub/backstage-plugin-scorecard-node`: + +- `Collector` +- `CollectorRegistry` +- `CollectorContract` +- `collectWithContract` +- `scorecardCollectorsExtensionPoint` + +## Collector ID convention + +Use `source:name` for collector IDs (for example `github:deployments`, `github:commit-pulls`). + +This keeps collector IDs visually distinct from metric/provider IDs that use dot notation. + +## Create a collector + +Implement `Collector` with an input schema, output schema, and collect function: + +```ts +import type { Entity } from '@backstage/catalog-model'; +import type { Collector } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; +import { z } from 'zod'; + +export class MyDeploymentsCollector + implements + Collector< + (typeof MyDeploymentsCollector)['inputSchema'], + (typeof MyDeploymentsCollector)['outputSchema'] + > +{ + static readonly inputSchema = z.object({ + from: z.string().datetime(), + to: z.string().datetime(), + }); + + static readonly outputSchema = z.object({ + deployments: z.array( + z.object({ + id: z.number(), + sha: z.string(), + createdAt: z.string(), + }), + ), + }); + + getCollectorId(): string { + return 'my-source:deployments'; + } + + getCollectorDescription(): string { + return 'Collect deployments in a time window'; + } + + getInputSchema() { + return MyDeploymentsCollector.inputSchema; + } + + getOutputSchema() { + return MyDeploymentsCollector.outputSchema; + } + + async collect(options: { + entity: Entity; + input: z.infer<(typeof MyDeploymentsCollector)['inputSchema']>; + }): Promise> { + return { + deployments: [], + }; + } +} +``` + +## Register collectors in a backend module + +Register collectors through `scorecardCollectorsExtensionPoint`: + +```ts +import { createBackendModule } from '@backstage/backend-plugin-api'; +import { scorecardCollectorsExtensionPoint } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; +import { MyDeploymentsCollector } from './collectors/MyDeploymentsCollector'; + +export const scorecardModuleMySource = createBackendModule({ + pluginId: 'scorecard', + moduleId: 'my-source', + register(reg) { + reg.registerInit({ + deps: { + collectors: scorecardCollectorsExtensionPoint, + }, + async init({ collectors }) { + collectors.addCollector(new MyDeploymentsCollector()); + }, + }); + }, +}); +``` + +## Use collectors from a metric provider + +Use `collectWithContract` to validate both sides of the contract (provider and collector expected input and output): + +```ts +import type { Entity } from '@backstage/catalog-model'; +import { + collectWithContract, + type CollectorRegistry, + type MetricProvider, +} from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; +import { z } from 'zod'; + +const inputSchema = z.object({ + from: z.string().datetime(), + to: z.string().datetime(), +}); + +const outputSchema = z.object({ + deployments: z.array(z.object({ id: z.number(), sha: z.string() })), +}); + +export class MyMetricProvider implements MetricProvider<'number'> { + constructor(private readonly collectorRegistry: CollectorRegistry) {} + + // Other MetricProvider methods omitted + + async calculateMetric(entity: Entity): Promise { + const collected = await collectWithContract({ + collectorRegistry: this.collectorRegistry, + collectorId: 'my-source:deployments', + contract: { + inputSchema, + outputSchema, + }, + entity, + input: { + from: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), + to: new Date().toISOString(), + }, + }); + + return collected.deployments.length; + } +} +``` diff --git a/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md b/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md index bcc8883137..806265e93c 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md +++ b/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md @@ -2,6 +2,15 @@ The Scorecard plugin collects metrics from third-party data sources using metric providers. The Scorecard node plugin provides `scorecardMetricsExtensionPoint` extension point that is used to connect your backend plugin module that exports custom metrics via metric providers to the Scorecard backend plugin. In this documentation, we will discuss how to create a simple metric provider backend module that will be used to collect and calculate metrics. +For provider data-fetching reuse across datasources, Scorecard provides collector contracts: + +- `scorecardCollectorsExtensionPoint` to register collectors +- `collectWithContract(...)` helper to call a collector with dual validation: + - provider expected input/output schemas + - collector declared input/output schemas + +For details and examples, see [collectors.md](./docs/collectors.md). + ## Getting started First step is to create a metric provider backend module using the following command: From 039281e63cd1254f26c4fed821258263f7b31382 Mon Sep 17 00:00:00 2001 From: Dominika Zemanovicova Date: Mon, 22 Jun 2026 12:52:08 +0200 Subject: [PATCH 4/4] Add api report Signed-off-by: Dominika Zemanovicova --- .../plugins/scorecard-node/report.api.md | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/workspaces/scorecard/plugins/scorecard-node/report.api.md b/workspaces/scorecard/plugins/scorecard-node/report.api.md index 0e1d830511..191a04193b 100644 --- a/workspaces/scorecard/plugins/scorecard-node/report.api.md +++ b/workspaces/scorecard/plugins/scorecard-node/report.api.md @@ -13,6 +13,51 @@ import { MetricType } from '@red-hat-developer-hub/backstage-plugin-scorecard-co import { MetricValue } from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; import { ThresholdConfig } from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; import { ThresholdRule } from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; +import type { z } from 'zod'; + +// @public +export interface Collector< + TInputSchema extends z.ZodTypeAny = z.ZodTypeAny, + TOutputSchema extends z.ZodTypeAny = z.ZodTypeAny, +> { + collect(options: { + entity: Entity; + input: z.infer; + }): Promise>; + getCollectorDescription(): string; + getCollectorId(): string; + getInputSchema(): TInputSchema; + getOutputSchema(): TOutputSchema; +} + +// @public +export type CollectorContract< + TInputSchema extends z.ZodTypeAny, + TOutputSchema extends z.ZodTypeAny, +> = { + inputSchema: TInputSchema; + outputSchema: TOutputSchema; +}; + +// @public +export interface CollectorRegistry { + // (undocumented) + getCollector(collectorId: string): Collector; + // (undocumented) + hasCollector(collectorId: string): boolean; +} + +// @public +export const collectWithContract: < + TInputSchema extends z.ZodTypeAny, + TOutputSchema extends z.ZodTypeAny, +>(options: { + collectorRegistry: CollectorRegistry; + collectorId: string; + contract: CollectorContract; + entity: Entity; + input: unknown; +}) => Promise>; // @public export type ComparisonOperator = { @@ -56,6 +101,19 @@ export type RangeOperator = { values: [number, number]; }; +// @public +export interface ScorecardCollectorsExtensionPoint { + // (undocumented) + addCollector(...collectors: Array): void; + // (undocumented) + getCollector(collectorId: string): Collector; + // (undocumented) + hasCollector(collectorId: string): boolean; +} + +// @public +export const scorecardCollectorsExtensionPoint: ExtensionPoint; + // @public export interface ScorecardMetricsExtensionPoint { // (undocumented)