diff --git a/docker-compose.yml b/docker-compose.yml index 4fb1faa2..0181dbd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,6 +69,20 @@ services: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9093 restart: on-failure + gcs-emulator: + image: fsouza/fake-gcs-server:1.52.3 + ports: + - '4443:4443' + command: -scheme http -port 4443 -external-url http://localhost:4443 + restart: on-failure + + pubsub-emulator: + image: google/cloud-sdk:547.0.0-emulators + ports: + - '8085:8085' + command: gcloud beta emulators pubsub start --project=test-project --host-port=0.0.0.0:8085 + restart: on-failure + volumes: rabbit_data: driver: local diff --git a/packages/gcp-pubsub/README.md b/packages/gcp-pubsub/README.md new file mode 100644 index 00000000..4779b33d --- /dev/null +++ b/packages/gcp-pubsub/README.md @@ -0,0 +1,1604 @@ +# @message-queue-toolkit/gcp-pubsub + +Google Cloud Pub/Sub implementation for the message-queue-toolkit. Provides a robust, type-safe abstraction for publishing and consuming messages from Google Cloud Pub/Sub topics and subscriptions. + +## Table of Contents + +- [Installation](#installation) +- [Features](#features) +- [Core Concepts](#core-concepts) +- [Quick Start](#quick-start) + - [Publisher](#publisher) + - [Consumer](#consumer) +- [Configuration](#configuration) + - [Resource Management](#resource-management) + - [Updating Existing Resources](#updating-existing-resources) + - [Resource Deletion](#resource-deletion) + - [Topic Creation](#topic-creation) + - [Subscription Configuration](#subscription-configuration) + - [Locator Config (Production)](#locator-config-production) + - [Publisher Options](#publisher-options) + - [Consumer Options](#consumer-options) +- [Advanced Features](#advanced-features) + - [Custom Message Field Names](#custom-message-field-names) + - [Payload Offloading](#payload-offloading) + - [Message Deduplication](#message-deduplication) + - [Dead Letter Queue](#dead-letter-queue) + - [Message Ordering](#message-ordering) + - [Message Retry Logic](#message-retry-logic) + - [Message Handlers](#message-handlers) + - [Pre-handlers and Barriers](#pre-handlers-and-barriers) + - [Handler Spies](#handler-spies) + - [Consumer Flow Control](#consumer-flow-control) + - [Multiple Message Types](#multiple-message-types) +- [Error Handling](#error-handling) +- [Testing](#testing) +- [API Reference](#api-reference) +- [Best Practices](#best-practices) +- [Troubleshooting](#troubleshooting) +- [Links](#links) + +## Installation + +```bash +npm install @message-queue-toolkit/gcp-pubsub @google-cloud/pubsub zod +``` + +**Peer Dependencies:** +- `@google-cloud/pubsub` - Google Cloud Pub/Sub client +- `zod` - Schema validation + +## Features + +- ✅ **Type-safe message handling** with Zod schema validation +- ✅ **Publisher** for publishing messages to topics +- ✅ **Consumer** for consuming messages from subscriptions +- ✅ **Automatic retry logic** with exponential backoff +- ✅ **Dead Letter Queue (DLQ)** support +- ✅ **Message deduplication** (publisher and consumer level) +- ✅ **Payload offloading** for large messages (>10MB, GCS integration) +- ✅ **Message ordering** with ordering keys +- ✅ **Exactly-once delivery** support +- ✅ **Handler spies** for testing +- ✅ **Pre-handlers and barriers** for complex message processing +- ✅ **Flow control** for throughput management +- ✅ **Automatic topic/subscription creation** with validation + +## Core Concepts + +### Google Pub/Sub Architecture + +Google Pub/Sub follows a strict topic/subscription model: + +``` +Publisher → Topic → Subscription → Consumer +``` + +**Key concepts:** +- **Topics**: Named resources to which messages are published +- **Subscriptions**: Named resources representing message streams from a topic +- ❌ You CANNOT publish directly to subscriptions +- ❌ You CANNOT consume directly from topics +- ✅ One topic can have multiple subscriptions (fan-out) +- ✅ One subscription per consumer (or consumer group) + +**Prerequisites:** +- Google Cloud Platform account +- Pub/Sub API enabled +- Appropriate IAM permissions + +### Publishers + +Publishers send messages to Pub/Sub topics. They handle: +- Message validation against Zod schemas +- Automatic serialization +- Optional deduplication (preventing duplicate sends) +- Optional payload offloading (for messages > 10MB) +- Message ordering (via ordering keys) + +### Consumers + +Consumers receive and process messages from Pub/Sub subscriptions. They handle: +- Message deserialization and validation +- Routing to appropriate handlers based on message type +- Automatic retry with exponential backoff +- Dead letter queue integration +- Optional deduplication (preventing duplicate processing) +- Message ordering guarantees +- Flow control for throughput management + +### Message Schemas + +Messages are validated using Zod schemas. Each message must have: +- A unique message type field (discriminator for routing) - configurable via `messageTypeField` (required) +- A message ID field (for tracking and deduplication) - configurable via `messageIdField` (default: `'id'`) +- A timestamp field (added automatically if missing) - configurable via `messageTimestampField` (default: `'timestamp'`) + +**Note:** All field names are configurable, allowing you to adapt the library to your existing message schemas without modification. + +## Quick Start + +### Publisher + +```typescript +import { PubSub } from '@google-cloud/pubsub' +import { AbstractPubSubPublisher } from '@message-queue-toolkit/gcp-pubsub' +import { z } from 'zod' + +const pubSubClient = new PubSub({ + projectId: 'my-project', + keyFilename: '/path/to/credentials.json', +}) + +// Define your message schema +const UserEventSchema = z.object({ + id: z.string(), + messageType: z.literal('user.created'), + timestamp: z.string().datetime(), + userId: z.string(), + email: z.string().email(), +}) + +type UserEvent = z.infer + +class UserEventPublisher extends AbstractPubSubPublisher { + constructor() { + super( + { + pubSubClient, + logger, + errorReporter, + }, + { + creationConfig: { + topic: { + name: 'user-events', + options: { + enableMessageOrdering: true, // Optional + }, + }, + }, + messageSchemas: [UserEventSchema], + messageTypeField: 'messageType', + logMessages: true, + } + ) + } +} + +// Usage +const publisher = new UserEventPublisher() +await publisher.init() + +await publisher.publish({ + id: '123', + messageType: 'user.created', + timestamp: new Date().toISOString(), + userId: 'user-456', + email: 'user@example.com', +}) +``` + +### Consumer + +```typescript +import { PubSub } from '@google-cloud/pubsub' +import { AbstractPubSubConsumer, MessageHandlerConfigBuilder } from '@message-queue-toolkit/gcp-pubsub' + +class UserEventConsumer extends AbstractPubSubConsumer { + constructor() { + super( + { + pubSubClient, + logger, + errorReporter, + consumerErrorResolver, + }, + { + creationConfig: { + topic: { + name: 'user-events', + }, + subscription: { + name: 'user-events-processor', + options: { + ackDeadlineSeconds: 60, + enableMessageOrdering: true, + }, + }, + }, + messageTypeField: 'messageType', + handlers: new MessageHandlerConfigBuilder() + .addConfig( + UserEventSchema, + async (message, context) => { + // Process the message + console.log('Processing user:', message.userId) + await saveToDatabase(message) + return { result: 'success' } + } + ) + .build(), + }, + {} // execution context + ) + } +} + +// Usage +const consumer = new UserEventConsumer() +await consumer.init() +await consumer.start() // Starts consuming messages +``` + +## Configuration + +### Resource Management + +#### Updating Existing Resources + +When using `creationConfig`, you can control whether existing resources should be updated with new configuration: + +```typescript +{ + creationConfig: { + topic: { + name: 'my-topic', + options: { + messageRetentionDuration: { seconds: 604800 }, + }, + }, + updateAttributesIfExists: true, // Update existing resources (default: false) + }, +} +``` + +**Behavior:** +- `updateAttributesIfExists: false` (default): If topic/subscription exists, uses it as-is without updates +- `updateAttributesIfExists: true`: If topic/subscription exists, updates its metadata with new options +- Applies to both topics and subscriptions +- Useful for managing configuration changes across environments + +#### Resource Deletion + +For testing and development, you can configure automatic resource deletion: + +```typescript +import { deletePubSub } from '@message-queue-toolkit/gcp-pubsub' + +{ + deletionConfig: { + deleteIfExists: true, // Enable automatic deletion + forceDeleteInProduction: false, // Safety: prevent production deletion (default: false) + waitForConfirmation: true, // Poll until deletion confirmed (default: true) + }, + creationConfig: { + topic: { name: 'test-topic' }, + subscription: { name: 'test-subscription' }, + }, +} +``` + +**Deletion Behavior:** +- Only deletes if both `deleteIfExists: true` and `creationConfig` are provided +- Deletes subscription first, then topic (proper order) +- Throws error if trying to delete in production without `forceDeleteInProduction: true` +- `waitForConfirmation: true`: Polls to confirm deletion completed (recommended) +- `waitForConfirmation: false`: Returns immediately after deletion request + +**Production Safety:** + +The library checks `process.env.NODE_ENV` to determine if running in production: +- `NODE_ENV === 'production'` → Production mode (deletion requires explicit override) +- Any other value → Development/test mode (deletion allowed) + +**Important:** The environment check uses a cached scope from `@lokalise/node-core`. If you change `NODE_ENV` at runtime (e.g., in tests), you must call `reloadConfig()`: + +```typescript +import { reloadConfig } from '@message-queue-toolkit/core' + +// In tests - changing NODE_ENV at runtime +process.env.NODE_ENV = 'production' +reloadConfig() // Required to pick up the change + +// In normal usage - NODE_ENV set before process starts +// No reloadConfig() needed, environment is read on initialization +``` + +**Examples:** + +```typescript +// Development - automatic cleanup +process.env.NODE_ENV = 'development' +{ + deletionConfig: { + deleteIfExists: true, // OK in development + }, +} + +// Production - requires explicit override +process.env.NODE_ENV = 'production' +{ + deletionConfig: { + deleteIfExists: true, + forceDeleteInProduction: true, // Required in production + }, +} +``` + +**Use Cases:** +- Integration tests: Clean up resources between test runs +- CI/CD pipelines: Ensure fresh environment for each build +- Development: Reset state without manual cleanup + +### Topic Creation + +When using `creationConfig`, the topic will be created automatically if it doesn't exist: + +```typescript +{ + creationConfig: { + topic: { + name: 'my-topic', + options: { + messageRetentionDuration: { + seconds: 604800, // 7 days + }, + messageStoragePolicy: { + allowedPersistenceRegions: ['us-central1'], + }, + enableMessageOrdering: true, + kmsKeyName: 'projects/my-project/locations/us/keyRings/my-ring/cryptoKeys/my-key', + }, + }, + }, +} +``` + +### Subscription Configuration + +For consumers, configure the subscription: + +```typescript +{ + creationConfig: { + topic: { + name: 'my-topic', + }, + subscription: { + name: 'my-subscription', + options: { + ackDeadlineSeconds: 60, + retainAckedMessages: false, + messageRetentionDuration: { + seconds: 604800, + }, + enableMessageOrdering: true, + enableExactlyOnceDelivery: true, + deadLetterPolicy: { + deadLetterTopic: 'projects/my-project/topics/my-dlq', + maxDeliveryAttempts: 5, + }, + filter: 'attributes.priority="high"', // Message filtering + }, + }, + }, +} +``` + +### Locator Config (Production) + +When using `locatorConfig`, you connect to existing resources without creating them: + +```typescript +{ + locatorConfig: { + topicName: 'existing-topic', + subscriptionName: 'existing-subscription', // For consumers + }, +} +``` + +### Publisher Options + +```typescript +{ + // Required - Message Schema Configuration + messageSchemas: [Schema1, Schema2], // Array of Zod schemas + messageTypeField: 'messageType', // Field containing message type discriminator + + // Topic Configuration (one of these required) + creationConfig: { + topic: { name: 'my-topic', options: { /* ... */ } }, + updateAttributesIfExists: false, // Update existing resources (default: false) + }, + locatorConfig: { /* ... */ }, // Use existing topic + + // Optional - Resource Deletion (testing/development) + deletionConfig: { + deleteIfExists: true, // Enable automatic deletion + forceDeleteInProduction: false, // Safety: prevent production deletion + waitForConfirmation: true, // Poll until deletion confirmed + }, + + // Optional - Message Field Configuration + messageIdField: 'id', // Field containing message ID (default: 'id') + messageTimestampField: 'timestamp', // Field containing timestamp (default: 'timestamp') + messageDeduplicationIdField: 'deduplicationId', // Field for deduplication ID (default: 'deduplicationId') + messageDeduplicationOptionsField: 'deduplicationOptions', // Field for deduplication options (default: 'deduplicationOptions') + + // Optional - Features + logMessages: false, // Log all published messages + handlerSpy: true, // Enable handler spy for testing + + // Optional - Deduplication + enablePublisherDeduplication: false, // Enable store-based deduplication + messageDeduplicationConfig: { + store: redisStore, // Redis-based deduplication store + deduplicationIdField: 'id', // Field to use for deduplication + }, + + // Optional - Payload Offloading + payloadStoreConfig: { + store: gcsStore, // GCS-based payload store + messageSizeThreshold: PUBSUB_MESSAGE_MAX_SIZE, // 10 MB + }, +} +``` + +### Consumer Options + +```typescript +{ + // Required - Message Handling Configuration + handlers: MessageHandlerConfigBuilder.build(), // Message handlers configuration + messageTypeField: 'messageType', // Field containing message type discriminator + + // Topic and Subscription Configuration (one of these required) + creationConfig: { + topic: { name: 'my-topic' }, + subscription: { name: 'my-subscription', options: { /* ... */ } }, + updateAttributesIfExists: false, // Update existing resources (default: false) + }, + locatorConfig: { /* ... */ }, + + // Optional - Resource Deletion (testing/development) + deletionConfig: { + deleteIfExists: true, // Enable automatic deletion + forceDeleteInProduction: false, // Safety: prevent production deletion + waitForConfirmation: true, // Poll until deletion confirmed + }, + + // Optional - Message Field Configuration + messageIdField: 'id', // Field containing message ID (default: 'id') + messageTimestampField: 'timestamp', // Field containing timestamp (default: 'timestamp') + messageDeduplicationIdField: 'deduplicationId', // Field for deduplication ID (default: 'deduplicationId') + messageDeduplicationOptionsField: 'deduplicationOptions', // Field for deduplication options (default: 'deduplicationOptions') + + // Optional - Retry Configuration + maxRetryDuration: 345600, // 4 days in seconds (default) + + // Optional - Dead Letter Queue + deadLetterQueue: { + deadLetterPolicy: { + maxDeliveryAttempts: 5, // Move to DLQ after 5 failed attempts (5-100) + }, + creationConfig: { + topic: { name: 'my-dlq-topic' }, // Create DLQ topic + }, + // OR + locatorConfig: { + topicName: 'existing-dlq-topic', // Use existing DLQ topic + }, + }, + + // Optional - Consumer Behavior + consumerOverrides: { + flowControl: { + maxMessages: 100, // Max concurrent messages + maxBytes: 10 * 1024 * 1024, // Max bytes in memory + }, + batching: { + maxMessages: 10, // Pull messages in batches + maxMilliseconds: 100, // Max wait time for batch + }, + }, + + // Optional - Deduplication + enableConsumerDeduplication: false, + messageDeduplicationConfig: { + store: redisStore, + deduplicationIdField: 'id', + deduplicationLockTimeout: 20000, // Lock timeout in milliseconds + }, + + // Optional - Payload Offloading + payloadStoreConfig: { + store: gcsStore, + }, + + // Optional - Other + logMessages: false, + handlerSpy: true, +} +``` + +## Advanced Features + +### Custom Message Field Names + +All message field names are configurable, allowing you to adapt the library to your existing message schemas: + +```typescript +// Your existing message schema with custom field names +const CustomMessageSchema = z.object({ + messageId: z.string(), // Custom ID field + eventType: z.literal('order.created'), // Custom type field + createdAt: z.string().datetime(), // Custom timestamp field + txId: z.string(), // Custom deduplication ID + txOptions: z.object({ // Custom deduplication options + deduplicationWindowSeconds: z.number().optional(), + }).optional(), + orderId: z.string(), + amount: z.number(), +}) + +// Configure the publisher to use your custom field names +class OrderPublisher extends AbstractPubSubPublisher { + constructor() { + super( + { pubSubClient, logger, errorReporter }, + { + messageSchemas: [CustomMessageSchema], + + // Map library's internal fields to your custom fields + messageIdField: 'messageId', // Default: 'id' + messageTypeField: 'eventType', // Required + messageTimestampField: 'createdAt', // Default: 'timestamp' + messageDeduplicationIdField: 'txId', // Default: 'deduplicationId' + messageDeduplicationOptionsField: 'txOptions', // Default: 'deduplicationOptions' + + creationConfig: { + topic: { name: 'orders-topic' }, + }, + } + ) + } +} + +// Use with your custom schema +await publisher.publish({ + messageId: 'msg-123', // Library will use this for tracking + eventType: 'order.created', // Library will use this for routing + createdAt: new Date().toISOString(), // Library will use this for retry tracking + txId: 'tx-456', // Library will use this for deduplication + orderId: 'order-789', + amount: 99.99, +}) +``` + +**Benefits:** +- ✅ No need to modify existing message schemas +- ✅ Maintain consistency with your domain model +- ✅ Gradual migration from legacy systems +- ✅ Works with all features (retry, deduplication, offloading) + +### Payload Offloading + +For messages larger than 10 MB, store the payload externally (e.g., Google Cloud Storage): + +```typescript +import { Storage } from '@google-cloud/storage' +import { GCSPayloadStore } from '@message-queue-toolkit/gcs-payload-store' +import { PUBSUB_MESSAGE_MAX_SIZE } from '@message-queue-toolkit/gcp-pubsub' + +const storage = new Storage({ projectId: 'my-project' }) +const payloadStore = new GCSPayloadStore( + { gcsStorage: storage }, + { bucketName: 'my-payload-bucket' } +) + +// Publisher configuration +class LargeMessagePublisher extends AbstractPubSubPublisher { + constructor() { + super(dependencies, { + creationConfig: { + topic: { name: 'large-messages' }, + }, + messageSchemas: [MyMessageSchema], + messageTypeField: 'type', + payloadStoreConfig: { + store: payloadStore, + messageSizeThreshold: PUBSUB_MESSAGE_MAX_SIZE, // 10 MB + }, + }) + } +} + +// Large message is automatically offloaded +await publisher.publish({ + id: '123', + messageType: 'document.processed', + largeData: hugeArrayOfData, // If total size > 10 MB, stored in GCS +}) +``` + +**How it works:** +1. Publisher checks message size before sending +2. If size exceeds `messageSizeThreshold`, stores payload in GCS +3. Replaces payload with pointer: `{ _offloadedPayload: { bucketName, key, size } }` +4. Sends pointer message to Pub/Sub +5. Consumer detects pointer, fetches payload from GCS +6. Processes message with full payload + +**Note:** Consumer automatically retrieves offloaded payloads - no special configuration needed! Payload cleanup is the responsibility of the store (e.g., GCS lifecycle policies). + +### Message Deduplication + +Prevent duplicate message publishing or processing: + +#### Publisher-Level Deduplication + +Prevents sending the same message multiple times: + +```typescript +import { InMemoryDeduplicationStore } from '@message-queue-toolkit/core' +// or +import { RedisMessageDeduplicationStore } from '@message-queue-toolkit/redis-message-deduplication-store' + +const deduplicationStore = new RedisMessageDeduplicationStore(redisClient) + +// Publisher configuration +{ + enablePublisherDeduplication: true, + messageDeduplicationIdField: 'deduplicationId', + messageDeduplicationConfig: { + store: deduplicationStore, + }, +} + +// Publishing with deduplication +await publisher.publish({ + id: '123', + messageType: 'user.created', + deduplicationId: 'user-456-creation', // Unique key for deduplication + deduplicationOptions: { + deduplicationWindowSeconds: 60, // Prevent duplicates for 60 seconds + }, + userId: 'user-456', +}) + +// Second publish with same deduplicationId within 60s is skipped +await publisher.publish({ + id: '124', + messageType: 'user.created', + deduplicationId: 'user-456-creation', // Duplicate - won't be sent + userId: 'user-456', +}) +``` + +#### Consumer-Level Deduplication + +Prevents processing the same message multiple times: + +```typescript +{ + enableConsumerDeduplication: true, + messageDeduplicationIdField: 'deduplicationId', + messageDeduplicationConfig: { + store: deduplicationStore, + }, +} + +// Message configuration +{ + deduplicationId: 'unique-operation-id', + deduplicationOptions: { + deduplicationWindowSeconds: 3600, // 1 hour + lockTimeoutSeconds: 20, // Lock duration while processing + acquireTimeoutSeconds: 20, // Max wait time to acquire lock + refreshIntervalSeconds: 10, // Lock refresh interval + }, +} +``` + +**How it works:** +1. Consumer receives message +2. Checks deduplication store for duplicate +3. If duplicate found (within window), skips processing +4. If not duplicate, acquires exclusive lock +5. Processes message +6. Releases lock and marks as processed +7. Subsequent messages with same ID are skipped + +### Dead Letter Queue + +Dead Letter Queues capture messages that cannot be processed after multiple attempts. The library supports configuring DLQ in two ways: + +#### Method 1: Create DLQ Topic Automatically + +```typescript +{ + creationConfig: { + topic: { name: 'my-topic' }, + subscription: { name: 'my-subscription' }, + }, + deadLetterQueue: { + deadLetterPolicy: { + maxDeliveryAttempts: 5, // Send to DLQ after 5 failed attempts (5-100) + }, + creationConfig: { + topic: { name: 'my-dlq-topic' }, // Creates topic if it doesn't exist + }, + }, +} +``` + +#### Method 2: Use Existing DLQ Topic + +```typescript +{ + creationConfig: { + topic: { name: 'my-topic' }, + subscription: { name: 'my-subscription' }, + }, + deadLetterQueue: { + deadLetterPolicy: { + maxDeliveryAttempts: 5, + }, + locatorConfig: { + topicName: 'existing-dlq-topic', // Must exist, or init() will throw + }, + }, +} +``` + +**How it works:** +1. Message fails processing (handler returns error or throws) +2. Message becomes available again (after ack deadline) +3. Consumer receives message again (delivery attempt increments) +4. Pub/Sub tracks delivery attempts = 1 + (NACKs + ack deadline exceeded) +5. After `maxDeliveryAttempts` attempts, Pub/Sub automatically forwards message to DLQ topic +6. DLQ messages can be inspected, reprocessed, or deleted + +**Important Notes:** +- `maxDeliveryAttempts` must be between 5 and 100 +- DLQ is handled natively by Google Pub/Sub (no manual forwarding needed) +- When message is forwarded to DLQ, it's wrapped with metadata attributes: + - `CloudPubSubDeadLetterSourceDeliveryCount`: Number of delivery attempts + - `CloudPubSubDeadLetterSourceSubscription`: Source subscription name + - `CloudPubSubDeadLetterSourceSubscriptionProject`: Source project + - `CloudPubSubDeadLetterSourceTopicPublishTime`: Original publish timestamp +- Create a subscription on the DLQ topic to process dead-lettered messages +- Ensure Pub/Sub service account has permissions on the DLQ topic + +### Message Retry Logic + +The library implements intelligent retry logic with exponential backoff: + +```typescript +{ + maxRetryDuration: 345600, // 4 days in seconds (default) +} +``` + +**Retry Flow:** + +1. **Handler returns `{ error: 'retryLater' }`** or **throws an error** +2. Consumer checks if message should be retried: + - Calculates how long the message has been retrying + - If within `maxRetryDuration`, re-queues message (nacks it) + - If exceeded, sends to DLQ (if configured) or marks as failed + +3. **Exponential Backoff:** + ``` + Attempt 1: Message nacked, redelivered by Pub/Sub + Attempt 2: Message nacked, redelivered by Pub/Sub + Attempt 3: Message nacked, redelivered by Pub/Sub + ... + After maxDeliveryAttempts: Sent to DLQ + ``` + +**Handler Return Types:** + +```typescript +type HandlerResult = Either<'retryLater', 'success'> + +// Success - message is acknowledged +return { result: 'success' } + +// Retry - message is nacked, will be retried +return { error: 'retryLater' } + +// Error thrown - automatically retries +throw new Error('Database connection failed') +``` + +### Message Ordering + +Enable ordered delivery of messages with the same ordering key: + +```typescript +// Publisher configuration +{ + creationConfig: { + topic: { + name: 'ordered-events', + options: { + enableMessageOrdering: true, + }, + }, + }, +} + +// Publish with ordering key +await publisher.publish(message, { + orderingKey: 'user-123', // All messages with this key are delivered in order +}) + +// Consumer configuration +{ + creationConfig: { + subscription: { + options: { + enableMessageOrdering: true, + }, + }, + }, +} +``` + +**Ordering guarantees:** +- ✅ Messages with the same ordering key are delivered in order +- ✅ Messages are processed exactly once (when combined with exactly-once delivery) +- ❌ No ordering guarantee across different ordering keys + +### Message Handlers + +Handlers process messages based on their type. Messages are routed to the appropriate handler using the discriminator field (configurable via `messageTypeField`): + +```typescript +import { MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' + +const handlers = new MessageHandlerConfigBuilder< + SupportedMessages, + ExecutionContext, + PrehandlerOutput +>() + .addConfig( + UserCreatedSchema, + async (message, context, preHandlingOutputs) => { + // Access execution context + await context.userService.createUser(message.userId) + + // Access pre-handler outputs + console.log('Pre-handler result:', preHandlingOutputs.preHandlerOutput) + console.log('Barrier result:', preHandlingOutputs.barrierOutput) + + return { result: 'success' } + }, + { + // Optional: Pre-handlers (run before main handler) + preHandlers: [ + (message, context, output, next) => { + console.log('Pre-processing message:', message.id) + output.processedAt = Date.now() + next({ result: 'success' }) + }, + ], + + // Optional: Barrier (controls whether message should be processed) + preHandlerBarrier: async (message, context, preHandlerOutput) => { + const isReady = await context.userService.isSystemReady() + return { + isPassing: isReady, + output: { systemStatus: 'ready' }, + } + }, + + // Optional: Custom message log formatter + messageLogFormatter: (message) => ({ + userId: message.userId, + action: 'create', + }), + } + ) + .addConfig(UserUpdatedSchema, handleUserUpdated) + .build() +``` + +### Pre-handlers and Barriers + +#### Pre-handlers + +Pre-handlers are middleware functions that run before the main message handler, allowing you to: +- Enrich the execution context with additional data +- Set up scoped resources (child loggers, database transactions) +- Validate prerequisites +- Transform message data +- Implement cross-cutting concerns (logging, metrics, caching) + +The output from pre-handlers is passed to both the barrier and the main handler, enabling a powerful data flow pattern. + +**Type Signature:** + +```typescript +type Prehandler = ( + message: Message, + context: Context, + output: Output, + next: (result: PrehandlerResult) => void +) => void +``` + +**Common Use Cases:** + +##### 1. Child Logger Resolution + +Create message-specific loggers with contextual information: + +```typescript +type PrehandlerOutput = { + logger: Logger +} + +const preHandlers: Prehandler[] = [ + (message, context, output, next) => { + // Create child logger with message context + output.logger = context.logger.child({ + messageId: message.id, + messageType: message.messageType, + userId: message.userId, + correlationId: message.correlationId, + }) + + output.logger.info('Message processing started') + next({ result: 'success' }) + }, +] + +// In your handler +const handler = async (message, context, preHandlingOutputs) => { + const logger = preHandlingOutputs.preHandlerOutput.logger + + logger.info('Processing user update') // Automatically includes message context + logger.error({ error: someError }, 'Failed to update user') + + return { result: 'success' } +} +``` + +##### 2. User Data and Permissions Resolution + +Fetch and cache user information needed by the handler: + +```typescript +type PrehandlerOutput = { + user: User + permissions: string[] + organizationId: string +} + +const preHandlers: Prehandler[] = [ + // Fetch user data + async (message, context, output, next) => { + try { + const user = await context.userRepository.findById(message.userId) + if (!user) { + next({ error: new Error(`User ${message.userId} not found`) }) + return + } + output.user = user + next({ result: 'success' }) + } catch (error) { + next({ error }) + } + }, + + // Resolve permissions + async (message, context, output, next) => { + try { + output.permissions = await context.permissionService.getPermissions(output.user.id) + output.organizationId = output.user.organizationId + next({ result: 'success' }) + } catch (error) { + next({ error }) + } + }, +] + +// In your handler - user data is already fetched +const handler = async (message, context, preHandlingOutputs) => { + const { user, permissions, organizationId } = preHandlingOutputs.preHandlerOutput + + // Check permissions + if (!permissions.includes('orders:create')) { + throw new Error('Insufficient permissions') + } + + // Use pre-fetched data + await context.orderService.createOrder({ + orderId: message.orderId, + userId: user.id, + organizationId, + userEmail: user.email, // Already available, no need to fetch again + }) + + return { result: 'success' } +} +``` + +#### Barriers + +Barriers are async functions that determine whether a message should be processed immediately or retried later. They are essential for handling message dependencies and ensuring prerequisites are met. + +**Type Signature:** + +```typescript +type BarrierCallback = ( + message: Message, + context: Context, + preHandlerOutput: PrehandlerOutput +) => Promise> + +type BarrierResult = { + isPassing: boolean // true = process now, false = retry later + output: Output // Additional data passed to the handler +} +``` + +**Common Use Cases:** + +##### 1. Message Ordering Dependencies + +Ensure messages are processed in the correct order when they arrive out of sequence: + +```typescript +// Scenario: Process order.updated only after order.created +const preHandlerBarrier = async (message: OrderUpdatedMessage, context, preHandlerOutput) => { + // Check if the order exists (created event was processed) + const orderExists = await context.orderRepository.exists(message.orderId) + + if (!orderExists) { + context.logger.warn('Order not found, retrying later', { + orderId: message.orderId, + messageId: message.id, + }) + + return { + isPassing: false, + output: { reason: 'order_not_created_yet' }, + } + } + + return { + isPassing: true, + output: { orderExists: true }, + } +} + +// Message will be automatically retried until order.created is processed +``` + +##### 2. Business Workflow Prerequisites + +Implement complex business logic gates: + +```typescript +// Scenario: Process payment only after KYC verification is complete +const preHandlerBarrier = async ( + message: PaymentMessage, + context, + preHandlerOutput +) => { + const { user } = preHandlerOutput // From pre-handler + + // Check KYC status + const kycStatus = await context.kycService.getStatus(user.id) + + if (kycStatus !== 'approved') { + context.logger.info('KYC not approved, retrying later', { + userId: user.id, + kycStatus, + }) + + return { + isPassing: false, + output: { + reason: 'kyc_pending', + kycStatus, + retriedAt: new Date(), + }, + } + } + + // Check account balance + const balance = await context.accountService.getBalance(user.id) + if (balance < message.amount) { + context.logger.info('Insufficient balance, retrying later', { + userId: user.id, + balance, + required: message.amount, + }) + + return { + isPassing: false, + output: { + reason: 'insufficient_balance', + balance, + required: message.amount, + }, + } + } + + return { + isPassing: true, + output: { + kycApproved: true, + currentBalance: balance, + }, + } +} + +const handler = async (message, context, preHandlingOutputs) => { + const { kycApproved, currentBalance } = preHandlingOutputs.barrierOutput + + // Safe to process payment - all prerequisites met + await context.paymentService.processPayment({ + userId: message.userId, + amount: message.amount, + currentBalance, // From barrier + }) + + return { result: 'success' } +} +``` + +**Configuration:** + +```typescript +new MessageHandlerConfigBuilder() + .addConfig( + MessageSchema, + handler, + { + preHandlers: [userDataPreHandler, permissionsPreHandler], + preHandlerBarrier: orderDependencyBarrier, + } + ) + .build() +``` + +**Important Notes:** + +- **Barriers return `isPassing: false`** → Message is automatically retried (nacked) +- **Barriers throw errors** → Message follows normal error handling (retry or DLQ) +- **Barrier output** → Available in handler via `preHandlingOutputs.barrierOutput` +- **Retry limits apply** → Messages exceeding `maxRetryDuration` will be sent to DLQ even if barrier keeps returning false + +### Handler Spies + +Handler spies solve the fundamental challenge of testing asynchronous message-based systems. + +**The Problem:** + +Testing message queues is complex because: +1. **Asynchronous processing** - Messages are published and consumed asynchronously with unpredictable timing +2. **Indirect interactions** - Business logic may trigger message publishing without explicit calls to the publisher +3. **Non-deterministic order** - Messages may be processed in different orders across test runs +4. **Hard to verify** - Traditional mocking/stubbing doesn't work well for async pub/sub patterns + +**The Solution:** + +Handler spies provide a way to wait for and inspect messages during tests without having to: +- Poll the topic/subscription directly +- Add artificial delays (`setTimeout`) +- Mock the entire message infrastructure +- Modify production code for testing + +#### Configuration + +```typescript +// Enable handler spy for publisher and/or consumer +const publisher = new UserEventsPublisher(pubSubClient, { + handlerSpy: true, // Track published messages +}) + +const consumer = new UserEventsConsumer(pubSubClient, { + handlerSpy: true, // Track consumed messages +}) +``` + +#### Example: Testing Message Publishing and Consumption + +```typescript +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +describe('User Events Flow', () => { + let publisher: UserEventsPublisher + let consumer: UserEventsConsumer + + beforeEach(async () => { + publisher = new UserEventsPublisher(pubSubClient, { handlerSpy: true }) + consumer = new UserEventsConsumer(pubSubClient, { handlerSpy: true }) + + await publisher.init() + await consumer.start() + }) + + afterEach(async () => { + await consumer.close() + await publisher.close() + }) + + it('processes user.created event', async () => { + // Act: Publish message + await publisher.publish({ + id: 'msg-123', + messageType: 'user.created', + userId: 'user-456', + email: 'test@example.com', + }) + + // Assert: Wait for message to be tracked by publisher spy + const publishedMessage = await publisher.handlerSpy.waitForMessageWithId( + 'msg-123', + 'published', + 5000 // 5 second timeout + ) + + expect(publishedMessage).toMatchObject({ + id: 'msg-123', + userId: 'user-456', + email: 'test@example.com', + }) + + // Assert: Wait for message to be consumed + const consumedMessage = await consumer.handlerSpy.waitForMessageWithId( + 'msg-123', + 'consumed', + 10000 // 10 second timeout + ) + + expect(consumedMessage.userId).toBe('user-456') + }) + + it('checks message without waiting', async () => { + await publisher.publish({ + id: 'msg-789', + messageType: 'user.deleted', + userId: 'user-123', + }) + + // Wait briefly for async processing + await new Promise(resolve => setTimeout(resolve, 100)) + + // Check without waiting + const result = consumer.handlerSpy.checkMessage( + (msg) => msg.id === 'msg-789' + ) + + if (result) { + expect(result.message.userId).toBe('user-123') + expect(result.processingResult.status).toBe('consumed') + } else { + throw new Error('Message not found') + } + }) +}) +``` + +#### Handler Spy API Reference + +```typescript +interface HandlerSpy { + // Wait for message by ID (with timeout) + waitForMessageWithId( + messageId: string, + state: 'consumed' | 'published' | 'retryLater', + timeout?: number // Default: 15000ms + ): Promise + + // Wait for message matching predicate (with timeout) + waitForMessage( + predicate: (message: Message) => boolean, + state: 'consumed' | 'published' | 'retryLater', + timeout?: number // Default: 15000ms + ): Promise + + // Check if message exists without waiting + checkMessage( + predicate: (message: Message) => boolean + ): { message: Message; processingResult: ProcessingResult } | undefined + + // Get all tracked messages (circular buffer, limited size) + getAllMessages(): Array<{ message: Message; processingResult: ProcessingResult }> +} +``` + +**Best Practices:** + +1. **Always set timeouts** - Tests can hang indefinitely if messages don't arrive +2. **Use specific predicates** - Avoid overly broad matchers that could match wrong messages +3. **Clean up between tests** - Reset handler spies or recreate publishers/consumers +4. **Use in integration tests** - Handler spies are most valuable for integration tests, not unit tests +5. **Don't use in production** - Handler spies add memory overhead (circular buffer of messages) + +### Consumer Flow Control + +Control message throughput: + +```typescript +{ + consumerOverrides: { + flowControl: { + maxMessages: 100, // Max concurrent messages + maxBytes: 10 * 1024 * 1024, // Max bytes in memory + }, + batching: { + maxMessages: 10, // Pull messages in batches + maxMilliseconds: 100, // Max wait time for batch + }, + }, +} +``` + +### Multiple Message Types + +Handle different message types in one consumer: + +```typescript +const UserCreatedSchema = z.object({ + messageType: z.literal('user.created'), + userId: z.string(), +}) + +const UserDeletedSchema = z.object({ + messageType: z.literal('user.deleted'), + userId: z.string(), +}) + +type UserEvent = z.infer | z.infer + +handlers: new MessageHandlerConfigBuilder() + .addConfig(UserCreatedSchema, async (message) => { + console.log('User created:', message.userId) + return { result: 'success' } + }) + .addConfig(UserDeletedSchema, async (message) => { + console.log('User deleted:', message.userId) + return { result: 'success' } + }) + .build() +``` + +## Error Handling + +### Handler Returns + +```typescript +type HandlerResult = Either<'retryLater', 'success'> + +async (message) => { + try { + await processMessage(message) + return { result: 'success' } // Message ACKed + } catch (error) { + if (isRetryable(error)) { + return { error: 'retryLater' } // Message NACKed, will be retried + } + throw error // Message NACKed, will be retried + } +} +``` + +### Error Resolver + +```typescript +import { PubSubConsumerErrorResolver } from '@message-queue-toolkit/gcp-pubsub' + +const consumerErrorResolver = new PubSubConsumerErrorResolver() + +// Or custom implementation +class CustomErrorResolver implements ErrorResolver { + processError(error: Error): void { + // Send to Sentry, log, etc. + console.error('Consumer error:', error) + } +} +``` + +## Testing + +The library is designed to be testable: + +### Integration Tests with Emulator + +```bash +# Start emulator (included in docker-compose) +docker compose up -d pubsub-emulator +``` + +```typescript +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { PubSub } from '@google-cloud/pubsub' + +describe('UserEventsConsumer', () => { + let pubSubClient: PubSub + let publisher: UserEventsPublisher + let consumer: UserEventsConsumer + + beforeEach(async () => { + pubSubClient = new PubSub({ + projectId: 'test-project', + apiEndpoint: 'localhost:8085', // Emulator + }) + + publisher = new UserEventsPublisher(pubSubClient) + consumer = new UserEventsConsumer(pubSubClient, userService) + + await publisher.init() + await consumer.start() + }) + + afterEach(async () => { + await consumer.close() + await publisher.close() + }) + + it('processes user.created message', async () => { + await publisher.publish({ + id: '123', + messageType: 'user.created', + userId: 'user-456', + email: 'test@example.com', + }) + + // Wait for message to be processed + await consumer.handlerSpy.waitForMessageWithId('123', 'consumed') + + // Verify side effects + expect(userService.createUser).toHaveBeenCalledWith('user-456', 'test@example.com') + }) + + it('retries failed messages', async () => { + let attempts = 0 + userService.createUser.mockImplementation(() => { + attempts++ + if (attempts < 3) throw new Error('Temporary failure') + return Promise.resolve() + }) + + await publisher.publish({ + id: '124', + messageType: 'user.created', + userId: 'user-789', + email: 'test2@example.com', + }) + + await consumer.handlerSpy.waitForMessageWithId('124', 'consumed') + + expect(attempts).toBe(3) + }) +}) +``` + +### Unit Tests with Handler Spies + +```typescript +it('publishes message', async () => { + await publisher.publish({ + id: '123', + messageType: 'user.created', + userId: 'user-456', + email: 'test@example.com', + }) + + const publishedMessage = await publisher.handlerSpy.waitForMessageWithId('123', 'published') + + expect(publishedMessage).toMatchObject({ + id: '123', + userId: 'user-456', + }) +}) +``` + +## API Reference + +### AbstractPubSubPublisher + +**Constructor Options:** +- `messageSchemas`: Array of Zod schemas for messages +- `messageTypeField`: Field name containing message type +- `creationConfig` / `locatorConfig`: Topic configuration +- `logMessages`: Enable message logging +- `payloadStoreConfig`: Payload offloading configuration +- `enablePublisherDeduplication`: Enable deduplication +- `messageDeduplicationConfig`: Deduplication store config + +**Methods:** +- `init()`: Initialize publisher (create/locate topic) +- `publish(message, options?)`: Publish a message +- `close()`: Close publisher +- `handlerSpy`: Access spy for testing + +**Publish Options:** +- `orderingKey`: String for message ordering +- `attributes`: Custom message attributes + +### AbstractPubSubConsumer + +**Constructor Options:** +- `handlers`: Message handler configuration +- `messageTypeField`: Field name containing message type +- `creationConfig` / `locatorConfig`: Topic + subscription configuration +- `logMessages`: Enable message logging +- `payloadStoreConfig`: Payload retrieval configuration +- `enableConsumerDeduplication`: Enable deduplication +- `messageDeduplicationConfig`: Deduplication store config +- `deadLetterQueue`: DLQ configuration +- `maxRetryDuration`: Max retry time in seconds +- `consumerOverrides`: Flow control settings + +**Methods:** +- `init()`: Initialize consumer (create/locate resources) +- `start()`: Start consuming messages +- `close()`: Stop consumer and close connections +- `handlerSpy`: Access spy for testing + +## Best Practices + +1. **Use message ordering** for related events (same user, same entity) +2. **Enable exactly-once delivery** for critical workflows +3. **Set appropriate ACK deadlines** (60s is a good default) +4. **Implement idempotent handlers** (at-least-once delivery) +5. **Use deduplication** for critical operations +6. **Configure DLQ** for poison message handling +7. **Monitor subscription backlog** in GCP console +8. **Use payload offloading** for large messages +9. **Test with emulator** before deploying +10. **Set appropriate flow control** limits based on your processing capacity + +## Troubleshooting + +### Messages not being consumed + +- Check subscription exists and is attached to the topic +- Verify ACK deadline is sufficient for processing +- Check flow control limits aren't too restrictive +- Ensure consumer is started (`await consumer.start()`) + +### Messages going to DLQ + +- Check `maxDeliveryAttempts` configuration +- Review handler error logs +- Verify message format matches schema +- Check retry duration hasn't been exceeded + +### Memory issues + +- Reduce `flowControl.maxMessages` +- Reduce `flowControl.maxBytes` +- Enable payload offloading for large messages + +### Emulator issues + +- Ensure emulator is running on port 8085 +- Set `PUBSUB_EMULATOR_HOST=localhost:8085` environment variable +- Or configure `apiEndpoint: 'localhost:8085'` in PubSub client + +## License + +MIT + +## Contributing + +Contributions are welcome! Please see the main repository for guidelines. + +## Links + +- [Main Repository](https://github.com/kibertoad/message-queue-toolkit) +- [Core Package](https://www.npmjs.com/package/@message-queue-toolkit/core) +- [GCS Payload Store](https://www.npmjs.com/package/@message-queue-toolkit/gcs-payload-store) +- [Redis Deduplication Store](https://www.npmjs.com/package/@message-queue-toolkit/redis-message-deduplication-store) +- [Google Cloud Pub/Sub Documentation](https://cloud.google.com/pubsub/docs) diff --git a/packages/gcp-pubsub/lib/errors/PubSubConsumerErrorResolver.ts b/packages/gcp-pubsub/lib/errors/PubSubConsumerErrorResolver.ts new file mode 100644 index 00000000..88d84154 --- /dev/null +++ b/packages/gcp-pubsub/lib/errors/PubSubConsumerErrorResolver.ts @@ -0,0 +1,32 @@ +import { types } from 'node:util' +import { type ErrorResolver, InternalError, isStandardizedError } from '@lokalise/node-core' +import { MessageInvalidFormatError, MessageValidationError } from '@message-queue-toolkit/core' +import { ZodError } from 'zod/v4' + +export class PubSubConsumerErrorResolver implements ErrorResolver { + public processError(error: unknown): InternalError { + if (types.isNativeError(error) && error?.name === 'SyntaxError') { + return new MessageInvalidFormatError({ + message: error.message, + }) + } + if (error instanceof ZodError) { + return new MessageValidationError({ + message: error.message, + details: { + error: error.issues, + }, + }) + } + if (isStandardizedError(error)) { + return new InternalError({ + message: error.message, + errorCode: error.code, + }) + } + return new InternalError({ + message: 'Error processing message', + errorCode: 'INTERNAL_ERROR', + }) + } +} diff --git a/packages/gcp-pubsub/lib/fakes/FakeConsumerErrorResolver.ts b/packages/gcp-pubsub/lib/fakes/FakeConsumerErrorResolver.ts new file mode 100644 index 00000000..1f5964b1 --- /dev/null +++ b/packages/gcp-pubsub/lib/fakes/FakeConsumerErrorResolver.ts @@ -0,0 +1,23 @@ +import { PubSubConsumerErrorResolver } from '../errors/PubSubConsumerErrorResolver.ts' + +export class FakeConsumerErrorResolver extends PubSubConsumerErrorResolver { + private _errors: unknown[] + + constructor() { + super() + this._errors = [] + } + + public override processError(error: unknown) { + this._errors.push(error) + return super.processError(error) + } + + get errors() { + return this._errors + } + + public clear(): void { + this._errors = [] + } +} diff --git a/packages/gcp-pubsub/lib/index.ts b/packages/gcp-pubsub/lib/index.ts new file mode 100644 index 00000000..4c34616b --- /dev/null +++ b/packages/gcp-pubsub/lib/index.ts @@ -0,0 +1,14 @@ +export * from './errors/PubSubConsumerErrorResolver.ts' +export * from './fakes/FakeConsumerErrorResolver.ts' +export * from './pubsub/AbstractPubSubConsumer.ts' +export * from './pubsub/AbstractPubSubPublisher.ts' +export * from './pubsub/AbstractPubSubService.ts' +export * from './pubsub/CommonPubSubPublisherFactory.ts' +export * from './pubsub/PubSubPublisherManager.ts' +export * from './schemas/pubSubSchemas.ts' +export * from './types/MessageTypes.ts' +export * from './utils/messageUtils.ts' +export * from './utils/pubSubInitter.ts' +export * from './utils/pubSubMessageDeserializer.ts' +export * from './utils/pubSubMessageReader.ts' +export * from './utils/pubSubUtils.ts' diff --git a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubConsumer.ts b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubConsumer.ts new file mode 100644 index 00000000..a782d648 --- /dev/null +++ b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubConsumer.ts @@ -0,0 +1,494 @@ +import type { Either, ErrorResolver } from '@lokalise/node-core' +import { + type BarrierResult, + DeduplicationRequesterEnum, + HandlerContainer, + type MessageSchemaContainer, + noopReleasableLock, + type PreHandlingOutputs, + type Prehandler, + parseMessage, + type QueueConsumer, + type QueueConsumerDependencies, + type QueueConsumerOptions, + type TransactionObservabilityManager, +} from '@message-queue-toolkit/core' + +import type { PubSubMessage } from '../types/MessageTypes.ts' +import { hasOffloadedPayload } from '../utils/messageUtils.ts' +import { deserializePubSubMessage } from '../utils/pubSubMessageDeserializer.ts' +import type { + PubSubCreationConfig, + PubSubDependencies, + PubSubQueueLocatorType, +} from './AbstractPubSubService.ts' +import { AbstractPubSubService } from './AbstractPubSubService.ts' + +const _ABORT_EARLY_EITHER: Either<'abort', never> = { + error: 'abort', +} +const DEFAULT_MAX_RETRY_DURATION = 4 * 24 * 60 * 60 // 4 days in seconds + +export type PubSubDeadLetterQueueOptions = { + deadLetterPolicy: { + maxDeliveryAttempts: number + } + creationConfig?: { + topic: { + name: string + } + } + locatorConfig?: { + topicName: string + } +} + +export type PubSubConsumerDependencies = PubSubDependencies & QueueConsumerDependencies + +export type PubSubConsumerOptions< + MessagePayloadSchemas extends object, + ExecutionContext, + PrehandlerOutput, + CreationConfigType extends PubSubCreationConfig = PubSubCreationConfig, + QueueLocatorType extends PubSubQueueLocatorType = PubSubQueueLocatorType, +> = QueueConsumerOptions< + CreationConfigType, + QueueLocatorType, + PubSubDeadLetterQueueOptions, + MessagePayloadSchemas, + ExecutionContext, + PrehandlerOutput, + PubSubCreationConfig, + PubSubQueueLocatorType +> & { + consumerOverrides?: { + flowControl?: { + maxMessages?: number + maxBytes?: number + } + batching?: { + maxMessages?: number + maxMilliseconds?: number + } + } +} + +export abstract class AbstractPubSubConsumer< + MessagePayloadType extends object, + ExecutionContext, + PrehandlerOutput = undefined, + CreationConfigType extends PubSubCreationConfig = PubSubCreationConfig, + QueueLocatorType extends PubSubQueueLocatorType = PubSubQueueLocatorType, + ConsumerOptionsType extends PubSubConsumerOptions< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput, + CreationConfigType, + QueueLocatorType + > = PubSubConsumerOptions< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput, + CreationConfigType, + QueueLocatorType + >, + > + extends AbstractPubSubService< + MessagePayloadType, + QueueLocatorType, + CreationConfigType, + ConsumerOptionsType, + PubSubConsumerDependencies, + ExecutionContext, + PrehandlerOutput + > + implements QueueConsumer +{ + private readonly transactionObservabilityManager?: TransactionObservabilityManager + private readonly consumerOverrides: Partial + private readonly handlerContainer: HandlerContainer< + MessagePayloadType, + ExecutionContext, + PrehandlerOutput + > + private readonly deadLetterQueueOptions?: PubSubDeadLetterQueueOptions + private readonly isDeduplicationEnabled: boolean + private maxRetryDuration: number + private isConsuming = false + + protected readonly errorResolver: ErrorResolver + protected readonly executionContext: ExecutionContext + + public dlqTopicName?: string + public readonly _messageSchemaContainer: MessageSchemaContainer + + protected constructor( + dependencies: PubSubConsumerDependencies, + options: ConsumerOptionsType, + executionContext: ExecutionContext, + ) { + super(dependencies, options) + this.transactionObservabilityManager = dependencies.transactionObservabilityManager + this.errorResolver = dependencies.consumerErrorResolver + this.consumerOverrides = options.consumerOverrides ?? {} + this.deadLetterQueueOptions = options.deadLetterQueue + this.maxRetryDuration = options.maxRetryDuration ?? DEFAULT_MAX_RETRY_DURATION + this.executionContext = executionContext + this.isDeduplicationEnabled = !!options.enableConsumerDeduplication + + this._messageSchemaContainer = this.resolveConsumerMessageSchemaContainer(options) + this.handlerContainer = new HandlerContainer({ + messageHandlers: options.handlers, + messageTypeField: options.messageTypeField, + }) + } + + public override async init(): Promise { + // Import at method level to avoid circular dependency + const { initPubSub } = await import('../utils/pubSubInitter.ts') + + if (this.deletionConfig && this.creationConfig) { + const { deletePubSub } = await import('../utils/pubSubInitter.ts') + await deletePubSub(this.pubSubClient, this.deletionConfig, this.creationConfig) + } + + const initResult = await initPubSub( + this.pubSubClient, + this.locatorConfig, + this.creationConfig, + this.deadLetterQueueOptions, + ) + + this.topicName = initResult.topicName + this.topic = initResult.topic + this.subscriptionName = initResult.subscriptionName + this.subscription = initResult.subscription + this.dlqTopicName = initResult.dlqTopicName + + this.isInitted = true + } + + public async start(): Promise { + await this.init() + + if (!this.subscription) { + throw new Error('Subscription not initialized after init()') + } + + // Verify subscription exists before starting to listen + const [subscriptionExists] = await this.subscription.exists() + if (!subscriptionExists) { + throw new Error(`Subscription ${this.subscriptionName} does not exist after init`) + } + + this.isConsuming = true + + // Configure message handler + this.subscription.on('message', async (message: PubSubMessage) => { + await this.handleMessage(message) + }) + + // Configure error handler + this.subscription.on('error', (error) => { + this.handleError(error) + }) + + // Configure flow control if provided + // @ts-expect-error - consumerOverrides may have flowControl + if (this.consumerOverrides?.flowControl) { + this.subscription.setOptions({ + // @ts-expect-error - flowControl is available + flowControl: this.consumerOverrides.flowControl, + }) + } + } + + public override async close(): Promise { + this.isConsuming = false + if (this.subscription) { + await this.subscription.close() + } + await super.close() + } + + // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: message handling requires complex logic + private async handleMessage(message: PubSubMessage): Promise { + if (!this.isConsuming) { + // If we're shutting down, nack the message + message.nack() + return + } + + const messageProcessingStartTimestamp = Date.now() + + try { + // Parse and validate message (deserializes once via resolveMessage) + const resolvedMessage = this.resolveMessage(message) + if ('error' in resolvedMessage) { + this.handleMessageProcessed({ + message: resolvedMessage.error.message as unknown as MessagePayloadType, + processingResult: { + status: 'error', + errorReason: 'invalidMessage', + }, + messageProcessingStartTimestamp, + queueName: this.subscriptionName ?? this.topicName, + }) + message.ack() // Invalid messages should be removed + return + } + + // Retrieve offloaded payload if needed + let messagePayload = resolvedMessage.result.body + if (hasOffloadedPayload(message.attributes)) { + const retrievalResult = await this.retrieveOffloadedMessagePayload(messagePayload) + if (retrievalResult.error) { + this.handleMessageProcessed({ + message: messagePayload as MessagePayloadType, + processingResult: { + status: 'error', + errorReason: 'invalidMessage', + }, + messageProcessingStartTimestamp, + queueName: this.subscriptionName ?? this.topicName, + }) + message.ack() + return + } + messagePayload = retrievalResult.result + } + + const resolveSchemaResult = this.resolveSchema(messagePayload as MessagePayloadType) + if ('error' in resolveSchemaResult) { + this.handleError(resolveSchemaResult.error) + message.ack() + return + } + + const parseResult = parseMessage( + messagePayload, + resolveSchemaResult.result, + this.errorResolver, + ) + + if ('error' in parseResult) { + this.handleMessageProcessed({ + message: messagePayload as MessagePayloadType, + processingResult: { + status: 'error', + errorReason: 'invalidMessage', + }, + messageProcessingStartTimestamp, + queueName: this.subscriptionName ?? this.topicName, + }) + message.ack() + return + } + + const validatedMessage = parseResult.result.parsedMessage as MessagePayloadType + + // Acquire lock for message processing + const acquireLockResult = this.isDeduplicationEnabledForMessage(validatedMessage) + ? await this.acquireLockForMessage(validatedMessage) + : { result: noopReleasableLock } + + // Lock cannot be acquired as it is already being processed by another consumer. + // We don't want to discard message yet as we don't know if the other consumer will be able to process it successfully. + // We're re-queueing the message, so it can be processed later. + if ('error' in acquireLockResult) { + message.nack() + return + } + + // While the consumer was waiting for a lock to be acquired, the message might have been processed + // by another consumer already, hence we need to check again if the message is not marked as duplicated. + if ( + this.isDeduplicationEnabledForMessage(validatedMessage) && + (await this.isMessageDuplicated(validatedMessage, DeduplicationRequesterEnum.Consumer)) + ) { + await acquireLockResult.result?.release() + this.handleMessageProcessed({ + message: validatedMessage, + processingResult: { status: 'consumed', skippedAsDuplicate: true }, + messageProcessingStartTimestamp, + queueName: this.subscriptionName ?? this.topicName, + }) + message.ack() + return + } + + const releaseLock = acquireLockResult.result + + // @ts-expect-error + const messageType = validatedMessage[this.messageTypeField] + + try { + // Process message + const processingResult = await this.internalProcessMessage(validatedMessage, messageType) + + if (processingResult.error === 'retryLater') { + // Check retry duration + if (this.isRetryDateExceeded(validatedMessage)) { + this.handleMessageProcessed({ + message: validatedMessage, + processingResult: { + status: 'error', + errorReason: 'retryLaterExceeded', + }, + messageProcessingStartTimestamp, + queueName: this.subscriptionName ?? this.topicName, + }) + message.ack() // Remove from queue (should go to DLQ if configured) + } else { + this.handleMessageProcessed({ + message: validatedMessage, + processingResult: { status: 'retryLater' }, + messageProcessingStartTimestamp, + queueName: this.subscriptionName ?? this.topicName, + }) + message.nack() // Retry later + } + await releaseLock.release() + return + } + + // Success + this.handleMessageProcessed({ + message: validatedMessage, + processingResult: { status: 'consumed' }, + messageProcessingStartTimestamp, + queueName: this.subscriptionName ?? this.topicName, + }) + message.ack() + await releaseLock.release() + } catch (error) { + await releaseLock.release() + this.handleError(error as Error) + this.handleMessageProcessed({ + message: validatedMessage, + processingResult: { + status: 'error', + errorReason: 'handlerError', + }, + messageProcessingStartTimestamp, + queueName: this.subscriptionName ?? this.topicName, + }) + message.nack() + } + } catch (error) { + this.handleError(error as Error) + message.nack() + } + } + + private async internalProcessMessage( + message: MessagePayloadType, + messageType: string, + ): Promise> { + const preHandlerOutput = await this.processPrehandlers(message, messageType) + const barrierResult = await this.preHandlerBarrier(message, messageType, preHandlerOutput) + + if (barrierResult.isPassing) { + return this.processMessage(message, messageType, { + preHandlerOutput, + barrierOutput: barrierResult.output, + }) + } + + return { error: 'retryLater' } + } + + protected override resolveMessage(message: PubSubMessage) { + const deserializedPayload = deserializePubSubMessage(message, this.errorResolver) + if (deserializedPayload.error) { + return deserializedPayload + } + + return { + result: { + body: deserializedPayload.result, + attributes: message.attributes, + }, + } + } + + protected override resolveSchema(messagePayload: MessagePayloadType) { + return this._messageSchemaContainer.resolveSchema(messagePayload) + } + + protected override processMessage( + message: MessagePayloadType, + messageType: string, + // biome-ignore lint/suspicious/noExplicitAny: Expected + preHandlingOutputs: PreHandlingOutputs, + ): Promise> { + const handler = this.handlerContainer.resolveHandler(messageType) + + return handler.handler(message, this.executionContext, preHandlingOutputs) + } + + protected override processPrehandlers(message: MessagePayloadType, messageType: string) { + const handlerConfig = this.handlerContainer.resolveHandler(messageType) + + return this.processPrehandlersInternal(handlerConfig.preHandlers, message) + } + + protected override preHandlerBarrier( + message: MessagePayloadType, + messageType: string, + preHandlerOutput: PrehandlerOutput, + ): Promise> { + const handler = this.handlerContainer.resolveHandler( + messageType, + ) + + return this.preHandlerBarrierInternal( + handler.preHandlerBarrier, + message, + this.executionContext, + preHandlerOutput, + ) + } + + protected override resolveNextFunction( + preHandlers: Prehandler[], + message: MessagePayloadType, + index: number, + preHandlerOutput: PrehandlerOutput, + resolve: (value: PrehandlerOutput | PromiseLike) => void, + reject: (err: Error) => void, + ) { + return this.resolveNextPreHandlerFunctionInternal( + preHandlers, + this.executionContext, + message, + index, + preHandlerOutput, + resolve, + reject, + ) + } + + protected override resolveMessageLog(message: MessagePayloadType, messageType: string): unknown { + const handler = this.handlerContainer.resolveHandler(messageType) + return handler.messageLogFormatter(message) + } + + protected override isDeduplicationEnabledForMessage(message: MessagePayloadType): boolean { + return this.isDeduplicationEnabled && super.isDeduplicationEnabledForMessage(message) + } + + private isRetryDateExceeded(message: MessagePayloadType): boolean { + // @ts-expect-error + const timestamp = message.timestamp + if (!timestamp) { + return false + } + + const messageTimestamp = new Date(timestamp).getTime() + const now = Date.now() + const elapsedSeconds = (now - messageTimestamp) / 1000 + + return elapsedSeconds > this.maxRetryDuration + } +} diff --git a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts new file mode 100644 index 00000000..d78bdb6c --- /dev/null +++ b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts @@ -0,0 +1,170 @@ +import type { Either } from '@lokalise/node-core' +import { InternalError } from '@lokalise/node-core' +import { + type AsyncPublisher, + type BarrierResult, + DeduplicationRequesterEnum, + type MessageInvalidFormatError, + type MessageSchemaContainer, + type MessageValidationError, + type QueuePublisherOptions, + type ResolvedMessage, +} from '@message-queue-toolkit/core' + +import type { PubSubMessage } from '../types/MessageTypes.ts' +import { buildOffloadedPayloadAttributes } from '../utils/messageUtils.ts' +import type { + PubSubCreationConfig, + PubSubDependencies, + PubSubQueueLocatorType, +} from './AbstractPubSubService.ts' +import { AbstractPubSubService } from './AbstractPubSubService.ts' + +export type PubSubMessageOptions = { + orderingKey?: string + attributes?: Record +} + +export abstract class AbstractPubSubPublisher + extends AbstractPubSubService + implements AsyncPublisher +{ + private readonly messageSchemaContainer: MessageSchemaContainer + private readonly isDeduplicationEnabled: boolean + private initPromise?: Promise + + constructor( + dependencies: PubSubDependencies, + options: QueuePublisherOptions< + PubSubCreationConfig, + PubSubQueueLocatorType, + MessagePayloadType + >, + ) { + super(dependencies, options) + + this.messageSchemaContainer = this.resolvePublisherMessageSchemaContainer(options) + this.isDeduplicationEnabled = !!options.enablePublisherDeduplication + } + + async publish(message: MessagePayloadType, options: PubSubMessageOptions = {}): Promise { + const messageSchemaResult = this.resolveSchema(message) + if (messageSchemaResult.error) { + throw messageSchemaResult.error + } + + // If it's not initted yet, do the lazy init + if (!this.isInitted) { + // avoid multiple concurrent inits + if (!this.initPromise) { + this.initPromise = this.init() + } + await this.initPromise + this.initPromise = undefined + } + + try { + const messageProcessingStartTimestamp = Date.now() + const parsedMessage = messageSchemaResult.result.parse(message) + + if (this.logMessages) { + // @ts-expect-error + const resolvedLogMessage = this.resolveMessageLog(message, message[this.messageTypeField]) + this.logMessage(resolvedLogMessage) + } + + message = this.updateInternalProperties(message) + const maybeOffloadedPayloadMessage = await this.offloadMessagePayloadIfNeeded(message, () => { + // Calculate message size for PubSub + const messageData = Buffer.from(JSON.stringify(message)) + return messageData.length + }) + + if ( + this.isDeduplicationEnabledForMessage(parsedMessage) && + (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) + .isDuplicated + ) { + this.handleMessageProcessed({ + message: parsedMessage, + processingResult: { status: 'published', skippedAsDuplicate: true }, + messageProcessingStartTimestamp, + queueName: this.topicName, + }) + return + } + + await this.sendMessage(maybeOffloadedPayloadMessage, options) + this.handleMessageProcessed({ + message: parsedMessage, + processingResult: { status: 'published' }, + messageProcessingStartTimestamp, + queueName: this.topicName, + }) + } catch (error) { + const err = error as Error + this.handleError(err) + throw new InternalError({ + message: `Error while publishing to PubSub: ${err.message}`, + errorCode: 'PUBSUB_PUBLISH_ERROR', + details: { + publisher: this.constructor.name, + topicName: this.topicName, + // @ts-expect-error + messageType: message[this.messageTypeField] ?? 'unknown', + }, + cause: err, + }) + } + } + + private async sendMessage( + message: MessagePayloadType | { offloadedPayloadPointer: string; offloadedPayloadSize: number }, + options: PubSubMessageOptions, + ): Promise { + const messageData = Buffer.from(JSON.stringify(message)) + const attributes = buildOffloadedPayloadAttributes(message as unknown, options.attributes) + + await this.topic.publishMessage({ + data: messageData, + attributes, + orderingKey: options.orderingKey, + }) + } + + /* c8 ignore start */ + protected override resolveNextFunction(): () => void { + throw new Error('Not implemented for publisher') + } + + protected resolveMessage( + _message: PubSubMessage, + ): Either { + throw new Error('Not implemented for publisher') + } + + protected override processPrehandlers(): Promise { + throw new Error('Not implemented for publisher') + } + + protected override preHandlerBarrier(): Promise> { + throw new Error('Not implemented for publisher') + } + + override processMessage(): Promise> { + throw new Error('Not implemented for publisher') + } + /* c8 ignore stop */ + + protected override isDeduplicationEnabledForMessage(message: MessagePayloadType): boolean { + return this.isDeduplicationEnabled && super.isDeduplicationEnabledForMessage(message) + } + + protected override resolveSchema(message: MessagePayloadType) { + return this.messageSchemaContainer.resolveSchema(message) + } + + protected override resolveMessageLog(message: MessagePayloadType, _messageType: string): unknown { + return message + } +} diff --git a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubService.ts b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubService.ts new file mode 100644 index 00000000..aeb02573 --- /dev/null +++ b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubService.ts @@ -0,0 +1,111 @@ +import type { PubSub, Subscription, Topic } from '@google-cloud/pubsub' +import type { QueueDependencies, QueueOptions } from '@message-queue-toolkit/core' +import { AbstractQueueService } from '@message-queue-toolkit/core' +import type { PubSubMessage } from '../types/MessageTypes.ts' +import { deletePubSub, initPubSub } from '../utils/pubSubInitter.ts' + +// 10MB message size limit for Pub/Sub +export const PUBSUB_MESSAGE_MAX_SIZE = 10 * 1024 * 1024 + +export type PubSubDependencies = QueueDependencies & { + pubSubClient: PubSub +} + +export type PubSubTopicConfig = { + name: string + options?: { + messageRetentionDuration?: { + seconds: number + nanos?: number + } + messageStoragePolicy?: { + allowedPersistenceRegions?: string[] + } + kmsKeyName?: string + enableMessageOrdering?: boolean + } +} + +export type PubSubSubscriptionConfig = { + name: string + options?: { + ackDeadlineSeconds?: number + retainAckedMessages?: boolean + messageRetentionDuration?: { + seconds: number + nanos?: number + } + enableMessageOrdering?: boolean + deadLetterPolicy?: { + deadLetterTopic: string + maxDeliveryAttempts: number + } + filter?: string + enableExactlyOnceDelivery?: boolean + } +} + +export type PubSubCreationConfig = { + topic: PubSubTopicConfig + subscription?: PubSubSubscriptionConfig + updateAttributesIfExists?: boolean +} + +export type PubSubQueueLocatorType = { + topicName: string + subscriptionName?: string +} + +export abstract class AbstractPubSubService< + MessagePayloadType extends object, + QueueLocatorType extends PubSubQueueLocatorType = PubSubQueueLocatorType, + CreationConfigType extends PubSubCreationConfig = PubSubCreationConfig, + PubSubOptionsType extends QueueOptions = QueueOptions< + CreationConfigType, + QueueLocatorType + >, + DependenciesType extends PubSubDependencies = PubSubDependencies, + ExecutionContext = unknown, + PrehandlerOutput = unknown, +> extends AbstractQueueService< + MessagePayloadType, + PubSubMessage, + DependenciesType, + CreationConfigType, + QueueLocatorType, + PubSubOptionsType, + ExecutionContext, + PrehandlerOutput +> { + protected readonly pubSubClient: PubSub + + protected topicName!: string + protected topic!: Topic + protected subscriptionName?: string + protected subscription?: Subscription + + constructor(dependencies: DependenciesType, options: PubSubOptionsType) { + super(dependencies, options) + this.pubSubClient = dependencies.pubSubClient + } + + public async init(): Promise { + if (this.deletionConfig && this.creationConfig) { + await deletePubSub(this.pubSubClient, this.deletionConfig, this.creationConfig) + } + + const initResult = await initPubSub(this.pubSubClient, this.locatorConfig, this.creationConfig) + + this.topicName = initResult.topicName + this.topic = initResult.topic + this.subscriptionName = initResult.subscriptionName + this.subscription = initResult.subscription + + this.isInitted = true + } + + public override async close(): Promise { + this.isInitted = false + await Promise.resolve() + } +} diff --git a/packages/gcp-pubsub/lib/pubsub/CommonPubSubPublisherFactory.ts b/packages/gcp-pubsub/lib/pubsub/CommonPubSubPublisherFactory.ts new file mode 100644 index 00000000..a43ce2e7 --- /dev/null +++ b/packages/gcp-pubsub/lib/pubsub/CommonPubSubPublisherFactory.ts @@ -0,0 +1,33 @@ +import type { PublisherBaseEventType, QueuePublisherOptions } from '@message-queue-toolkit/core' + +import { AbstractPubSubPublisher } from './AbstractPubSubPublisher.ts' +import type { + PubSubCreationConfig, + PubSubDependencies, + PubSubQueueLocatorType, +} from './AbstractPubSubService.ts' + +export type PubSubPublisherFactory< + T extends AbstractPubSubPublisher, + M extends PublisherBaseEventType, +> = { + buildPublisher( + dependencies: PubSubDependencies, + options: QueuePublisherOptions, + ): T +} + +export class CommonPubSubPublisher< + M extends PublisherBaseEventType = PublisherBaseEventType, +> extends AbstractPubSubPublisher {} + +export class CommonPubSubPublisherFactory + implements PubSubPublisherFactory, M> +{ + buildPublisher( + dependencies: PubSubDependencies, + options: QueuePublisherOptions, + ): CommonPubSubPublisher { + return new CommonPubSubPublisher(dependencies, options) + } +} diff --git a/packages/gcp-pubsub/lib/pubsub/PubSubPublisherManager.ts b/packages/gcp-pubsub/lib/pubsub/PubSubPublisherManager.ts new file mode 100644 index 00000000..222d7ba4 --- /dev/null +++ b/packages/gcp-pubsub/lib/pubsub/PubSubPublisherManager.ts @@ -0,0 +1,128 @@ +import type { + EventRegistry, + MessagePublishType, + MessageSchemaType, + MetadataFiller, + PublisherBaseEventType, + PublisherMessageMetadataType, +} from '@message-queue-toolkit/core' +import { AbstractPublisherManager } from '@message-queue-toolkit/core' +import type z from 'zod/v4' + +import type { PubSubAwareEventDefinition } from '../schemas/pubSubSchemas.ts' +import type { AbstractPubSubPublisher, PubSubMessageOptions } from './AbstractPubSubPublisher.ts' +import type { + PubSubCreationConfig, + PubSubDependencies, + PubSubQueueLocatorType, +} from './AbstractPubSubService.ts' +import type { PubSubPublisherFactory } from './CommonPubSubPublisherFactory.ts' +import { CommonPubSubPublisherFactory } from './CommonPubSubPublisherFactory.ts' + +export type { PubSubAwareEventDefinition } + +export type PubSubPublisherManagerDependencies< + SupportedEvents extends PubSubAwareEventDefinition[], +> = { + eventRegistry: EventRegistry +} & PubSubDependencies + +export type PubSubPublisherManagerOptions< + T extends AbstractPubSubPublisher, + EventType extends PublisherBaseEventType, + MetadataType, +> = { + metadataField?: string + publisherFactory?: PubSubPublisherFactory + metadataFiller: MetadataFiller + newPublisherOptions: Omit< + import('@message-queue-toolkit/core').QueuePublisherOptions< + PubSubCreationConfig, + PubSubQueueLocatorType, + EventType + >, + 'messageSchemas' | 'creationConfig' | 'locatorConfig' + > & { + creationConfig?: Omit + } +} + +export type PubSubMessageSchemaType = z.output< + T['publisherSchema'] +> + +export class PubSubPublisherManager< + T extends AbstractPubSubPublisher>, + SupportedEventDefinitions extends PubSubAwareEventDefinition[], + MetadataType = PublisherMessageMetadataType, +> extends AbstractPublisherManager< + PubSubAwareEventDefinition, + NonNullable, + AbstractPubSubPublisher>, + PubSubDependencies, + PubSubCreationConfig, + PubSubQueueLocatorType, + PubSubMessageSchemaType, + Omit< + import('@message-queue-toolkit/core').QueuePublisherOptions< + PubSubCreationConfig, + PubSubQueueLocatorType, + z.input + >, + 'messageSchemas' | 'creationConfig' | 'locatorConfig' + >, + SupportedEventDefinitions, + MetadataType, + PubSubMessageOptions +> { + constructor( + dependencies: PubSubPublisherManagerDependencies, + options: PubSubPublisherManagerOptions< + T, + z.input, + MetadataType + >, + ) { + super({ + isAsync: true, + eventRegistry: dependencies.eventRegistry, + metadataField: options.metadataField ?? 'metadata', + metadataFiller: options.metadataFiller, + newPublisherOptions: options.newPublisherOptions, + publisherDependencies: { + pubSubClient: dependencies.pubSubClient, + logger: dependencies.logger, + errorReporter: dependencies.errorReporter, + }, + publisherFactory: options.publisherFactory ?? new CommonPubSubPublisherFactory(), + }) + } + + override publish( + topic: NonNullable, + message: MessagePublishType, + precedingEventMetadata?: Partial, + messageOptions?: PubSubMessageOptions, + ): Promise> { + // Purpose of this override is to provide better name for the first argument + // For PubSub it is going to be topic + return super.publish(topic, message, precedingEventMetadata, messageOptions) + } + + protected override resolveCreationConfig( + eventTarget: NonNullable, + ): PubSubCreationConfig { + return { + ...this.newPublisherOptions, + topic: { + name: eventTarget, + }, + } + } + + protected override resolveEventTarget( + event: PubSubAwareEventDefinition, + ): NonNullable | undefined { + return event.pubSubTopic + } +} diff --git a/packages/gcp-pubsub/lib/schemas/pubSubSchemas.ts b/packages/gcp-pubsub/lib/schemas/pubSubSchemas.ts new file mode 100644 index 00000000..bed6564c --- /dev/null +++ b/packages/gcp-pubsub/lib/schemas/pubSubSchemas.ts @@ -0,0 +1,5 @@ +import type { CommonEventDefinition } from '@message-queue-toolkit/core' + +export type PubSubAwareEventDefinition = { + pubSubTopic?: string +} & CommonEventDefinition diff --git a/packages/gcp-pubsub/lib/types/MessageTypes.ts b/packages/gcp-pubsub/lib/types/MessageTypes.ts new file mode 100644 index 00000000..1b35b598 --- /dev/null +++ b/packages/gcp-pubsub/lib/types/MessageTypes.ts @@ -0,0 +1,4 @@ +import type { Message } from '@google-cloud/pubsub' + +// PubSub message type from Google Cloud SDK +export type PubSubMessage = Message diff --git a/packages/gcp-pubsub/lib/utils/messageUtils.ts b/packages/gcp-pubsub/lib/utils/messageUtils.ts new file mode 100644 index 00000000..2ae917f8 --- /dev/null +++ b/packages/gcp-pubsub/lib/utils/messageUtils.ts @@ -0,0 +1,23 @@ +export const OFFLOADED_PAYLOAD_SIZE_ATTRIBUTE = 'offloadedPayloadSize' + +export function hasOffloadedPayload(attributes?: { [key: string]: string }): boolean { + return !!attributes && OFFLOADED_PAYLOAD_SIZE_ATTRIBUTE in attributes +} + +export function buildOffloadedPayloadAttributes( + payload: unknown, + attributes: Record = {}, +): Record { + // Check if payload has been offloaded + if ( + typeof payload === 'object' && + payload !== null && + 'offloadedPayloadPointer' in payload && + 'offloadedPayloadSize' in payload + ) { + const offloadedPayload = payload as { offloadedPayloadSize: number } + attributes[OFFLOADED_PAYLOAD_SIZE_ATTRIBUTE] = offloadedPayload.offloadedPayloadSize.toString() + } + + return attributes +} diff --git a/packages/gcp-pubsub/lib/utils/pubSubInitter.ts b/packages/gcp-pubsub/lib/utils/pubSubInitter.ts new file mode 100644 index 00000000..982a6085 --- /dev/null +++ b/packages/gcp-pubsub/lib/utils/pubSubInitter.ts @@ -0,0 +1,269 @@ +import type { PubSub, Subscription, Topic } from '@google-cloud/pubsub' +import type { DeletionConfig } from '@message-queue-toolkit/core' +import { isProduction, waitAndRetry } from '@message-queue-toolkit/core' +import type { + PubSubCreationConfig, + PubSubQueueLocatorType, +} from '../pubsub/AbstractPubSubService.ts' + +export type PubSubInitResult = { + topicName: string + topic: Topic + subscriptionName?: string + subscription?: Subscription + dlqTopicName?: string + dlqTopic?: Topic +} + +export type PubSubDeadLetterQueueConfig = { + deadLetterPolicy: { + maxDeliveryAttempts: number + } + creationConfig?: { + topic: { + name: string + } + } + locatorConfig?: { + topicName: string + } +} + +/** + * Initializes Pub/Sub resources (topics and subscriptions). + * + * Config precedence: + * - If both locatorConfig and creationConfig are provided, an error is thrown + * - locatorConfig: Locates existing resources and fails if they don't exist (production-safe) + * - creationConfig: Creates resources if they don't exist, or uses existing ones (dev-friendly) + * + * The updateAttributesIfExists flag (creationConfig only): + * - When true and resources already exist, their attributes/metadata will be updated via setMetadata() + * - When false (default), existing resources are used as-is without updates + * - Applies to both topics and subscriptions + * - Useful for updating resource configurations without manual intervention + * - Example use cases: updating retention policies, ack deadlines, or dead letter policies + * + * Dead Letter Queue configuration: + * - Optional deadLetterQueueConfig parameter enables DLQ support for subscriptions + * - Can use either locatorConfig (locate existing DLQ topic) or creationConfig (create if missing) + * - Automatically configures the subscription's deadLetterPolicy with the DLQ topic ARN + * - maxDeliveryAttempts determines how many times a message is retried before moving to DLQ + */ +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: topic/subscription initialization requires complex logic +export async function initPubSub( + pubSubClient: PubSub, + locatorConfig?: PubSubQueueLocatorType, + creationConfig?: PubSubCreationConfig, + deadLetterQueueConfig?: PubSubDeadLetterQueueConfig, +): Promise { + if (!locatorConfig && !creationConfig) { + throw new Error('Either locatorConfig or creationConfig must be provided') + } + + if (locatorConfig && creationConfig) { + throw new Error( + 'Cannot provide both locatorConfig and creationConfig. Use locatorConfig to locate existing resources or creationConfig to create resources.', + ) + } + + let topic: Topic + let topicName: string + let subscription: Subscription | undefined + let subscriptionName: string | undefined + let dlqTopicName: string | undefined + let dlqTopic: Topic | undefined + + if (locatorConfig) { + // Locate existing resources + topicName = locatorConfig.topicName + topic = pubSubClient.topic(topicName) + + const [topicExists] = await topic.exists() + if (!topicExists) { + throw new Error(`Topic ${topicName} does not exist`) + } + + if (locatorConfig.subscriptionName) { + subscriptionName = locatorConfig.subscriptionName + subscription = pubSubClient.subscription(subscriptionName) + + const [subscriptionExists] = await subscription.exists() + if (!subscriptionExists) { + throw new Error(`Subscription ${subscriptionName} does not exist`) + } + } + } else if (creationConfig) { + // Create resources if they don't exist + topicName = creationConfig.topic.name + topic = pubSubClient.topic(topicName) + + const [topicExists] = await topic.exists() + if (!topicExists) { + // Create topic first + const [createdTopic] = await topic.create() + topic = createdTopic + + // Set topic options if provided + if (creationConfig.topic.options) { + await topic.setMetadata(creationConfig.topic.options) + } + } else if (creationConfig.updateAttributesIfExists && creationConfig.topic.options) { + // Update existing topic attributes if requested + await topic.setMetadata(creationConfig.topic.options) + } + + // Handle DLQ configuration if provided (before subscription creation) + if (deadLetterQueueConfig) { + // Resolve DLQ topic name from config + if (deadLetterQueueConfig.locatorConfig) { + dlqTopicName = deadLetterQueueConfig.locatorConfig.topicName + dlqTopic = pubSubClient.topic(dlqTopicName) + + const [dlqTopicExists] = await dlqTopic.exists() + if (!dlqTopicExists) { + throw new Error(`Dead letter topic ${dlqTopicName} does not exist`) + } + } else if (deadLetterQueueConfig.creationConfig) { + dlqTopicName = deadLetterQueueConfig.creationConfig.topic.name + dlqTopic = pubSubClient.topic(dlqTopicName) + + const [dlqTopicExists] = await dlqTopic.exists() + if (!dlqTopicExists) { + const [createdDlqTopic] = await dlqTopic.create() + dlqTopic = createdDlqTopic + } + } else { + throw new Error('Either locatorConfig or creationConfig must be provided for DLQ') + } + } + + // Create subscription if config provided (for consumers) + if (creationConfig.subscription) { + subscriptionName = creationConfig.subscription.name + subscription = topic.subscription(subscriptionName) + + const [subscriptionExists] = await subscription.exists() + if (!subscriptionExists) { + // Merge deadLetterPolicy with subscription options if DLQ is configured + const subscriptionOptions = { ...creationConfig.subscription.options } + if (dlqTopic && deadLetterQueueConfig) { + subscriptionOptions.deadLetterPolicy = { + deadLetterTopic: dlqTopic.name, + maxDeliveryAttempts: deadLetterQueueConfig.deadLetterPolicy.maxDeliveryAttempts, + } + } + + const [createdSubscription] = await topic.createSubscription( + subscriptionName, + subscriptionOptions, + ) + subscription = createdSubscription + } else if (creationConfig.updateAttributesIfExists) { + // Update existing subscription attributes if requested + const updateOptions = { ...creationConfig.subscription.options } + if (dlqTopic && deadLetterQueueConfig) { + updateOptions.deadLetterPolicy = { + deadLetterTopic: dlqTopic.name, + maxDeliveryAttempts: deadLetterQueueConfig.deadLetterPolicy.maxDeliveryAttempts, + } + } + + if (Object.keys(updateOptions).length > 0) { + await subscription.setMetadata(updateOptions) + } + } + } + } else { + throw new Error('Unreachable code') + } + + return { + topicName, + topic, + subscriptionName, + subscription, + dlqTopicName, + dlqTopic, + } +} + +/** + * Deletes Pub/Sub resources (topics and subscriptions). + * + * Deletion behavior: + * - Only deletes if deletionConfig.deleteIfExists is true and creationConfig is provided + * - Checks forceDeleteInProduction flag to prevent accidental deletion in production environments + * - Deletes subscription first (if exists), then topic + * - If waitForConfirmation is true (default), polls to confirm resources are actually deleted + * using the core waitAndRetry utility (similar to SQS implementation) + */ +export async function deletePubSub( + pubSubClient: PubSub, + deletionConfig: DeletionConfig, + creationConfig?: PubSubCreationConfig, +): Promise { + if (!deletionConfig.deleteIfExists || !creationConfig) { + return + } + + if (isProduction() && !deletionConfig.forceDeleteInProduction) { + throw new Error( + 'You are running autodeletion in production. This can and probably will cause a loss of data. If you are absolutely sure you want to do this, please set deletionConfig.forceDeleteInProduction to true', + ) + } + + const shouldWaitForConfirmation = deletionConfig.waitForConfirmation !== false + + // Delete subscription first (if it exists) + if (creationConfig.subscription) { + const subscriptionName = creationConfig.subscription.name + const subscription = pubSubClient.subscription(subscriptionName) + const [subscriptionExists] = await subscription.exists() + if (subscriptionExists) { + await subscription.delete() + + if (shouldWaitForConfirmation) { + // Poll to confirm subscription is actually deleted + await waitAndRetry( + async () => { + try { + const [exists] = await subscription.exists() + return !exists + } catch { + // If exists() throws an error, the resource is deleted + return true + } + }, + 100, // 100ms sleep between checks (vs default 20ms, since we're making API calls) + 15, // max 15 retry attempts (matches SQS default) + ) + } + } + } + + // Delete topic + const topicName = creationConfig.topic.name + const topic = pubSubClient.topic(topicName) + const [topicExists] = await topic.exists() + if (topicExists) { + await topic.delete() + + if (shouldWaitForConfirmation) { + // Poll to confirm topic is actually deleted + await waitAndRetry( + async () => { + try { + const [exists] = await topic.exists() + return !exists + } catch { + // If exists() throws an error, the resource is deleted + return true + } + }, + 100, // 100ms sleep between checks + 15, // max 15 retry attempts + ) + } + } +} diff --git a/packages/gcp-pubsub/lib/utils/pubSubMessageDeserializer.ts b/packages/gcp-pubsub/lib/utils/pubSubMessageDeserializer.ts new file mode 100644 index 00000000..649c970a --- /dev/null +++ b/packages/gcp-pubsub/lib/utils/pubSubMessageDeserializer.ts @@ -0,0 +1,21 @@ +import type { Either, ErrorResolver } from '@lokalise/node-core' +import { isMessageError, type MessageInvalidFormatError } from '@message-queue-toolkit/core' +import type { PubSubMessage } from '../types/MessageTypes.ts' +import { readPubSubMessage } from './pubSubMessageReader.ts' + +export function deserializePubSubMessage( + message: PubSubMessage, + errorResolver: ErrorResolver, +): Either { + const readResult = readPubSubMessage(message) + + if ('error' in readResult) { + const resolvedError = errorResolver.processError(readResult.error) + if (isMessageError(resolvedError)) { + return { error: resolvedError } + } + return readResult + } + + return readResult +} diff --git a/packages/gcp-pubsub/lib/utils/pubSubMessageReader.ts b/packages/gcp-pubsub/lib/utils/pubSubMessageReader.ts new file mode 100644 index 00000000..cfad0fcc --- /dev/null +++ b/packages/gcp-pubsub/lib/utils/pubSubMessageReader.ts @@ -0,0 +1,23 @@ +import type { Either } from '@lokalise/node-core' +import { MessageInvalidFormatError } from '@message-queue-toolkit/core' +import type { PubSubMessage } from '../types/MessageTypes.ts' + +export function readPubSubMessage( + message: PubSubMessage, +): Either { + try { + const messageData = message.data.toString() + const parsedData = JSON.parse(messageData) + return { result: parsedData } + } catch (error) { + return { + error: new MessageInvalidFormatError({ + message: 'Invalid message format', + details: { + messageId: message.id, + processingError: (error as Error).message, + }, + }), + } + } +} diff --git a/packages/gcp-pubsub/lib/utils/pubSubUtils.ts b/packages/gcp-pubsub/lib/utils/pubSubUtils.ts new file mode 100644 index 00000000..99840ad9 --- /dev/null +++ b/packages/gcp-pubsub/lib/utils/pubSubUtils.ts @@ -0,0 +1,18 @@ +import type { PubSub } from '@google-cloud/pubsub' + +export type PubSubConfig = { + projectId: string + emulatorHost?: string +} + +export function createPubSubClient(config: PubSubConfig): PubSub { + const { PubSub } = require('@google-cloud/pubsub') + + if (config.emulatorHost) { + process.env.PUBSUB_EMULATOR_HOST = config.emulatorHost + } + + return new PubSub({ + projectId: config.projectId, + }) +} diff --git a/packages/gcp-pubsub/package.json b/packages/gcp-pubsub/package.json new file mode 100644 index 00000000..c86251be --- /dev/null +++ b/packages/gcp-pubsub/package.json @@ -0,0 +1,80 @@ +{ + "name": "@message-queue-toolkit/gcp-pubsub", + "version": "1.0.0", + "private": false, + "license": "MIT", + "description": "Google Cloud Pub/Sub adapter for message-queue-toolkit", + "maintainers": [ + { + "name": "Igor Savin", + "email": "kibertoad@gmail.com" + } + ], + "type": "module", + "main": "./dist/index.js", + "exports": { + ".": "./dist/index.js", + "./package.json": "./package.json" + }, + "scripts": { + "build": "npm run clean && tsc --project tsconfig.build.json", + "clean": "rimraf dist", + "test": "vitest", + "test:coverage": "npm run test -- --coverage", + "lint": "biome check . && tsc", + "lint:fix": "biome check --write .", + "docker:start": "docker compose up -d pubsub-emulator redis", + "docker:stop": "docker compose down", + "prepublishOnly": "npm run lint && npm run build" + }, + "dependencies": { + "@lokalise/node-core": "^14.4.2" + }, + "peerDependencies": { + "@google-cloud/pubsub": "^5.2.0", + "@message-queue-toolkit/core": ">=22.0.0", + "zod": ">=3.25.76 <5.0.0" + }, + "devDependencies": { + "@google-cloud/pubsub": "^5.2.0", + "@biomejs/biome": "^2.3.6", + "@lokalise/biome-config": "^3.1.0", + "@lokalise/tsconfig": "^3.0.0", + "@message-queue-toolkit/core": "*", + "@message-queue-toolkit/gcs-payload-store": "*", + "@message-queue-toolkit/redis-message-deduplication-store": "*", + "@message-queue-toolkit/schemas": "*", + "@types/node": "^24.10.1", + "@vitest/coverage-v8": "^3.2.4", + "awilix": "^12.0.5", + "awilix-manager": "^6.1.0", + "ioredis": "^5.7.0", + "rimraf": "^6.0.1", + "typescript": "^5.9.3", + "vitest": "^3.2.4", + "zod": "^4.1.12" + }, + "homepage": "https://github.com/kibertoad/message-queue-toolkit", + "repository": { + "type": "git", + "url": "git://github.com/kibertoad/message-queue-toolkit.git" + }, + "keywords": [ + "message", + "queue", + "queues", + "pubsub", + "pub-sub", + "google-cloud", + "gcp", + "google-cloud-pubsub", + "abstract", + "common", + "utils" + ], + "files": [ + "README.md", + "LICENSE", + "dist/*" + ] +} diff --git a/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.deadLetterQueue.spec.ts b/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.deadLetterQueue.spec.ts new file mode 100644 index 00000000..88ee7254 --- /dev/null +++ b/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.deadLetterQueue.spec.ts @@ -0,0 +1,360 @@ +import type { PubSub, Message as PubSubMessageType } from '@google-cloud/pubsub' +import { waitAndRetry } from '@lokalise/node-core' +import type { AwilixContainer } from 'awilix' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import type { PubSubPermissionPublisher } from '../publishers/PubSubPermissionPublisher.ts' +import { deletePubSubTopicAndSubscription } from '../utils/cleanupPubSub.ts' +import type { Dependencies } from '../utils/testContext.ts' +import { registerDependencies } from '../utils/testContext.ts' +import { PubSubPermissionConsumer } from './PubSubPermissionConsumer.ts' +import type { + PERMISSIONS_ADD_MESSAGE_TYPE, + PERMISSIONS_REMOVE_MESSAGE_TYPE, +} from './userConsumerSchemas.ts' + +describe('PubSubPermissionConsumer - Dead Letter Queue', () => { + const queueName = PubSubPermissionConsumer.TOPIC_NAME + const subscriptionName = PubSubPermissionConsumer.SUBSCRIPTION_NAME + const deadLetterTopicName = `${queueName}-dlq` + const deadLetterSubscriptionName = `${subscriptionName}-dlq` + + let diContainer: AwilixContainer + let pubSubClient: PubSub + let permissionPublisher: PubSubPermissionPublisher + let consumer: PubSubPermissionConsumer | undefined + let dlqSubscription: ReturnType | undefined + + beforeAll(async () => { + diContainer = await registerDependencies() + pubSubClient = diContainer.cradle.pubSubClient + permissionPublisher = diContainer.cradle.permissionPublisher + }) + + beforeEach(async () => { + await deletePubSubTopicAndSubscription(pubSubClient, queueName, subscriptionName) + await deletePubSubTopicAndSubscription( + pubSubClient, + deadLetterTopicName, + deadLetterSubscriptionName, + ) + }) + + afterEach(async () => { + await consumer?.close() + if (dlqSubscription) { + dlqSubscription.removeAllListeners() + await dlqSubscription.close() + } + dlqSubscription = undefined + consumer = undefined + }) + + afterAll(async () => { + await diContainer.cradle.awilixManager.executeDispose() + await diContainer.dispose() + }) + + describe('init', () => { + it('creates dead letter topic', async () => { + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + creationConfig: { + topic: { name: queueName }, + subscription: { name: subscriptionName }, + }, + deadLetterQueue: { + deadLetterPolicy: { maxDeliveryAttempts: 5 }, + creationConfig: { topic: { name: deadLetterTopicName } }, + }, + }) + + await consumer.init() + + expect(consumer.dlqTopicName).toBe(deadLetterTopicName) + + const dlqTopic = pubSubClient.topic(deadLetterTopicName) + const [topicExists] = await dlqTopic.exists() + expect(topicExists).toBe(true) + + // Verify subscription has deadLetterPolicy configured + const subscription = pubSubClient.subscription(subscriptionName) + const [metadata] = await subscription.getMetadata() + expect(metadata.deadLetterPolicy).toBeDefined() + expect(metadata.deadLetterPolicy?.maxDeliveryAttempts).toBe(5) + expect(metadata.deadLetterPolicy?.deadLetterTopic).toContain(deadLetterTopicName) + }) + + it('throws an error when invalid dlq locator is passed', async () => { + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + creationConfig: { + topic: { name: queueName }, + subscription: { name: subscriptionName }, + }, + deadLetterQueue: { + deadLetterPolicy: { maxDeliveryAttempts: 5 }, + locatorConfig: { topicName: 'nonexistent-topic' }, + }, + }) + + await expect(() => consumer?.init()).rejects.toThrow(/does not exist/) + }) + + it('uses existing dead letter topic when locator is passed', async () => { + // Create DLQ topic first + const dlqTopic = pubSubClient.topic(deadLetterTopicName) + await dlqTopic.create() + + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + creationConfig: { + topic: { name: queueName }, + subscription: { name: subscriptionName }, + }, + deadLetterQueue: { + deadLetterPolicy: { maxDeliveryAttempts: 5 }, + locatorConfig: { topicName: deadLetterTopicName }, + }, + }) + + await consumer.init() + + expect(consumer.dlqTopicName).toBe(deadLetterTopicName) + }) + }) + + describe('messages with errors on process should go to DLQ', () => { + it('after errors, messages should go to DLQ', async () => { + let counter = 0 + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + creationConfig: { + topic: { name: queueName }, + subscription: { name: subscriptionName }, + }, + deadLetterQueue: { + deadLetterPolicy: { maxDeliveryAttempts: 2 }, + creationConfig: { topic: { name: deadLetterTopicName } }, + }, + removeHandlerOverride: () => { + counter++ + throw new Error('Error') + }, + }) + await consumer.start() + + // Create DLQ subscription to listen for messages + const dlqTopic = pubSubClient.topic(deadLetterTopicName) + const [dlqSub] = await dlqTopic.createSubscription(deadLetterSubscriptionName) + dlqSubscription = dlqSub + + let dlqMessage: PubSubMessageType | undefined + dlqSubscription.on('message', (message: PubSubMessageType) => { + dlqMessage = message + message.ack() + }) + + await permissionPublisher.publish({ + id: '1', + messageType: 'remove', + timestamp: new Date().toISOString(), + userIds: [], + }) + + await waitAndRetry(() => dlqMessage, 50, 40) + + expect(dlqMessage).toBeDefined() + expect(counter).toBe(2) + + const dlqMessageBody = JSON.parse(dlqMessage!.data.toString()) + expect(dlqMessageBody).toMatchObject({ + id: '1', + messageType: 'remove', + timestamp: expect.any(String), + }) + }) + + it('messages with retryLater should be retried with exponential delay and not go to DLQ', async () => { + const pubsubMessage: PERMISSIONS_REMOVE_MESSAGE_TYPE = { + id: '1', + messageType: 'remove', + timestamp: new Date().toISOString(), + userIds: [], + } + + let counter = 0 + const messageArrivalTime: number[] = [] + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + creationConfig: { + topic: { name: queueName }, + subscription: { name: subscriptionName }, + }, + deadLetterQueue: { + deadLetterPolicy: { maxDeliveryAttempts: 10 }, + creationConfig: { topic: { name: deadLetterTopicName } }, + }, + removeHandlerOverride: (message) => { + if (message.id !== pubsubMessage.id) { + throw new Error('not expected message') + } + counter++ + messageArrivalTime.push(Date.now()) + return counter < 2 + ? Promise.resolve({ error: 'retryLater' }) + : Promise.resolve({ result: 'success' }) + }, + }) + await consumer.start() + + await permissionPublisher.publish(pubsubMessage) + + const handlerSpyResult = await consumer.handlerSpy.waitForMessageWithId('1', 'consumed') + expect(handlerSpyResult.processingResult).toEqual({ status: 'consumed' }) + expect(handlerSpyResult.message).toMatchObject({ id: '1', messageType: 'remove' }) + + expect(counter).toBe(2) + + // Verify retry delay (should be at least 1 second due to exponential backoff) + const secondsRetry = (messageArrivalTime[1]! - messageArrivalTime[0]!) / 1000 + expect(secondsRetry).toBeGreaterThan(1) + }) + + it('messages with deserialization errors should go to DLQ', async () => { + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + creationConfig: { + topic: { name: queueName }, + subscription: { name: subscriptionName }, + }, + deadLetterQueue: { + deadLetterPolicy: { maxDeliveryAttempts: 2 }, + creationConfig: { topic: { name: deadLetterTopicName } }, + }, + }) + await consumer.start() + + // Create DLQ subscription + const dlqTopic = pubSubClient.topic(deadLetterTopicName) + const [dlqSub] = await dlqTopic.createSubscription(deadLetterSubscriptionName) + dlqSubscription = dlqSub + + let dlqMessage: PubSubMessageType | undefined + dlqSubscription.on('message', (message: PubSubMessageType) => { + dlqMessage = message + message.ack() + }) + + // Publish invalid message directly + const topic = pubSubClient.topic(queueName) + await topic.publishMessage({ + data: Buffer.from(JSON.stringify({ id: '1', messageType: 'bad' })), + }) + + await waitAndRetry(async () => dlqMessage, 50, 40) + + expect(dlqMessage).toBeDefined() + expect(dlqMessage!.data.toString()).toBe(JSON.stringify({ id: '1', messageType: 'bad' })) + }) + }) + + describe('messages stuck should be marked as consumed and go to DLQ', () => { + it('messages stuck on barrier', async () => { + let counter = 0 + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + creationConfig: { + topic: { name: queueName }, + subscription: { name: subscriptionName }, + }, + deadLetterQueue: { + deadLetterPolicy: { maxDeliveryAttempts: 5 }, + creationConfig: { topic: { name: deadLetterTopicName } }, + }, + maxRetryDuration: 2, + addPreHandlerBarrier: (_msg) => { + counter++ + return Promise.resolve({ isPassing: false }) + }, + }) + await consumer.start() + + // Create DLQ subscription + const dlqTopic = pubSubClient.topic(deadLetterTopicName) + const [dlqSub] = await dlqTopic.createSubscription(deadLetterSubscriptionName) + dlqSubscription = dlqSub + + let dlqMessage: PubSubMessageType | undefined + dlqSubscription.on('message', (message: PubSubMessageType) => { + dlqMessage = message + message.ack() + }) + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: '1', + messageType: 'add', + timestamp: new Date(Date.now() - 1000).toISOString(), + } + await permissionPublisher.publish(message) + + const spyResult = await consumer.handlerSpy.waitForMessageWithId('1', 'error') + expect(spyResult.message).toEqual(message) + // Due to exponential backoff and timestamp, message is retried a few times before being moved to DLQ + expect(counter).toBeGreaterThanOrEqual(2) + + await waitAndRetry(() => dlqMessage, 50, 40) + const messageBody = JSON.parse(dlqMessage!.data.toString()) + expect(messageBody).toMatchObject({ + id: '1', + messageType: 'add', + timestamp: message.timestamp, + }) + }) + + it('messages stuck on handler', async () => { + let counter = 0 + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + creationConfig: { + topic: { name: queueName }, + subscription: { name: subscriptionName }, + }, + deadLetterQueue: { + deadLetterPolicy: { maxDeliveryAttempts: 5 }, + creationConfig: { topic: { name: deadLetterTopicName } }, + }, + maxRetryDuration: 2, + removeHandlerOverride: () => { + counter++ + return Promise.resolve({ error: 'retryLater' }) + }, + }) + await consumer.start() + + // Create DLQ subscription + const dlqTopic = pubSubClient.topic(deadLetterTopicName) + const [dlqSub] = await dlqTopic.createSubscription(deadLetterSubscriptionName) + dlqSubscription = dlqSub + + let dlqMessage: PubSubMessageType | undefined + dlqSubscription.on('message', (message: PubSubMessageType) => { + dlqMessage = message + message.ack() + }) + + const message: PERMISSIONS_REMOVE_MESSAGE_TYPE = { + id: '2', + messageType: 'remove', + timestamp: new Date(Date.now() - 1000).toISOString(), + userIds: [], + } + await permissionPublisher.publish(message) + + const spyResult = await consumer.handlerSpy.waitForMessageWithId('2', 'error') + expect(spyResult.message).toEqual(message) + // Due to exponential backoff and timestamp, message is retried a few times before being moved to DLQ + expect(counter).toBeGreaterThanOrEqual(2) + + await waitAndRetry(() => dlqMessage, 50, 40) + const messageBody = JSON.parse(dlqMessage!.data.toString()) + expect(messageBody).toMatchObject({ + id: '2', + messageType: 'remove', + timestamp: message.timestamp, + }) + }, 10000) + }) +}) diff --git a/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.payloadOffloading.spec.ts b/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.payloadOffloading.spec.ts new file mode 100644 index 00000000..8cf10b7c --- /dev/null +++ b/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.payloadOffloading.spec.ts @@ -0,0 +1,193 @@ +import type { PubSub } from '@google-cloud/pubsub' +import type { Storage } from '@google-cloud/storage' +import type { PayloadStoreConfig } from '@message-queue-toolkit/core' +import { GCSPayloadStore } from '@message-queue-toolkit/gcs-payload-store' +import type { AwilixContainer } from 'awilix' +import { asValue } from 'awilix' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { PUBSUB_MESSAGE_MAX_SIZE } from '../../lib/pubsub/AbstractPubSubService.ts' +import { PubSubPermissionPublisher } from '../publishers/PubSubPermissionPublisher.ts' +import { deletePubSubTopicAndSubscription } from '../utils/cleanupPubSub.ts' +import { assertBucket, emptyBucket } from '../utils/gcsUtils.ts' +import type { Dependencies } from '../utils/testContext.ts' +import { registerDependencies } from '../utils/testContext.ts' +import { PubSubPermissionConsumer } from './PubSubPermissionConsumer.ts' +import type { PERMISSIONS_ADD_MESSAGE_TYPE } from './userConsumerSchemas.ts' + +describe('PubSubPermissionConsumer - Payload Offloading', () => { + describe('consume', () => { + const largeMessageSizeThreshold = PUBSUB_MESSAGE_MAX_SIZE + const gcsBucketName = 'test-bucket' + + let diContainer: AwilixContainer + let gcsStorage: Storage + let pubSubClient: PubSub + let payloadStoreConfig: PayloadStoreConfig + + let publisher: PubSubPermissionPublisher + let consumer: PubSubPermissionConsumer + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + gcsStorage = diContainer.cradle.gcsStorage + pubSubClient = diContainer.cradle.pubSubClient + + await assertBucket(gcsStorage, gcsBucketName) + payloadStoreConfig = { + messageSizeThreshold: largeMessageSizeThreshold, + store: new GCSPayloadStore(diContainer.cradle, { bucketName: gcsBucketName }), + } + }) + + beforeEach(async () => { + consumer = new PubSubPermissionConsumer(diContainer.cradle, { + payloadStoreConfig, + }) + publisher = new PubSubPermissionPublisher(diContainer.cradle, { + payloadStoreConfig, + }) + + await deletePubSubTopicAndSubscription( + pubSubClient, + PubSubPermissionConsumer.TOPIC_NAME, + PubSubPermissionConsumer.SUBSCRIPTION_NAME, + ) + + await publisher.init() + await consumer.init() + await consumer.start() + }) + + afterEach(async () => { + await publisher.close() + await consumer.close() + }) + + afterAll(async () => { + await emptyBucket(gcsStorage, gcsBucketName) + + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('consumes large message with offloaded payload', async () => { + // Craft a message that is larger than the max message size + const message = { + id: 'large-message-1', + messageType: 'add', + timestamp: new Date().toISOString(), + metadata: { + largeField: 'a'.repeat(largeMessageSizeThreshold), + }, + } satisfies PERMISSIONS_ADD_MESSAGE_TYPE + + expect(JSON.stringify(message).length).toBeGreaterThan(largeMessageSizeThreshold) + + await publisher.publish(message) + + const consumptionResult = await consumer.handlerSpy.waitForMessageWithId( + message.id, + 'consumed', + ) + expect(consumptionResult.message).toMatchObject(message) + expect(consumer.addCounter).toBe(1) + }) + + it('consumes normal-sized message without offloading', async () => { + const message = { + id: 'normal-message-1', + messageType: 'add', + timestamp: new Date().toISOString(), + userIds: ['user1', 'user2'], + } satisfies PERMISSIONS_ADD_MESSAGE_TYPE + + expect(JSON.stringify(message).length).toBeLessThan(largeMessageSizeThreshold) + + await publisher.publish(message) + + const consumptionResult = await consumer.handlerSpy.waitForMessageWithId( + message.id, + 'consumed', + ) + expect(consumptionResult.message).toMatchObject(message) + expect(consumer.addCounter).toBe(1) + }) + + it('consumes offloaded message with array field and validates schema correctly', async () => { + // Create a large array of userIds to trigger offloading + const largeUserIdArray = Array.from({ length: 10000 }, (_, i) => `user-${i}`) + + const message = { + id: 'large-array-message-1', + messageType: 'add', + timestamp: new Date().toISOString(), + userIds: largeUserIdArray, + } satisfies PERMISSIONS_ADD_MESSAGE_TYPE + + // Verify the message is large enough to trigger offloading + expect(JSON.stringify(message).length).toBeGreaterThan(largeMessageSizeThreshold) + + await publisher.publish(message) + + // Wait for the message to be consumed + const consumptionResult = await consumer.handlerSpy.waitForMessageWithId( + message.id, + 'consumed', + ) + + // Verify the full payload was received including the large array + expect(consumptionResult.message).toMatchObject({ + id: message.id, + messageType: message.messageType, + userIds: largeUserIdArray, + }) + expect(consumptionResult.message.userIds).toHaveLength(largeUserIdArray.length) + expect(consumer.addCounter).toBe(1) + }) + + it('validates schema correctly after retrieving offloaded payload', async () => { + // Create a message with metadata that will be validated against the schema + const message = { + id: 'schema-validation-1', + messageType: 'add', + timestamp: new Date().toISOString(), + metadata: { + largeField: 'x'.repeat(largeMessageSizeThreshold + 1000), + }, + userIds: ['test-user'], + } satisfies PERMISSIONS_ADD_MESSAGE_TYPE + + expect(JSON.stringify(message).length).toBeGreaterThan(largeMessageSizeThreshold) + + await publisher.publish(message) + + const consumptionResult = await consumer.handlerSpy.waitForMessageWithId( + message.id, + 'consumed', + ) + + // Verify all fields were properly deserialized and validated + expect(consumptionResult.message).toMatchObject({ + id: message.id, + messageType: message.messageType, + userIds: message.userIds, + metadata: { + largeField: message.metadata.largeField, + }, + }) + + // Type guard to access metadata property + if (consumptionResult.message.messageType === 'add') { + expect(consumptionResult.message.metadata?.largeField).toHaveLength( + message.metadata.largeField.length, + ) + } + expect(consumer.addCounter).toBe(1) + }) + }) +}) diff --git a/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.spec.ts b/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.spec.ts new file mode 100644 index 00000000..64ccb7ee --- /dev/null +++ b/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.spec.ts @@ -0,0 +1,220 @@ +import type { PubSub } from '@google-cloud/pubsub' +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' +// biome-ignore lint/style/useImportType: need class for static properties +import { PubSubPermissionPublisher } from '../publishers/PubSubPermissionPublisher.ts' +import { deletePubSubTopicAndSubscription } from '../utils/cleanupPubSub.ts' +import { registerDependencies } from '../utils/testContext.ts' +import { PubSubPermissionConsumer } from './PubSubPermissionConsumer.ts' + +describe('PubSubPermissionConsumer', () => { + let diContainer: Awaited> + let consumer: PubSubPermissionConsumer + let publisher: PubSubPermissionPublisher + let pubSubClient: PubSub + + beforeAll(async () => { + diContainer = await registerDependencies() + consumer = diContainer.cradle.permissionConsumer + publisher = diContainer.cradle.permissionPublisher + pubSubClient = diContainer.cradle.pubSubClient + }) + + beforeEach(async () => { + consumer.addCounter = 0 + consumer.removeCounter = 0 + consumer.processedMessagesIds.clear() + + // Clean up topics and subscriptions + await deletePubSubTopicAndSubscription( + pubSubClient, + PubSubPermissionConsumer.TOPIC_NAME, + PubSubPermissionConsumer.SUBSCRIPTION_NAME, + ) + + // Reinitialize + await consumer.close() + await publisher.close() + await publisher.init() + await consumer.init() + await consumer.start() + }) + + afterAll(async () => { + await diContainer.dispose() + }) + + describe('init', () => { + it('creates topic and subscription', async () => { + const topic = pubSubClient.topic(PubSubPermissionConsumer.TOPIC_NAME) + const subscription = topic.subscription(PubSubPermissionConsumer.SUBSCRIPTION_NAME) + + const [topicExists] = await topic.exists() + const [subExists] = await subscription.exists() + + expect(topicExists).toBe(true) + expect(subExists).toBe(true) + }) + }) + + describe('message consumption', () => { + it('consumes add messages', async () => { + const message = { + id: 'add-1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['user1', 'user2'], + } + + await publisher.publish(message) + + // Wait for message to be processed + await consumer.handlerSpy.waitForMessageWithId('add-1', 'consumed') + + expect(consumer.addCounter).toBe(1) + expect(consumer.removeCounter).toBe(0) + expect(consumer.processedMessagesIds.has('add-1')).toBe(true) + }) + + it('consumes remove messages', async () => { + const message = { + id: 'remove-1', + messageType: 'remove' as const, + timestamp: new Date().toISOString(), + userIds: ['user1'], + } + + await publisher.publish(message) + + await consumer.handlerSpy.waitForMessageWithId('remove-1', 'consumed') + + expect(consumer.addCounter).toBe(0) + expect(consumer.removeCounter).toBe(1) + }) + + it('consumes multiple messages in order', async () => { + const messages = [ + { + id: 'msg-1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['user1'], + }, + { + id: 'msg-2', + messageType: 'remove' as const, + timestamp: new Date().toISOString(), + userIds: ['user2'], + }, + { + id: 'msg-3', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['user3'], + }, + ] + + for (const msg of messages) { + await publisher.publish(msg) + } + + await consumer.handlerSpy.waitForMessageWithId('msg-1', 'consumed') + await consumer.handlerSpy.waitForMessageWithId('msg-2', 'consumed') + await consumer.handlerSpy.waitForMessageWithId('msg-3', 'consumed') + + expect(consumer.addCounter).toBe(2) + expect(consumer.removeCounter).toBe(1) + expect(consumer.processedMessagesIds.size).toBe(2) // Only add messages tracked + }) + }) + + describe('handler spy', () => { + it('tracks consumed messages', async () => { + const message = { + id: 'spy-test-1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['user1'], + } + + await publisher.publish(message) + + const spyResult = await consumer.handlerSpy.waitForMessageWithId('spy-test-1', 'consumed') + + expect(spyResult).toBeDefined() + expect(spyResult.message.id).toBe('spy-test-1') + expect(spyResult.processingResult.status).toBe('consumed') + }) + + it('waitForMessageWithId waits for messages published after the spy starts waiting', async () => { + const message = { + id: 'wait-test-1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['user1'], + } + + // Start waiting BEFORE publishing + const spyPromise = consumer.handlerSpy.waitForMessageWithId('wait-test-1', 'consumed') + + // Now publish the message + await publisher.publish(message) + + // The spy should resolve once the message is processed + const spyResult = await spyPromise + + expect(spyResult).toBeDefined() + expect(spyResult.message.id).toBe('wait-test-1') + expect(spyResult.processingResult.status).toBe('consumed') + }) + }) + + describe('error handling', () => { + it('handles invalid message format gracefully', async () => { + // Publish directly via Pub/Sub to bypass validation + const topic = pubSubClient.topic(PubSubPermissionConsumer.TOPIC_NAME) + + await topic.publishMessage({ + data: Buffer.from('invalid json'), + }) + + // Wait a bit for processing + await new Promise((resolve) => setTimeout(resolve, 1000)) + + // Consumer should still be running + expect(consumer.addCounter).toBe(0) + expect(consumer.removeCounter).toBe(0) + }) + + it('tracks schema validation errors with handlerSpy', async () => { + const topic = pubSubClient.topic(PubSubPermissionConsumer.TOPIC_NAME) + + // Create a message with valid JSON but invalid schema (userIds should be array, not string) + const invalidMessage = { + id: 'error-test-1', + messageType: 'add', + timestamp: new Date().toISOString(), + userIds: 'invalid-should-be-array', // Invalid type - should fail validation + } + + // Start waiting for the error + const spyPromise = consumer.handlerSpy.waitForMessage({ id: 'error-test-1' }, 'error') + + // Publish the invalid message + await topic.publishMessage({ + data: Buffer.from(JSON.stringify(invalidMessage)), + }) + + // Wait for the error to be tracked + const spyResult = await spyPromise + + expect(spyResult).toBeDefined() + expect(spyResult.processingResult.status).toBe('error') + // @ts-expect-error field is there + expect(spyResult.processingResult.errorReason).toBe('invalidMessage') + + // Consumer should still be running and not have processed the message + expect(consumer.addCounter).toBe(0) + expect(consumer.removeCounter).toBe(0) + }) + }) +}) diff --git a/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.ts b/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.ts new file mode 100644 index 00000000..68762907 --- /dev/null +++ b/packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.ts @@ -0,0 +1,140 @@ +import type { Either } from '@lokalise/node-core' +import type { BarrierResult, PreHandlingOutputs, Prehandler } from '@message-queue-toolkit/core' +import { MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' + +import type { + PubSubConsumerDependencies, + PubSubConsumerOptions, +} from '../../lib/pubsub/AbstractPubSubConsumer.ts' +import { AbstractPubSubConsumer } from '../../lib/pubsub/AbstractPubSubConsumer.ts' + +import type { + PERMISSIONS_ADD_MESSAGE_TYPE, + PERMISSIONS_REMOVE_MESSAGE_TYPE, +} from './userConsumerSchemas.ts' +import { + PERMISSIONS_ADD_MESSAGE_SCHEMA, + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, +} from './userConsumerSchemas.ts' + +export type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE + +type PubSubPermissionConsumerOptions = Pick< + PubSubConsumerOptions, + | 'creationConfig' + | 'locatorConfig' + | 'logMessages' + | 'deletionConfig' + | 'deadLetterQueue' + | 'consumerOverrides' + | 'maxRetryDuration' + | 'payloadStoreConfig' + | 'messageDeduplicationConfig' + | 'enableConsumerDeduplication' +> & { + addPreHandlerBarrier?: ( + message: SupportedMessages, + _executionContext: ExecutionContext, + preHandlerOutput: PrehandlerOutput, + ) => Promise> + removeHandlerOverride?: ( + _message: SupportedMessages, + context: ExecutionContext, + preHandlingOutputs: PreHandlingOutputs, + ) => Promise> + addHandlerOverride?: ( + message: SupportedMessages, + context: ExecutionContext, + preHandlingOutputs: PreHandlingOutputs, + ) => Promise> + removePreHandlers?: Prehandler[] +} + +type ExecutionContext = { + incrementAmount: number +} +type PrehandlerOutput = { + messageId: string +} + +export class PubSubPermissionConsumer extends AbstractPubSubConsumer< + SupportedMessages, + ExecutionContext, + PrehandlerOutput +> { + public addCounter = 0 + public removeCounter = 0 + public processedMessagesIds: Set = new Set() + public static readonly TOPIC_NAME = 'user_permissions' + public static readonly SUBSCRIPTION_NAME = 'user_permissions_sub' + + constructor( + dependencies: PubSubConsumerDependencies, + options: PubSubPermissionConsumerOptions = { + creationConfig: { + topic: { + name: PubSubPermissionConsumer.TOPIC_NAME, + }, + subscription: { + name: PubSubPermissionConsumer.SUBSCRIPTION_NAME, + }, + }, + }, + ) { + const defaultRemoveHandler = ( + _message: SupportedMessages, + context: ExecutionContext, + _preHandlingOutputs: PreHandlingOutputs, + ): Promise> => { + this.removeCounter += context.incrementAmount + return Promise.resolve({ + result: 'success', + }) + } + + const defaultAddHandler = ( + message: SupportedMessages, + context: ExecutionContext, + barrierOutput: PreHandlingOutputs, + ): Promise> => { + if (options.addPreHandlerBarrier && !barrierOutput) { + return Promise.resolve({ error: 'retryLater' }) + } + this.addCounter += context.incrementAmount + this.processedMessagesIds.add(message.id) + return Promise.resolve({ result: 'success' }) + } + + super( + dependencies, + { + ...options, + messageTypeField: 'messageType', + handlerSpy: true, + handlers: new MessageHandlerConfigBuilder< + SupportedMessages, + ExecutionContext, + PrehandlerOutput + >() + .addConfig( + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, + options.removeHandlerOverride ?? defaultRemoveHandler, + { + preHandlers: options.removePreHandlers, + }, + ) + .addConfig( + PERMISSIONS_ADD_MESSAGE_SCHEMA, + options.addHandlerOverride ?? defaultAddHandler, + { + preHandlerBarrier: options.addPreHandlerBarrier, + }, + ) + .build(), + }, + { + incrementAmount: 1, + }, + ) + } +} diff --git a/packages/gcp-pubsub/test/consumers/userConsumerSchemas.ts b/packages/gcp-pubsub/test/consumers/userConsumerSchemas.ts new file mode 100644 index 00000000..fd9d489b --- /dev/null +++ b/packages/gcp-pubsub/test/consumers/userConsumerSchemas.ts @@ -0,0 +1,19 @@ +import { z } from 'zod/v4' + +export const PERMISSIONS_ADD_MESSAGE_SCHEMA = z.object({ + id: z.string(), + messageType: z.literal('add'), + timestamp: z.string().datetime(), + userIds: z.array(z.string()).optional(), + metadata: z.object({ largeField: z.string() }).optional(), +}) + +export const PERMISSIONS_REMOVE_MESSAGE_SCHEMA = z.object({ + id: z.string(), + messageType: z.literal('remove'), + timestamp: z.string().datetime(), + userIds: z.array(z.string()), +}) + +export type PERMISSIONS_ADD_MESSAGE_TYPE = z.infer +export type PERMISSIONS_REMOVE_MESSAGE_TYPE = z.infer diff --git a/packages/gcp-pubsub/test/fakes/FakeLogger.ts b/packages/gcp-pubsub/test/fakes/FakeLogger.ts new file mode 100644 index 00000000..2900f40a --- /dev/null +++ b/packages/gcp-pubsub/test/fakes/FakeLogger.ts @@ -0,0 +1,52 @@ +import type { CommonLogger } from '@lokalise/node-core' +import type pino from 'pino' +import type { Bindings, ChildLoggerOptions } from 'pino' + +export class FakeLogger implements CommonLogger { + public readonly loggedMessages: unknown[] = [] + public readonly loggedWarnings: unknown[] = [] + public readonly loggedErrors: unknown[] = [] + + public readonly level: pino.LevelWithSilentOrString + + constructor(level: pino.LevelWithSilentOrString = 'debug') { + this.level = level + } + + get msgPrefix(): string | undefined { + return undefined + } + + debug(obj: unknown) { + this.loggedMessages.push(obj) + } + error(obj: unknown) { + this.loggedErrors.push(obj) + } + fatal(obj: unknown) { + this.loggedErrors.push(obj) + } + info(obj: unknown) { + this.loggedMessages.push(obj) + } + trace(obj: unknown) { + this.loggedMessages.push(obj) + } + warn(obj: unknown) { + this.loggedWarnings.push(obj) + } + // Noop function + silent(_obj: unknown) { + return + } + + // Child has no effect for FakeLogger + child(_bindings: Bindings, _options?: ChildLoggerOptions): CommonLogger { + return this + } + + isLevelEnabled(_level: pino.LevelWithSilentOrString): boolean { + // For FakeLogger we want to track all logs + return true + } +} diff --git a/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.spec.ts b/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.spec.ts new file mode 100644 index 00000000..8e99e475 --- /dev/null +++ b/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.spec.ts @@ -0,0 +1,145 @@ +import type { PubSub } from '@google-cloud/pubsub' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { deletePubSubTopic } from '../utils/cleanupPubSub.ts' +import { registerDependencies } from '../utils/testContext.ts' +import { PubSubPermissionPublisher } from './PubSubPermissionPublisher.ts' + +describe('PubSubPermissionPublisher', () => { + let diContainer: Awaited> + let publisher: PubSubPermissionPublisher + let pubSubClient: PubSub + + beforeAll(async () => { + diContainer = await registerDependencies() + publisher = diContainer.cradle.permissionPublisher + pubSubClient = diContainer.cradle.pubSubClient + + // Clean up - delete topic if exists before tests start + await deletePubSubTopic(pubSubClient, PubSubPermissionPublisher.TOPIC_NAME) + }) + + afterAll(async () => { + await diContainer.dispose() + }) + + describe('init', () => { + it('creates a new topic', async () => { + const newPublisher = diContainer.cradle.permissionPublisher + + await newPublisher.init() + + const [exists] = await pubSubClient.topic(PubSubPermissionPublisher.TOPIC_NAME).exists() + expect(exists).toBe(true) + + await newPublisher.close() + }) + + it('does not throw an error when initiated twice', async () => { + const newPublisher = diContainer.cradle.permissionPublisher + + await newPublisher.init() + await newPublisher.init() + + const [exists] = await pubSubClient.topic(PubSubPermissionPublisher.TOPIC_NAME).exists() + expect(exists).toBe(true) + + await newPublisher.close() + }) + }) + + describe('publish', () => { + it('publishes a message to topic', async () => { + const message = { + id: '1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['userId1'], + } + + await publisher.publish(message) + + // Verify message was published (topic should exist and have been used) + const [exists] = await pubSubClient.topic(PubSubPermissionPublisher.TOPIC_NAME).exists() + expect(exists).toBe(true) + }) + + it('publishes multiple messages', async () => { + const message1 = { + id: '1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['userId1'], + } + + const message2 = { + id: '2', + messageType: 'remove' as const, + timestamp: new Date().toISOString(), + userIds: ['userId2'], + } + + await publisher.publish(message1) + await publisher.publish(message2) + + const [exists] = await pubSubClient.topic(PubSubPermissionPublisher.TOPIC_NAME).exists() + expect(exists).toBe(true) + }) + + it('publishes message with ordering key', async () => { + const message = { + id: '1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['userId1'], + } + + await publisher.publish(message, { orderingKey: 'user-123' }) + + const [exists] = await pubSubClient.topic(PubSubPermissionPublisher.TOPIC_NAME).exists() + expect(exists).toBe(true) + }) + + it('publishes message with custom attributes', async () => { + const message = { + id: '1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['userId1'], + } + + await publisher.publish(message, { + attributes: { + customKey: 'customValue', + }, + }) + + const [exists] = await pubSubClient.topic(PubSubPermissionPublisher.TOPIC_NAME).exists() + expect(exists).toBe(true) + }) + }) + + describe('handler spy', () => { + it('records published messages', async () => { + const newPublisher = diContainer.cradle.permissionPublisher + + const message = { + id: 'spy-test-1', + messageType: 'add' as const, + timestamp: new Date().toISOString(), + userIds: ['userId1'], + } + + await newPublisher.publish(message) + + const spy = newPublisher.handlerSpy + const spyResult = await spy.waitForMessageWithId('spy-test-1', 'published') + + expect(spyResult).toBeDefined() + expect(spyResult.message.id).toBe('spy-test-1') + expect(spyResult.processingResult.status).toBe('published') + + await newPublisher.close() + }) + }) +}) diff --git a/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.ts b/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.ts new file mode 100644 index 00000000..c5db7763 --- /dev/null +++ b/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.ts @@ -0,0 +1,59 @@ +import type { PubSubMessageOptions } from '../../lib/pubsub/AbstractPubSubPublisher.ts' +import { AbstractPubSubPublisher } from '../../lib/pubsub/AbstractPubSubPublisher.ts' +import type { PubSubDependencies } from '../../lib/pubsub/AbstractPubSubService.ts' +import type { + PERMISSIONS_ADD_MESSAGE_TYPE, + PERMISSIONS_REMOVE_MESSAGE_TYPE, +} from '../consumers/userConsumerSchemas.ts' +import { + PERMISSIONS_ADD_MESSAGE_SCHEMA, + PERMISSIONS_REMOVE_MESSAGE_SCHEMA, +} from '../consumers/userConsumerSchemas.ts' + +type SupportedMessages = PERMISSIONS_ADD_MESSAGE_TYPE | PERMISSIONS_REMOVE_MESSAGE_TYPE + +type PubSubPermissionPublisherOptions = { + creationConfig?: { + topic: { + name: string + } + } + locatorConfig?: { + topicName: string + } + logMessages?: boolean + deletionConfig?: { + deleteIfExists: boolean + } + payloadStoreConfig?: any + enablePublisherDeduplication?: boolean +} + +export class PubSubPermissionPublisher extends AbstractPubSubPublisher { + public static TOPIC_NAME = 'user_permissions' + + constructor( + dependencies: PubSubDependencies, + options: PubSubPermissionPublisherOptions = { + creationConfig: { + topic: { + name: PubSubPermissionPublisher.TOPIC_NAME, + }, + }, + }, + ) { + super(dependencies, { + ...options, + messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], + messageTypeField: 'messageType', + handlerSpy: true, + }) + } + + override async publish( + message: SupportedMessages, + options?: PubSubMessageOptions, + ): Promise { + return await super.publish(message, options) + } +} diff --git a/packages/gcp-pubsub/test/pubsub/PubSubPublisherManager.spec.ts b/packages/gcp-pubsub/test/pubsub/PubSubPublisherManager.spec.ts new file mode 100644 index 00000000..80aef960 --- /dev/null +++ b/packages/gcp-pubsub/test/pubsub/PubSubPublisherManager.spec.ts @@ -0,0 +1,246 @@ +import { randomUUID } from 'node:crypto' + +import { + CommonMetadataFiller, + EventRegistry, + enrichMessageSchemaWithBase, +} from '@message-queue-toolkit/core' +import type { AwilixContainer } from 'awilix' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import z from 'zod/v4' +import { CommonPubSubPublisher } from '../../lib/pubsub/CommonPubSubPublisherFactory.ts' +import type { PubSubAwareEventDefinition } from '../../lib/pubsub/PubSubPublisherManager.ts' +import { PubSubPublisherManager } from '../../lib/pubsub/PubSubPublisherManager.ts' +import type { Dependencies } from '../utils/testContext.ts' +import { registerDependencies } from '../utils/testContext.ts' + +const TestEvents = { + created: { + ...enrichMessageSchemaWithBase( + 'entity.created', + z.object({ + newData: z.string(), + }), + ), + schemaVersion: '1.0.1', + pubSubTopic: 'test-topic-created', + }, + + updated: { + ...enrichMessageSchemaWithBase( + 'entity.updated', + z.object({ + updatedData: z.string(), + }), + ), + pubSubTopic: 'test-topic-updated', + }, +} as const satisfies Record + +type TestEventsType = (typeof TestEvents)[keyof typeof TestEvents][] +type TestEventPublishPayloadsType = z.output + +describe('PubSubPublisherManager', () => { + let diContainer: AwilixContainer + let publisherManager: PubSubPublisherManager< + CommonPubSubPublisher, + TestEventsType + > + + beforeAll(async () => { + diContainer = await registerDependencies() + + const eventRegistry = new EventRegistry(Object.values(TestEvents)) + + publisherManager = new PubSubPublisherManager( + { + ...diContainer.cradle, + eventRegistry, + }, + { + metadataField: 'metadata', + metadataFiller: new CommonMetadataFiller({ + serviceId: 'test-service', + }), + newPublisherOptions: { + handlerSpy: true, + messageTypeField: 'type', + logMessages: false, + }, + }, + ) + + await publisherManager.initRegisteredPublishers() + }) + + afterAll(async () => { + await diContainer.dispose() + }) + + describe('publish', () => { + it('publishes to a correct publisher', async () => { + // When + const publishedMessage = await publisherManager.publish(TestEvents.created.pubSubTopic, { + payload: { + newData: 'msg', + }, + type: 'entity.created', + }) + + const publishedMessageResult = await publisherManager + .handlerSpy(TestEvents.created.pubSubTopic) + .waitForMessageWithId(publishedMessage.id) + + expect(publishedMessageResult.processingResult.status).toEqual('published') + + expect(publishedMessage).toMatchObject({ + id: expect.any(String), + metadata: { + correlationId: expect.any(String), + originatedFrom: 'test-service', + producedBy: 'test-service', + schemaVersion: '1.0.1', + }, + payload: { + newData: 'msg', + }, + timestamp: expect.any(String), + type: 'entity.created', + }) + }) + + it('publishes to different topics', async () => { + // Publish to created topic + const createdMessage = await publisherManager.publish(TestEvents.created.pubSubTopic, { + payload: { + newData: 'created msg', + }, + type: 'entity.created', + }) + + // Publish to updated topic + const updatedMessage = await publisherManager.publish(TestEvents.updated.pubSubTopic, { + payload: { + updatedData: 'updated msg', + }, + type: 'entity.updated', + }) + + const createdResult = await publisherManager + .handlerSpy(TestEvents.created.pubSubTopic) + .waitForMessageWithId(createdMessage.id) + + const updatedResult = await publisherManager + .handlerSpy(TestEvents.updated.pubSubTopic) + .waitForMessageWithId(updatedMessage.id) + + expect(createdResult.processingResult.status).toEqual('published') + expect(updatedResult.processingResult.status).toEqual('published') + expect(createdMessage.type).toBe('entity.created') + expect(updatedMessage.type).toBe('entity.updated') + }) + + it('message publishing is type-safe', async () => { + await expect( + publisherManager.publish(TestEvents.created.pubSubTopic, { + payload: { + // @ts-expect-error This should be causing a compilation error + updatedData: 'edwe', + }, + type: 'entity.created', + }), + ).rejects.toThrow(/invalid_type/) + }) + + it('publish to a non-existing topic will throw error', async () => { + await expect( + // @ts-expect-error Testing error scenario + publisherManager.publish('non-existing-topic', { + type: 'entity.created', + payload: { + newData: 'msg', + }, + }), + ).rejects.toThrow('No publisher for target non-existing-topic') + }) + + it('publish to an incorrect topic/message combination will throw error', async () => { + await expect( + publisherManager.publish(TestEvents.created.pubSubTopic, { + // @ts-expect-error Testing error scenario + type: 'dummy.type', + payload: { + newData: 'msg', + }, + }), + ).rejects.toThrow( + 'MessageDefinition for target "test-topic-created" and type "dummy.type" not found in EventRegistry', + ) + }) + }) + + describe('handlerSpy', () => { + it('returns correct handler spy', () => { + const spy = publisherManager.handlerSpy(TestEvents.created.pubSubTopic) + expect(spy).toBeDefined() + }) + + it('returns error when no publisher for topic', () => { + // @ts-expect-error Testing incorrect scenario + expect(() => publisherManager.handlerSpy('non-existing-topic')).toThrow( + 'No publisher for target non-existing-topic', + ) + }) + }) + + describe('injectPublisher', () => { + it('works correctly', async () => { + // Given + const topic = 'test-injected-topic' + const injectedSchema = enrichMessageSchemaWithBase( + 'entity.created', + z.object({}).catchall(z.any()), + ) + const newPublisher = new CommonPubSubPublisher(diContainer.cradle, { + creationConfig: { + topic: { + name: topic, + }, + }, + handlerSpy: true, + messageTypeField: 'type', + messageSchemas: [injectedSchema.consumerSchema], + }) + + await newPublisher.init() + + // When + const messageId = randomUUID() + // @ts-expect-error + publisherManager.injectPublisher(topic, newPublisher) + publisherManager.injectEventDefinition({ + ...injectedSchema, + pubSubTopic: topic, + schemaVersion: '2.0.0', + }) + + // Then + const { timestamp } = publisherManager.resolveBaseFields() + await publisherManager.publish( + // @ts-expect-error + topic, + { + id: messageId, + type: 'entity.created', + timestamp, + payload: {}, + }, + ) + + const result = await newPublisher.handlerSpy.waitForMessageWithId(messageId) + expect(result.processingResult.status).toBe('published') + + await newPublisher.close() + }) + }) +}) diff --git a/packages/gcp-pubsub/test/utils/cleanRedis.ts b/packages/gcp-pubsub/test/utils/cleanRedis.ts new file mode 100644 index 00000000..b2af5181 --- /dev/null +++ b/packages/gcp-pubsub/test/utils/cleanRedis.ts @@ -0,0 +1,5 @@ +import type { Redis } from 'ioredis' + +export async function cleanRedis(redis: Redis): Promise { + await redis.flushdb() +} diff --git a/packages/gcp-pubsub/test/utils/cleanupPubSub.ts b/packages/gcp-pubsub/test/utils/cleanupPubSub.ts new file mode 100644 index 00000000..f3d506c9 --- /dev/null +++ b/packages/gcp-pubsub/test/utils/cleanupPubSub.ts @@ -0,0 +1,42 @@ +import type { PubSub } from '@google-cloud/pubsub' + +export async function deletePubSubTopic(pubSubClient: PubSub, topicName: string): Promise { + const topic = pubSubClient.topic(topicName) + const [exists] = await topic.exists() + if (exists) { + await topic.delete() + } +} + +export async function deletePubSubSubscription( + pubSubClient: PubSub, + topicName: string, + subscriptionName: string, +): Promise { + const topic = pubSubClient.topic(topicName) + const [topicExists] = await topic.exists() + if (topicExists) { + const subscription = topic.subscription(subscriptionName) + const [subExists] = await subscription.exists() + if (subExists) { + await subscription.delete() + } + } +} + +export async function deletePubSubTopicAndSubscription( + pubSubClient: PubSub, + topicName: string, + subscriptionName: string, +): Promise { + const topic = pubSubClient.topic(topicName) + const [topicExists] = await topic.exists() + if (topicExists) { + const subscription = topic.subscription(subscriptionName) + const [subExists] = await subscription.exists() + if (subExists) { + await subscription.delete() + } + await topic.delete() + } +} diff --git a/packages/gcp-pubsub/test/utils/gcsUtils.ts b/packages/gcp-pubsub/test/utils/gcsUtils.ts new file mode 100644 index 00000000..11fe5958 --- /dev/null +++ b/packages/gcp-pubsub/test/utils/gcsUtils.ts @@ -0,0 +1,24 @@ +import type { Storage } from '@google-cloud/storage' + +export async function assertBucket(storage: Storage, bucketName: string) { + const bucket = storage.bucket(bucketName) + const [exists] = await bucket.exists() + + if (!exists) { + await bucket.create() + } + + return bucket +} + +export async function emptyBucket(storage: Storage, bucketName: string) { + const bucket = storage.bucket(bucketName) + const [exists] = await bucket.exists() + + if (!exists) { + return + } + + const [files] = await bucket.getFiles() + await Promise.all(files.map((file) => file.delete({ ignoreNotFound: true }))) +} diff --git a/packages/gcp-pubsub/test/utils/pubSubInitter.spec.ts b/packages/gcp-pubsub/test/utils/pubSubInitter.spec.ts new file mode 100644 index 00000000..91ebe056 --- /dev/null +++ b/packages/gcp-pubsub/test/utils/pubSubInitter.spec.ts @@ -0,0 +1,356 @@ +import type { PubSub } from '@google-cloud/pubsub' +import type { DeletionConfig } from '@message-queue-toolkit/core' +import { reloadConfig } from '@message-queue-toolkit/core' +import type { AwilixContainer } from 'awilix' +import { asValue } from 'awilix' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { deletePubSub, initPubSub } from '../../lib/utils/pubSubInitter.ts' +import { deletePubSubTopicAndSubscription } from '../utils/cleanupPubSub.ts' +import type { Dependencies } from '../utils/testContext.ts' +import { registerDependencies } from '../utils/testContext.ts' + +describe('pubSubInitter', () => { + let diContainer: AwilixContainer + let pubSubClient: PubSub + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + pubSubClient = diContainer.cradle.pubSubClient + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + describe('updateAttributesIfExists', () => { + const topicName = 'test-update-attributes-topic' + const subscriptionName = 'test-update-attributes-subscription' + + afterEach(async () => { + await deletePubSubTopicAndSubscription(pubSubClient, topicName, subscriptionName) + }) + + it('creates topic with options on first init', async () => { + const result = await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + options: { + messageRetentionDuration: { + seconds: 86400, // 1 day + }, + }, + }, + updateAttributesIfExists: false, + }) + + expect(result.topicName).toBe(topicName) + expect(result.topic).toBeDefined() + + // Verify topic was created with retention + const [metadata] = await result.topic.getMetadata() + expect(metadata.messageRetentionDuration).toBeDefined() + }, 10000) + + it('updates existing topic attributes when updateAttributesIfExists is true', async () => { + // Create topic first + await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + options: { + messageRetentionDuration: { + seconds: 86400, // 1 day + }, + }, + }, + }) + + // Update with different retention + const result = await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + options: { + messageRetentionDuration: { + seconds: 172800, // 2 days + }, + }, + }, + updateAttributesIfExists: true, + }) + + expect(result.topicName).toBe(topicName) + + // Verify topic was updated + const [metadata] = await result.topic.getMetadata() + expect(metadata.messageRetentionDuration?.seconds).toBe('172800') + }) + + it('does not update topic attributes when updateAttributesIfExists is false', async () => { + // Create topic first + await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + options: { + messageRetentionDuration: { + seconds: 86400, // 1 day + }, + }, + }, + }) + + const [originalMetadata] = await pubSubClient.topic(topicName).getMetadata() + + // Try to init again with different options but updateAttributesIfExists false + await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + options: { + messageRetentionDuration: { + seconds: 172800, // 2 days + }, + }, + }, + updateAttributesIfExists: false, + }) + + // Verify topic was NOT updated + const [currentMetadata] = await pubSubClient.topic(topicName).getMetadata() + expect(currentMetadata.messageRetentionDuration).toEqual( + originalMetadata.messageRetentionDuration, + ) + }) + + it('updates existing subscription attributes when updateAttributesIfExists is true', async () => { + // Create topic and subscription first + await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + options: { + ackDeadlineSeconds: 10, + }, + }, + }) + + // Update with different ackDeadlineSeconds + const result = await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + options: { + ackDeadlineSeconds: 30, + }, + }, + updateAttributesIfExists: true, + }) + + expect(result.subscription).toBeDefined() + + // Verify subscription was updated + const [metadata] = await result.subscription!.getMetadata() + expect(metadata.ackDeadlineSeconds).toBe(30) + }) + + it('does not update subscription attributes when updateAttributesIfExists is false', async () => { + // Create topic and subscription first + await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + options: { + ackDeadlineSeconds: 10, + }, + }, + }) + + // Try to init again with different options but updateAttributesIfExists false + await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + options: { + ackDeadlineSeconds: 30, + }, + }, + updateAttributesIfExists: false, + }) + + // Verify subscription was NOT updated + const subscription = pubSubClient.subscription(subscriptionName) + const [metadata] = await subscription.getMetadata() + expect(metadata.ackDeadlineSeconds).toBe(10) + }) + }) + + describe('deletion behavior', () => { + const topicName = 'test-deletion-topic' + const subscriptionName = 'test-deletion-subscription' + const originalNodeEnv = process.env.NODE_ENV + + beforeEach(async () => { + // Create a test topic and subscription + await initPubSub(pubSubClient, undefined, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + }, + }) + }) + + afterEach(async () => { + // Restore original NODE_ENV + if (originalNodeEnv !== undefined) { + process.env.NODE_ENV = originalNodeEnv + } else { + delete process.env.NODE_ENV + } + + // Clean up resources if they still exist + try { + await deletePubSubTopicAndSubscription(pubSubClient, topicName, subscriptionName) + } catch { + // Ignore errors - resources might already be deleted + } + }) + + it('throws error when deleting in production without forceDeleteInProduction flag', async () => { + process.env.NODE_ENV = 'production' + reloadConfig() // Reload config to pick up env change + + const deletionConfig: DeletionConfig = { + deleteIfExists: true, + forceDeleteInProduction: false, + } + + await expect( + deletePubSub(pubSubClient, deletionConfig, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + }, + }), + ).rejects.toThrow(/autodeletion in production/) + }) + + it('deletes resources with waitForConfirmation true (default)', async () => { + process.env.NODE_ENV = 'development' + reloadConfig() // Reload config to pick up env change + + const deletionConfig: DeletionConfig = { + deleteIfExists: true, + waitForConfirmation: true, // Explicitly set to true + } + + await deletePubSub(pubSubClient, deletionConfig, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + }, + }) + + // Verify both resources are deleted + const topic = pubSubClient.topic(topicName) + const subscription = pubSubClient.subscription(subscriptionName) + + const [topicExists] = await topic.exists() + const [subscriptionExists] = await subscription.exists() + + expect(topicExists).toBe(false) + expect(subscriptionExists).toBe(false) + }) + + it('deletes resources with waitForConfirmation false', async () => { + process.env.NODE_ENV = 'development' + reloadConfig() // Reload config to pick up env change + + const deletionConfig: DeletionConfig = { + deleteIfExists: true, + waitForConfirmation: false, // Don't wait for confirmation + } + + await deletePubSub(pubSubClient, deletionConfig, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + }, + }) + + // Give it a moment since we're not waiting for confirmation + await new Promise((resolve) => setTimeout(resolve, 100)) + + // Verify resources are eventually deleted (may take a bit longer) + const topic = pubSubClient.topic(topicName) + const subscription = pubSubClient.subscription(subscriptionName) + + const [topicExists] = await topic.exists() + const [subscriptionExists] = await subscription.exists() + + expect(topicExists).toBe(false) + expect(subscriptionExists).toBe(false) + }) + + it('allows deletion in production when forceDeleteInProduction is true', async () => { + process.env.NODE_ENV = 'production' + reloadConfig() // Reload config to pick up env change + + const deletionConfig: DeletionConfig = { + deleteIfExists: true, + forceDeleteInProduction: true, + } + + await deletePubSub(pubSubClient, deletionConfig, { + topic: { + name: topicName, + }, + subscription: { + name: subscriptionName, + }, + }) + + // Verify resources were deleted + const topic = pubSubClient.topic(topicName) + const subscription = pubSubClient.subscription(subscriptionName) + + const [topicExists] = await topic.exists() + const [subscriptionExists] = await subscription.exists() + + expect(topicExists).toBe(false) + expect(subscriptionExists).toBe(false) + }) + }) + + describe('config validation', () => { + it('throws error when both locatorConfig and creationConfig are provided', async () => { + await expect( + initPubSub(pubSubClient, { topicName: 'some-topic' }, { topic: { name: 'some-topic' } }), + ).rejects.toThrow(/Cannot provide both/) + }) + + it('throws error when neither locatorConfig nor creationConfig are provided', async () => { + await expect(initPubSub(pubSubClient, undefined, undefined)).rejects.toThrow( + /Either locatorConfig or creationConfig must be provided/, + ) + }) + }) +}) diff --git a/packages/gcp-pubsub/test/utils/testContext.ts b/packages/gcp-pubsub/test/utils/testContext.ts new file mode 100644 index 00000000..5259bf96 --- /dev/null +++ b/packages/gcp-pubsub/test/utils/testContext.ts @@ -0,0 +1,148 @@ +import { PubSub } from '@google-cloud/pubsub' +import { Storage } from '@google-cloud/storage' +import type { CommonLogger, ErrorReporter, ErrorResolver } from '@lokalise/node-core' +import type { + MessageMetricsManager, + TransactionObservabilityManager, +} from '@message-queue-toolkit/core' +import type { Resolver } from 'awilix' +import { asClass, asFunction, createContainer, Lifetime } from 'awilix' +import { AwilixManager } from 'awilix-manager' +import { Redis } from 'ioredis' +import { PubSubConsumerErrorResolver } from '../../lib/errors/PubSubConsumerErrorResolver.ts' +import { PubSubPermissionConsumer } from '../consumers/PubSubPermissionConsumer.ts' +import { PubSubPermissionPublisher } from '../publishers/PubSubPermissionPublisher.ts' +import { TEST_PUBSUB_CONFIG } from './testPubSubConfig.ts' +import { TEST_REDIS_CONFIG } from './testRedisConfig.ts' + +export const SINGLETON_CONFIG = { lifetime: Lifetime.SINGLETON } + +export type DependencyOverrides = Partial + +// @ts-expect-error +const TestLogger: CommonLogger = console + +export async function registerDependencies(dependencyOverrides: DependencyOverrides = {}) { + const diContainer = createContainer({ + injectionMode: 'PROXY', + }) + const awilixManager = new AwilixManager({ + diContainer, + asyncDispose: true, + asyncInit: true, + eagerInject: true, + }) + + const diConfig: DiConfig = { + logger: asFunction(() => { + return TestLogger + }, SINGLETON_CONFIG), + awilixManager: asFunction(() => { + return awilixManager + }, SINGLETON_CONFIG), + + pubSubClient: asFunction( + () => { + return new PubSub({ + projectId: TEST_PUBSUB_CONFIG.projectId, + apiEndpoint: TEST_PUBSUB_CONFIG.apiEndpoint, + }) + }, + { + lifetime: Lifetime.SINGLETON, + }, + ), + + gcsStorage: asFunction(() => { + return new Storage({ + projectId: 'test-project', + apiEndpoint: 'http://127.0.0.1:4443', + }) + }), + + consumerErrorResolver: asFunction(() => { + return new PubSubConsumerErrorResolver() + }), + + redis: asFunction( + () => { + const redisConfig = TEST_REDIS_CONFIG + + return new Redis({ + host: redisConfig.host, + db: redisConfig.db, + port: redisConfig.port, + username: redisConfig.username, + password: redisConfig.password, + connectTimeout: redisConfig.connectTimeout, + commandTimeout: redisConfig.commandTimeout, + tls: redisConfig.useTls ? {} : undefined, + maxRetriesPerRequest: null, + lazyConnect: true, + }) + }, + { + asyncInitPriority: 0, + asyncInit: 'connect', + dispose: (redis) => { + return new Promise((resolve) => { + void redis.quit((_err, result) => { + return resolve(result) + }) + }) + }, + lifetime: Lifetime.SINGLETON, + }, + ), + + permissionConsumer: asClass(PubSubPermissionConsumer, { + lifetime: Lifetime.SINGLETON, + asyncInit: 'start', + asyncDispose: 'close', + asyncDisposePriority: 10, + }), + permissionPublisher: asClass(PubSubPermissionPublisher, { + lifetime: Lifetime.SINGLETON, + asyncInit: 'init', + asyncDispose: 'close', + asyncDisposePriority: 20, + }), + + transactionObservabilityManager: asFunction(() => { + return undefined + }, SINGLETON_CONFIG), + messageMetricsManager: asFunction(() => undefined, SINGLETON_CONFIG), + errorReporter: asFunction(() => { + return { + report: () => {}, + } satisfies ErrorReporter + }), + } + diContainer.register(diConfig) + + for (const [dependencyKey, dependencyValue] of Object.entries(dependencyOverrides)) { + diContainer.register(dependencyKey, dependencyValue) + } + + await awilixManager.executeInit() + + return diContainer +} + +type DiConfig = Record> + +export interface Dependencies { + logger: CommonLogger + pubSubClient: PubSub + gcsStorage: Storage + awilixManager: AwilixManager + redis: Redis + + transactionObservabilityManager: TransactionObservabilityManager + messageMetricsManager: MessageMetricsManager + + errorReporter: ErrorReporter + consumerErrorResolver: ErrorResolver + permissionConsumer: PubSubPermissionConsumer + permissionPublisher: PubSubPermissionPublisher +} diff --git a/packages/gcp-pubsub/test/utils/testPubSubConfig.ts b/packages/gcp-pubsub/test/utils/testPubSubConfig.ts new file mode 100644 index 00000000..40effb56 --- /dev/null +++ b/packages/gcp-pubsub/test/utils/testPubSubConfig.ts @@ -0,0 +1,4 @@ +export const TEST_PUBSUB_CONFIG = { + projectId: 'test-project', + apiEndpoint: 'localhost:8085', +} diff --git a/packages/gcp-pubsub/test/utils/testRedisConfig.ts b/packages/gcp-pubsub/test/utils/testRedisConfig.ts new file mode 100644 index 00000000..383f6665 --- /dev/null +++ b/packages/gcp-pubsub/test/utils/testRedisConfig.ts @@ -0,0 +1,10 @@ +export const TEST_REDIS_CONFIG = { + host: 'localhost', + port: 6379, + db: 0, + username: undefined, + password: 'sOmE_sEcUrE_pAsS', + useTls: false, + connectTimeout: 5000, + commandTimeout: 5000, +} diff --git a/packages/gcp-pubsub/tsconfig.build.json b/packages/gcp-pubsub/tsconfig.build.json new file mode 100644 index 00000000..1b3cbf11 --- /dev/null +++ b/packages/gcp-pubsub/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": ["./tsconfig.json", "@lokalise/tsconfig/build-public-lib"], + "include": ["lib/**/*"], + "exclude": ["lib/**/*.spec.ts", "lib/**/*.test.ts"] +} diff --git a/packages/gcp-pubsub/tsconfig.json b/packages/gcp-pubsub/tsconfig.json new file mode 100644 index 00000000..a6868075 --- /dev/null +++ b/packages/gcp-pubsub/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@lokalise/tsconfig/tsc", + "include": ["lib/**/*", "test/**/*", "vitest.config.ts"], + "compilerOptions": { + "types": ["vitest/globals"] + } +} diff --git a/packages/gcp-pubsub/vitest.config.ts b/packages/gcp-pubsub/vitest.config.ts new file mode 100644 index 00000000..57512a80 --- /dev/null +++ b/packages/gcp-pubsub/vitest.config.ts @@ -0,0 +1,25 @@ +import { defineConfig } from 'vitest/config' + +// biome-ignore lint/style/noDefaultExport: vite expects default export +export default defineConfig({ + test: { + globals: true, + watch: false, + restoreMocks: true, + pool: 'threads', + poolOptions: { + threads: { singleThread: true }, + }, + coverage: { + provider: 'v8', + include: ['lib/**/*.ts'], + exclude: ['vitest.config.ts', 'lib/**/index.ts'], + thresholds: { + lines: 66, + functions: 87, + branches: 62, + statements: 66, + }, + }, + }, +}) diff --git a/packages/gcs-payload-store/README.md b/packages/gcs-payload-store/README.md new file mode 100644 index 00000000..5f16f496 --- /dev/null +++ b/packages/gcs-payload-store/README.md @@ -0,0 +1,345 @@ +# @message-queue-toolkit/gcs-payload-store + +Google Cloud Storage-based payload store implementation for message-queue-toolkit. Enables offloading large message payloads to GCS to comply with message size limitations in queue systems. + +## Overview + +This package provides a GCS-based implementation of the `PayloadStoreTypes` interface, allowing you to automatically store large message payloads in Google Cloud Storage while keeping only a reference pointer in the actual message. + +This is particularly useful when: +- Message payloads exceed queue system limits (e.g., 256 KB for SQS, 10 MB for Pub/Sub) +- You want to reduce message processing costs by offloading large data +- You need to handle variable-sized payloads efficiently + +## Installation + +```bash +npm install @message-queue-toolkit/gcs-payload-store @google-cloud/storage +``` + +## Prerequisites + +- Google Cloud Platform account +- GCS bucket for payload storage +- Appropriate IAM permissions for GCS access + +## Basic Usage + +### Configuration + +```typescript +import { Storage } from '@google-cloud/storage' +import { GCSPayloadStore } from '@message-queue-toolkit/gcs-payload-store' +import { AbstractSqsPublisher } from '@message-queue-toolkit/sqs' + +const storage = new Storage({ + projectId: 'my-project', + keyFilename: '/path/to/credentials.json', +}) + +class MyPublisher extends AbstractSqsPublisher { + constructor() { + super(dependencies, { + // ... other options + payloadStoreConfig: { + store: new GCSPayloadStore( + { gcsStorage: storage }, + { + bucketName: 'my-payload-bucket', + keyPrefix: 'message-payloads', // optional + } + ), + messageSizeThreshold: 256 * 1024, // 256 KB + }, + }) + } +} +``` + +### Using the Helper Function + +```typescript +import { resolvePayloadStoreConfig } from '@message-queue-toolkit/gcs-payload-store' + +const payloadStoreConfig = resolvePayloadStoreConfig( + { gcsStorage: storage }, + { + gcsPayloadOffloadingBucket: 'my-payload-bucket', + messageSizeThreshold: 256 * 1024, + } +) + +// Returns undefined if bucket not configured +// Throws error if storage client not provided +``` + +## Configuration Options + +### GCSPayloadStoreConfiguration + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `bucketName` | `string` | ✅ | GCS bucket name for storing payloads | +| `keyPrefix` | `string` | ❌ | Optional prefix for all stored keys (useful for organizing payloads) | + +### PayloadStoreConfig + +| Option | Type | Required | Description | +|--------|------|----------|-------------| +| `store` | `PayloadStoreTypes` | ✅ | Instance of `GCSPayloadStore` | +| `messageSizeThreshold` | `number` | ✅ | Size threshold in bytes - payloads exceeding this will be offloaded | +| `serializer` | `PayloadSerializer` | ❌ | Custom payload serializer (defaults to JSON) | + +## How It Works + +### Publishing Flow + +1. **Message size check**: When publishing, the toolkit calculates message size +2. **Offload decision**: If size exceeds `messageSizeThreshold`, payload is offloaded +3. **Store in GCS**: Payload is serialized and stored in GCS with a UUID key +4. **Publish pointer**: Only a small pointer object is sent through the queue: + ```typescript + { + offloadedPayloadPointer: "prefix/uuid-key", + offloadedPayloadSize: 1234567, + // ... message metadata (id, type, timestamp, etc.) + } + ``` + +### Consumption Flow + +1. **Detect pointer**: Consumer detects the offloaded payload pointer +2. **Retrieve from GCS**: Payload is retrieved from GCS using the pointer +3. **Deserialize**: Payload is deserialized back to original format +4. **Process normally**: Message handler receives the full payload + +## Lifecycle Management + +**Important**: Payloads are **not automatically deleted** after message processing. + +### Why Not Auto-Delete? + +1. **Fan-out complexity**: With multiple consumers, tracking when all have processed is difficult +2. **DLQ scenarios**: Messages sent to dead letter queues still reference payloads +3. **Retry scenarios**: Failed messages may be retried and need the payload + +### Recommended Approach: GCS Lifecycle Policies + +Set up GCS lifecycle rules to automatically delete old payloads: + +```bash +# Using gcloud CLI +gcloud storage buckets update gs://my-payload-bucket \ + --lifecycle-file=lifecycle.json +``` + +**lifecycle.json**: +```json +{ + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 7, + "matchesPrefix": ["message-payloads/"] + } + } + ] + } +} +``` + +This deletes payloads older than 7 days, regardless of whether they've been consumed. + +## Testing with Emulator + +### Using fake-gcs-server + +The package includes support for testing with the GCS emulator: + +```bash +# Start emulator (included in docker-compose) +docker compose up -d gcs-emulator +``` + +**Test configuration**: +```typescript +import { Storage } from '@google-cloud/storage' + +const storage = new Storage({ + projectId: 'test-project', + apiEndpoint: 'http://127.0.0.1:4443', +}) + +const store = new GCSPayloadStore( + { gcsStorage: storage }, + { bucketName: 'test-bucket' } +) +``` + +## API Reference + +### GCSPayloadStore + +#### Constructor + +```typescript +new GCSPayloadStore( + dependencies: GCSPayloadStoreDependencies, + config: GCSPayloadStoreConfiguration +) +``` + +#### Methods + +**`storePayload(payload: SerializedPayload): Promise`** + +Stores a payload in GCS and returns a unique key. + +- **Parameters:** + - `payload.value`: `string | Readable` - The payload data + - `payload.size`: `number` - Size in bytes +- **Returns:** Promise resolving to the storage key + +**`retrievePayload(key: string): Promise`** + +Retrieves a previously stored payload. + +- **Parameters:** + - `key`: The storage key returned by `storePayload` +- **Returns:** Promise resolving to a Readable stream, or `null` if not found + +**`deletePayload(key: string): Promise`** + +Deletes a payload from storage. + +- **Parameters:** + - `key`: The storage key +- **Returns:** Promise that resolves when deletion is complete + +## Integration Examples + +### With SQS + +```typescript +import { Storage } from '@google-cloud/storage' +import { GCSPayloadStore } from '@message-queue-toolkit/gcs-payload-store' +import { AbstractSqsPublisher, SQS_MESSAGE_MAX_SIZE } from '@message-queue-toolkit/sqs' + +const storage = new Storage({ projectId: 'my-project' }) + +class LargeMessagePublisher extends AbstractSqsPublisher { + constructor(dependencies) { + super(dependencies, { + creationConfig: { + queue: { QueueName: 'large-messages' }, + }, + messageSchemas: [MY_MESSAGE_SCHEMA], + messageTypeField: 'type', + payloadStoreConfig: { + store: new GCSPayloadStore( + { gcsStorage: storage }, + { bucketName: 'sqs-large-payloads' } + ), + messageSizeThreshold: SQS_MESSAGE_MAX_SIZE, + }, + }) + } +} +``` + +### With Pub/Sub + +```typescript +import { Storage } from '@google-cloud/storage' +import { GCSPayloadStore } from '@message-queue-toolkit/gcs-payload-store' +import { AbstractPubSubPublisher, PUBSUB_MESSAGE_MAX_SIZE } from '@message-queue-toolkit/pubsub' + +const storage = new Storage({ projectId: 'my-project' }) + +class PubSubLargeMessagePublisher extends AbstractPubSubPublisher { + constructor(dependencies) { + super(dependencies, { + creationConfig: { + topic: { name: 'large-events' }, + }, + messageSchemas: [MY_MESSAGE_SCHEMA], + messageTypeField: 'type', + payloadStoreConfig: { + store: new GCSPayloadStore( + { gcsStorage: storage }, + { bucketName: 'pubsub-large-payloads', keyPrefix: 'events' } + ), + messageSizeThreshold: PUBSUB_MESSAGE_MAX_SIZE, + }, + }) + } +} +``` + +## Error Handling + +The GCSPayloadStore handles errors gracefully: + +- **Not found**: Returns `null` instead of throwing +- **Permission errors**: Thrown as-is for proper handling +- **Network errors**: Thrown as-is for retry logic + +```typescript +try { + const payload = await store.retrievePayload('some-key') + if (payload === null) { + // Payload not found - handle gracefully + } +} catch (error) { + // Permission or network error - log and alert +} +``` + +## Best Practices + +1. **Set appropriate thresholds**: Use queue-specific limits (e.g., `SQS_MESSAGE_MAX_SIZE`) +2. **Use key prefixes**: Organize payloads by message type or tenant +3. **Configure lifecycle policies**: Always set up automatic cleanup +4. **Monitor storage costs**: Track bucket size and set up alerts +5. **Use IAM roles**: Prefer IAM roles over service account keys in production +6. **Test with emulator**: Use fake-gcs-server for local development + +## Troubleshooting + +### Payloads not being deleted + +Set up GCS lifecycle policies. The store intentionally does not auto-delete to handle fan-out and retry scenarios. + +### Authentication errors + +Ensure your Storage client has proper credentials: +```typescript +const storage = new Storage({ + projectId: 'my-project', + keyFilename: '/path/to/service-account.json', +}) +``` + +### Bucket not found errors + +Ensure the bucket exists before using: +```bash +gsutil mb gs://my-payload-bucket +``` + +Or create programmatically: +```typescript +await storage.createBucket('my-payload-bucket') +``` + +## License + +MIT + +## Contributing + +Contributions are welcome! Please see the main [message-queue-toolkit repository](https://github.com/kibertoad/message-queue-toolkit) for contribution guidelines. diff --git a/packages/gcs-payload-store/lib/GCSPayloadStore.ts b/packages/gcs-payload-store/lib/GCSPayloadStore.ts new file mode 100644 index 00000000..e6fe6fdf --- /dev/null +++ b/packages/gcs-payload-store/lib/GCSPayloadStore.ts @@ -0,0 +1,120 @@ +import { randomUUID } from 'node:crypto' +import type { Readable } from 'node:stream' + +import type { Bucket, Storage } from '@google-cloud/storage' +import type { PayloadStoreTypes, SerializedPayload } from '@message-queue-toolkit/core' + +export type GCSAwareDependencies = { + gcsStorage?: Storage +} + +export type MessageQueuePayloadOffloadingConfig = { + gcsPayloadOffloadingBucket?: string + messageSizeThreshold: number +} + +export function resolvePayloadStoreConfig( + dependencies: GCSAwareDependencies, + config?: MessageQueuePayloadOffloadingConfig, +) { + if (!config?.gcsPayloadOffloadingBucket) return undefined + if (!dependencies.gcsStorage) { + throw new Error('Google Cloud Storage client is required for payload offloading') + } + + return { + store: new GCSPayloadStore( + { gcsStorage: dependencies.gcsStorage }, + { bucketName: config.gcsPayloadOffloadingBucket }, + ), + messageSizeThreshold: config.messageSizeThreshold, + } +} + +export type GCSPayloadStoreDependencies = { + gcsStorage: Storage +} + +export type GCSPayloadStoreConfiguration = { + bucketName: string + keyPrefix?: string +} + +export class GCSPayloadStore implements PayloadStoreTypes { + private readonly storage: Storage + private readonly bucket: Bucket + private readonly config: GCSPayloadStoreConfiguration + + constructor({ gcsStorage }: GCSPayloadStoreDependencies, config: GCSPayloadStoreConfiguration) { + this.storage = gcsStorage + this.bucket = this.storage.bucket(config.bucketName) + this.config = config + } + + async storePayload(payload: SerializedPayload): Promise { + const id = randomUUID() + const key = this.config?.keyPrefix?.length ? `${this.config.keyPrefix}/${id}` : id + + const file = this.bucket.file(key) + + // Handle both string and stream payloads + if (typeof payload.value === 'string') { + await file.save(payload.value, { + metadata: { + contentLength: payload.size, + }, + }) + } else { + // Stream + await new Promise((resolve, reject) => { + const writeStream = file.createWriteStream({ + metadata: { + contentLength: payload.size, + }, + }) + + ;(payload.value as Readable) + .pipe(writeStream) + .on('finish', () => resolve()) + .on('error', reject) + }) + } + + return key + } + + async retrievePayload(key: string): Promise { + try { + const file = this.bucket.file(key) + const [exists] = await file.exists() + + if (!exists) { + return null + } + + return file.createReadStream() + } catch (error) { + // Check if it's a not-found error (404) + // biome-ignore lint/suspicious/noExplicitAny: error type is unknown + if ((error as any)?.code === 404) { + return null + } + throw error + } + } + + async deletePayload(key: string): Promise { + try { + const file = this.bucket.file(key) + await file.delete({ ignoreNotFound: true }) + } catch (error) { + // Gracefully handle 404 errors (file already deleted or never existed) + // biome-ignore lint/suspicious/noExplicitAny: error type is unknown + if ((error as any)?.code === 404) { + return + } + // Re-throw other errors + throw error + } + } +} diff --git a/packages/gcs-payload-store/lib/index.ts b/packages/gcs-payload-store/lib/index.ts new file mode 100644 index 00000000..e1848c9e --- /dev/null +++ b/packages/gcs-payload-store/lib/index.ts @@ -0,0 +1 @@ +export * from './GCSPayloadStore.ts' diff --git a/packages/gcs-payload-store/package.json b/packages/gcs-payload-store/package.json new file mode 100644 index 00000000..59152d5d --- /dev/null +++ b/packages/gcs-payload-store/package.json @@ -0,0 +1,66 @@ +{ + "name": "@message-queue-toolkit/gcs-payload-store", + "version": "1.0.0", + "private": false, + "license": "MIT", + "description": "Google Cloud Storage-based message store implementation for message-queue-toolkit", + "maintainers": [ + { + "name": "Igor Savin", + "email": "kibertoad@gmail.com" + } + ], + "type": "module", + "main": "./dist/index.js", + "exports": { + ".": "./dist/index.js", + "./package.json": "./package.json" + }, + "scripts": { + "build": "npm run clean && tsc --project tsconfig.build.json", + "clean": "rimraf dist", + "test": "vitest", + "test:coverage": "npm run test -- --coverage", + "lint": "biome check . && tsc", + "lint:fix": "biome check --write .", + "docker:start": "docker compose up -d gcs-emulator", + "docker:stop": "docker compose down", + "prepublishOnly": "npm run lint && npm run build" + }, + "dependencies": {}, + "peerDependencies": { + "@google-cloud/storage": "^7.17.0", + "@message-queue-toolkit/core": ">=21.0.0" + }, + "devDependencies": { + "@google-cloud/storage": "^7.17.2", + "@message-queue-toolkit/core": "*", + "@biomejs/biome": "^2.2.0", + "@lokalise/biome-config": "^3.1.0", + "@lokalise/tsconfig": "^3.0.0", + "@types/node": "^24.0.3", + "@vitest/coverage-v8": "^3.2.4", + "rimraf": "^6.0.1", + "typescript": "^5.9.2", + "vitest": "^3.2.4" + }, + "homepage": "https://github.com/kibertoad/message-queue-toolkit", + "repository": { + "type": "git", + "url": "git://github.com/kibertoad/message-queue-toolkit.git" + }, + "keywords": [ + "message", + "queue", + "storage", + "gcs", + "google-cloud-storage", + "payload-offloading", + "claim-check" + ], + "files": [ + "README.md", + "LICENSE", + "dist/*" + ] +} diff --git a/packages/gcs-payload-store/test/store/GCSPayloadStore.spec.ts b/packages/gcs-payload-store/test/store/GCSPayloadStore.spec.ts new file mode 100644 index 00000000..e9a211e3 --- /dev/null +++ b/packages/gcs-payload-store/test/store/GCSPayloadStore.spec.ts @@ -0,0 +1,145 @@ +import { Readable } from 'node:stream' + +import type { Storage } from '@google-cloud/storage' +import { beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { GCSPayloadStore, resolvePayloadStoreConfig } from '../../lib/GCSPayloadStore.ts' +import { assertEmptyBucket, getObjectContent, objectExists } from '../utils/gcsUtils.ts' +import { streamToString } from '../utils/streamUtils.ts' +import { createTestGCSClient } from '../utils/testGCSConfig.ts' + +const TEST_BUCKET = 'test-bucket' + +describe('GCSPayloadStore', () => { + let storage: Storage + let store: GCSPayloadStore + + beforeAll(() => { + storage = createTestGCSClient() + store = new GCSPayloadStore({ gcsStorage: storage }, { bucketName: TEST_BUCKET }) + }) + + beforeEach(async () => { + await assertEmptyBucket(storage, TEST_BUCKET) + }) + + describe('storePayload', () => { + it('stores string payload in the bucket', async () => { + const payload = 'test' + + const stringPayloadKey = await store.storePayload({ + value: payload, + size: payload.length, + }) + + expect(await getObjectContent(storage, TEST_BUCKET, stringPayloadKey)).toBe(payload) + }) + + it('stores stream payload in the bucket', async () => { + const payload = 'test stream content' + + const streamPayloadKey = await store.storePayload({ + value: Readable.from(payload), + size: payload.length, + }) + + expect(await getObjectContent(storage, TEST_BUCKET, streamPayloadKey)).toBe(payload) + }) + + it('uses key prefix if provided', async () => { + const prefixedStore = new GCSPayloadStore( + { gcsStorage: storage }, + { bucketName: TEST_BUCKET, keyPrefix: 'prefix' }, + ) + const payload = 'test' + + const stringPayloadKey = await prefixedStore.storePayload({ + value: payload, + size: payload.length, + }) + + expect(stringPayloadKey).toContain('prefix/') + expect(await getObjectContent(storage, TEST_BUCKET, stringPayloadKey)).toBe(payload) + }) + }) + + describe('retrievePayload', () => { + it('retrieves previously stored payload', async () => { + const payload = 'test retrieval content' + const key = await store.storePayload({ + value: Readable.from(payload), + size: payload.length, + }) + + const result = await store.retrievePayload(key) + + expect(result).toBeInstanceOf(Readable) + await expect(streamToString(result!)).resolves.toBe(payload) + }) + + it('returns null if payload cannot be found', async () => { + const result = await store.retrievePayload('non-existing-key') + expect(result).toBe(null) + }) + + it('throws if other than not-found error occurs', async () => { + const invalidStore = new GCSPayloadStore( + { gcsStorage: storage }, + { bucketName: 'non-existing-bucket' }, + ) + await expect(invalidStore.retrievePayload('some-key')).rejects.toThrow() + }) + }) + + describe('deletePayload', () => { + it('successfully deletes previously stored payload', async () => { + const payload = 'test deletion content' + const key = await store.storePayload({ + value: Readable.from(payload), + size: payload.length, + }) + await expect(objectExists(storage, TEST_BUCKET, key)).resolves.toBeTruthy() + + await store.deletePayload(key) + + await expect(objectExists(storage, TEST_BUCKET, key)).resolves.toBeFalsy() + }) + + it('gracefully handles non-existing key', async () => { + await expect(store.deletePayload('non-existing-key')).resolves.not.toThrow() + }) + }) + + describe('resolvePayloadStoreConfig', () => { + it('should return undefined if gcsPayloadOffloadingBucket is not set', () => { + const result = resolvePayloadStoreConfig({ gcsStorage: {} as any }) + expect(result).toBeUndefined() + }) + + it('should throw an error if GCS storage client is not defined', () => { + expect(() => + resolvePayloadStoreConfig( + { gcsStorage: undefined }, + { + gcsPayloadOffloadingBucket: 'test-bucket', + messageSizeThreshold: 1, + }, + ), + ).toThrowError('Google Cloud Storage client is required for payload offloading') + }) + + it('should return payload store config', () => { + const result = resolvePayloadStoreConfig( + { gcsStorage: {} as any }, + { + gcsPayloadOffloadingBucket: 'test-bucket', + messageSizeThreshold: 1, + }, + ) + expect(result).toEqual({ + store: expect.any(GCSPayloadStore), + messageSizeThreshold: 1, + }) + }) + }) +}) diff --git a/packages/gcs-payload-store/test/utils/gcsUtils.ts b/packages/gcs-payload-store/test/utils/gcsUtils.ts new file mode 100644 index 00000000..1aaacd01 --- /dev/null +++ b/packages/gcs-payload-store/test/utils/gcsUtils.ts @@ -0,0 +1,49 @@ +import type { Bucket, Storage } from '@google-cloud/storage' + +export async function assertEmptyBucket(storage: Storage, bucketName: string): Promise { + const bucket = storage.bucket(bucketName) + + // Create bucket if it doesn't exist + const [exists] = await bucket.exists() + if (!exists) { + await bucket.create() + return + } + + // Delete all files in the bucket + const [files] = await bucket.getFiles() + await Promise.all(files.map((file) => file.delete({ ignoreNotFound: true }))) +} + +export async function getObjectContent( + storage: Storage, + bucketName: string, + key: string, +): Promise { + const bucket = storage.bucket(bucketName) + const file = bucket.file(key) + const [content] = await file.download() + return content.toString() +} + +export async function objectExists( + storage: Storage, + bucketName: string, + key: string, +): Promise { + const bucket = storage.bucket(bucketName) + const file = bucket.file(key) + const [exists] = await file.exists() + return exists +} + +export async function ensureBucket(storage: Storage, bucketName: string): Promise { + const bucket = storage.bucket(bucketName) + const [exists] = await bucket.exists() + + if (!exists) { + await bucket.create() + } + + return bucket +} diff --git a/packages/gcs-payload-store/test/utils/streamUtils.ts b/packages/gcs-payload-store/test/utils/streamUtils.ts new file mode 100644 index 00000000..60bec38f --- /dev/null +++ b/packages/gcs-payload-store/test/utils/streamUtils.ts @@ -0,0 +1,10 @@ +import type { Readable } from 'node:stream' + +export function streamToString(stream: Readable): Promise { + const chunks: Buffer[] = [] + return new Promise((resolve, reject) => { + stream.on('data', (chunk: Buffer) => chunks.push(chunk)) + stream.on('error', reject) + stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))) + }) +} diff --git a/packages/gcs-payload-store/test/utils/testGCSConfig.ts b/packages/gcs-payload-store/test/utils/testGCSConfig.ts new file mode 100644 index 00000000..826ee1a4 --- /dev/null +++ b/packages/gcs-payload-store/test/utils/testGCSConfig.ts @@ -0,0 +1,13 @@ +import { Storage } from '@google-cloud/storage' + +export const TEST_GCS_CONFIG = { + projectId: 'test-project', + apiEndpoint: 'http://127.0.0.1:4443', +} + +export function createTestGCSClient(): Storage { + return new Storage({ + projectId: TEST_GCS_CONFIG.projectId, + apiEndpoint: TEST_GCS_CONFIG.apiEndpoint, + }) +} diff --git a/packages/gcs-payload-store/tsconfig.build.json b/packages/gcs-payload-store/tsconfig.build.json new file mode 100644 index 00000000..1b3cbf11 --- /dev/null +++ b/packages/gcs-payload-store/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": ["./tsconfig.json", "@lokalise/tsconfig/build-public-lib"], + "include": ["lib/**/*"], + "exclude": ["lib/**/*.spec.ts", "lib/**/*.test.ts"] +} diff --git a/packages/gcs-payload-store/tsconfig.json b/packages/gcs-payload-store/tsconfig.json new file mode 100644 index 00000000..a6868075 --- /dev/null +++ b/packages/gcs-payload-store/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@lokalise/tsconfig/tsc", + "include": ["lib/**/*", "test/**/*", "vitest.config.ts"], + "compilerOptions": { + "types": ["vitest/globals"] + } +} diff --git a/packages/gcs-payload-store/vitest.config.ts b/packages/gcs-payload-store/vitest.config.ts new file mode 100644 index 00000000..ed945528 --- /dev/null +++ b/packages/gcs-payload-store/vitest.config.ts @@ -0,0 +1,25 @@ +import { defineConfig } from 'vitest/config' + +// biome-ignore lint/style/noDefaultExport: vite expects default export +export default defineConfig({ + test: { + globals: true, + watch: false, + restoreMocks: true, + pool: 'threads', + poolOptions: { + threads: { singleThread: true }, + }, + coverage: { + provider: 'v8', + include: ['lib/**/*.ts'], + exclude: ['vitest.config.ts', 'lib/**/index.ts'], + thresholds: { + lines: 88, + functions: 100, + branches: 74, + statements: 88, + }, + }, + }, +})