From 0794af6066892b46d3e16faa0caca0a74006d22e Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Mon, 18 May 2026 17:30:26 +0200 Subject: [PATCH 01/23] feat: add zstd message compression codec for SNS and SQS --- packages/core/lib/codec/messageCodec.ts | 43 +++ packages/core/lib/index.ts | 8 + .../core/lib/queues/AbstractQueueService.ts | 3 + packages/core/lib/types/queueOptionsTypes.ts | 9 + packages/core/package.json | 1 + packages/sns/lib/sns/AbstractSnsPublisher.ts | 9 +- .../sns/lib/sns/AbstractSnsSqsConsumer.ts | 10 +- packages/sns/package.json | 8 +- .../SnsSqsPermissionConsumer.codec.spec.ts | 104 ++++++++ .../consumers/SnsSqsPermissionConsumer.ts | 2 + .../test/publishers/SnsPermissionPublisher.ts | 2 + packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 11 + packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 9 +- packages/sqs/package.json | 8 +- .../SqsPermissionConsumer.codec.spec.ts | 123 +++++++++ .../test/consumers/SqsPermissionConsumer.ts | 2 + .../test/publishers/SqsPermissionPublisher.ts | 2 + pnpm-lock.yaml | 252 ++++++++++++++---- pnpm-workspace.yaml | 1 + 19 files changed, 542 insertions(+), 65 deletions(-) create mode 100644 packages/core/lib/codec/messageCodec.ts create mode 100644 packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts create mode 100644 packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts new file mode 100644 index 00000000..4709a8de --- /dev/null +++ b/packages/core/lib/codec/messageCodec.ts @@ -0,0 +1,43 @@ +import { compress, decompress } from '@mongodb-js/zstd' + +export const SUPPORTED_CODECS = ['zstd'] as const +export type MessageCodec = (typeof SUPPORTED_CODECS)[number] + +const CODEC_FIELD = '__codec' +const DATA_FIELD = '__data' + +export type CodecEnvelope = { + [CODEC_FIELD]: MessageCodec + [DATA_FIELD]: string +} + +export function isCodecEnvelope(value: unknown): value is CodecEnvelope { + return ( + typeof value === 'object' && + value !== null && + CODEC_FIELD in value && + DATA_FIELD in value && + SUPPORTED_CODECS.includes((value as Record)[CODEC_FIELD] as MessageCodec) + ) +} + +export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { + if (codec === 'zstd') { + const compressed = await compress(Buffer.from(jsonBody, 'utf8')) + const envelope: CodecEnvelope = { + [CODEC_FIELD]: codec, + [DATA_FIELD]: compressed.toString('base64'), + } + return JSON.stringify(envelope) + } + throw new Error(`Unsupported codec: ${codec}`) +} + +export async function decompressMessageBody(envelope: CodecEnvelope): Promise { + if (envelope[CODEC_FIELD] === 'zstd') { + const compressed = Buffer.from(envelope[DATA_FIELD], 'base64') + const decompressed = await decompress(compressed) + return JSON.parse(decompressed.toString('utf8')) + } + throw new Error(`Unsupported codec: ${envelope[CODEC_FIELD]}`) +} diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index bb97ea60..97855fb7 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,3 +1,11 @@ +export { + type CodecEnvelope, + compressMessageBody, + decompressMessageBody, + isCodecEnvelope, + type MessageCodec, + SUPPORTED_CODECS, +} from './codec/messageCodec.ts' export { DoNotProcessMessageError } from './errors/DoNotProcessError.ts' export { isMessageError, diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 40fa618b..ca305d0c 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -14,6 +14,7 @@ import { } from '@message-queue-toolkit/schemas' import { getProperty, setProperty } from 'dot-prop' import type { ZodSchema, ZodType } from 'zod/v4' +import type { MessageCodec } from '../codec/messageCodec.ts' import type { MessageInvalidFormatError, MessageValidationError } from '../errors/Errors.ts' import { type AcquireLockTimeoutError, @@ -135,6 +136,7 @@ export abstract class AbstractQueueService< protected readonly messageDeduplicationConfig?: MessageDeduplicationConfig protected readonly messageMetricsManager?: MessageMetricsManager protected readonly _handlerSpy?: HandlerSpy + protected readonly codec?: MessageCodec protected isInitted: boolean @@ -172,6 +174,7 @@ export abstract class AbstractQueueService< } : undefined this.messageDeduplicationConfig = options.messageDeduplicationConfig + this.codec = options.codec this.logMessages = options.logMessages ?? false this._handlerSpy = resolveHandlerSpy(options) diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index d8f26445..3310ca5b 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -1,5 +1,6 @@ import type { CommonLogger, ErrorReporter, ErrorResolver } from '@lokalise/node-core' import type { ZodSchema } from 'zod/v4' +import type { MessageCodec } from '../codec/messageCodec.ts' import type { MessageDeduplicationConfig } from '../message-deduplication/messageDeduplicationTypes.ts' import type { PayloadStoreConfig } from '../payload-store/payloadStoreTypes.ts' import type { MessageHandlerConfig } from '../queues/HandlerContainer.ts' @@ -139,6 +140,14 @@ export type CommonQueueOptions = { deletionConfig?: DeletionConfig payloadStoreConfig?: PayloadStoreConfig messageDeduplicationConfig?: MessageDeduplicationConfig + /** + * Codec to use for compressing outgoing messages and decompressing incoming messages. + * When set on a publisher, messages are compressed before sending. + * When set on a consumer, it signals that incoming messages may be compressed. + * Compressed messages are self-describing (the codec is embedded in the message envelope), + * so consumers can decompress even without this option explicitly set. + */ + codec?: MessageCodec } export type CommonCreationConfigType = { diff --git a/packages/core/package.json b/packages/core/package.json index 041db97b..12b5e3be 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -28,6 +28,7 @@ "@lokalise/node-core": "^14.2.0", "@lokalise/universal-ts-utils": "^4.5.1", "@message-queue-toolkit/schemas": "^7.0.0", + "@mongodb-js/zstd": "^7.0.0", "dot-prop": "^10.1.0", "fast-equals": "^6.0.0", "json-stream-stringify": "^3.1.6", diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index f6c14ad4..6e9f95f7 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -5,7 +5,9 @@ import { InternalError } from '@lokalise/node-core' import { type AsyncPublisher, type BarrierResult, + compressMessageBody, DeduplicationRequesterEnum, + isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, type MessageSchemaContainer, type MessageValidationError, @@ -211,8 +213,13 @@ export abstract class AbstractSnsPublisher options: SNSMessageOptions, ): Promise { const attributes = resolveOutgoingMessageAttributes(payload) + const jsonBody = JSON.stringify(payload) + const body = + this.codec && !isOffloadedPayloadPointerPayload(payload) + ? await compressMessageBody(jsonBody, this.codec) + : jsonBody const command = new PublishCommand({ - Message: JSON.stringify(payload), + Message: body, MessageAttributes: attributes, TopicArn: this.topicArn, ...options, diff --git a/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts index 9e1113e1..37f2d857 100644 --- a/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts +++ b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts @@ -1,5 +1,11 @@ import type { SNSClient } from '@aws-sdk/client-sns' import type { STSClient } from '@aws-sdk/client-sts' +import type { Either } from '@lokalise/node-core' +import type { + MessageInvalidFormatError, + MessageValidationError, + ResolvedMessage, +} from '@message-queue-toolkit/core' import type { SQSConsumerDependencies, SQSConsumerOptions, @@ -201,7 +207,9 @@ export abstract class AbstractSnsSqsConsumer< await this.startConsumers() } - protected override resolveMessage(message: SQSMessage) { + protected override resolveMessage( + message: SQSMessage, + ): Either { const result = readSnsMessage(message, this.errorResolver) if (result.result) { return result diff --git a/packages/sns/package.json b/packages/sns/package.json index 06a5030e..f47833f4 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -47,10 +47,10 @@ "@biomejs/biome": "^2.3.6", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/core": "*", - "@message-queue-toolkit/redis-message-deduplication-store": "*", - "@message-queue-toolkit/s3-payload-store": "*", - "@message-queue-toolkit/sqs": "*", + "@message-queue-toolkit/core": "workspace:*", + "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", + "@message-queue-toolkit/s3-payload-store": "workspace:*", + "@message-queue-toolkit/sqs": "workspace:*", "@types/node": "^25.0.2", "@vitest/coverage-v8": "^4.0.15", "awilix": "^13.0.3", diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts new file mode 100644 index 00000000..72611d4a --- /dev/null +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts @@ -0,0 +1,104 @@ +import type { AwilixContainer } from 'awilix' +import { asValue } from 'awilix' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { SnsPermissionPublisher } from '../publishers/SnsPermissionPublisher.ts' +import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' +import type { Dependencies } from '../utils/testContext.ts' +import { registerDependencies } from '../utils/testContext.ts' +import { SnsSqsPermissionConsumer } from './SnsSqsPermissionConsumer.ts' +import type { PERMISSIONS_ADD_MESSAGE_TYPE } from './userConsumerSchemas.ts' + +describe('SnsSqsPermissionConsumer - zstd codec', () => { + let diContainer: AwilixContainer + let testAdmin: TestAwsResourceAdmin + let publisher: SnsPermissionPublisher + let consumer: SnsSqsPermissionConsumer + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + testAdmin = diContainer.cradle.testAdmin + }) + + beforeEach(async () => { + await testAdmin.deleteQueues(SnsSqsPermissionConsumer.CONSUMED_QUEUE_NAME) + await testAdmin.deleteTopics(SnsSqsPermissionConsumer.SUBSCRIBED_TOPIC_NAME) + + consumer = new SnsSqsPermissionConsumer(diContainer.cradle, { + codec: 'zstd', + }) + publisher = new SnsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + }) + + await consumer.start() + await publisher.init() + }) + + afterEach(async () => { + await publisher.close() + await consumer.close() + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('publishes a compressed SNS message and consumer decompresses it correctly', async () => { + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'sns-codec-test-1', + messageType: 'add', + metadata: { info: 'hello sns zstd' }, + } + + await publisher.publish(message) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + }, 15000) + + it('consumer correctly handles multiple compressed SNS messages in sequence', async () => { + const messages: PERMISSIONS_ADD_MESSAGE_TYPE[] = [ + { id: 'sns-codec-seq-1', messageType: 'add' }, + { id: 'sns-codec-seq-2', messageType: 'add' }, + { id: 'sns-codec-seq-3', messageType: 'add' }, + ] + + for (const msg of messages) { + await publisher.publish(msg) + } + + for (const msg of messages) { + const result = await consumer.handlerSpy.waitForMessageWithId(msg.id, 'consumed') + expect(result.message).toMatchObject(msg) + } + }, 15000) + + it('consumer without codec option auto-detects and decompresses zstd messages from SNS', async () => { + // Consumer without explicit codec — decompression is auto-detected from envelope __codec field + const autoConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { + locatorConfig: { + queueUrl: consumer.subscriptionProps.queueUrl, + topicArn: consumer.subscriptionProps.topicArn, + subscriptionArn: consumer.subscriptionProps.subscriptionArn, + }, + }) + await autoConsumer.start() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'sns-codec-auto-1', + messageType: 'add', + } + await publisher.publish(message) + + const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await autoConsumer.close() + }, 15000) +}) diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts index 3155b3fd..c4f6ab03 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts @@ -40,6 +40,7 @@ type SnsSqsPermissionConsumerOptions = Pick< | 'maxRetryDuration' | 'payloadStoreConfig' | 'concurrentConsumersAmount' + | 'codec' > & { addPreHandlerBarrier?: ( message: SupportedMessages, @@ -147,6 +148,7 @@ export class SnsSqsPermissionConsumer extends AbstractSnsSqsConsumer< deleteIfExists: false, }, payloadStoreConfig: options.payloadStoreConfig, + codec: options.codec, consumerOverrides: options.consumerOverrides ?? { terminateVisibilityTimeout: true, // this allows to retry failed messages immediately }, diff --git a/packages/sns/test/publishers/SnsPermissionPublisher.ts b/packages/sns/test/publishers/SnsPermissionPublisher.ts index e5c2dda7..659f8d87 100644 --- a/packages/sns/test/publishers/SnsPermissionPublisher.ts +++ b/packages/sns/test/publishers/SnsPermissionPublisher.ts @@ -26,6 +26,7 @@ export class SnsPermissionPublisher extends AbstractSnsPublisher | 'payloadStoreConfig' | 'messageDeduplicationConfig' | 'enablePublisherDeduplication' + | 'codec' >, ) { super(dependencies, { @@ -40,6 +41,7 @@ export class SnsPermissionPublisher extends AbstractSnsPublisher deleteIfExists: false, }, payloadStoreConfig: options?.payloadStoreConfig, + codec: options?.codec, messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], handlerSpy: true, messageTypeResolver: { messageTypePath: 'messageType' }, diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index 81a5c1b3..7ce99a76 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -10,7 +10,9 @@ import { type BarrierResult, type DeadLetterQueueOptions, DeduplicationRequesterEnum, + decompressMessageBody, HandlerContainer, + isCodecEnvelope, isMessageError, type MessageSchemaContainer, noopReleasableLock, @@ -891,6 +893,15 @@ export abstract class AbstractSqsConsumer< return ABORT_EARLY_EITHER } resolveMessageResult.result.body = retrieveOffloadedMessagePayloadResult.result + } else if (isCodecEnvelope(resolveMessageResult.result.body)) { + try { + resolveMessageResult.result.body = await decompressMessageBody( + resolveMessageResult.result.body, + ) + } catch (err) { + this.handleError(err as Error) + return ABORT_EARLY_EITHER + } } return resolveMessageResult diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index f4ed02d4..54a2e43e 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -5,7 +5,9 @@ import { InternalError } from '@lokalise/node-core' import { type AsyncPublisher, type BarrierResult, + compressMessageBody, DeduplicationRequesterEnum, + isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, type MessageSchemaContainer, type MessageValidationError, @@ -204,11 +206,16 @@ export abstract class AbstractSqsPublisher options: SQSMessageOptions, ): Promise { const attributes = resolveOutgoingMessageAttributes(payload) + const jsonBody = JSON.stringify(payload) + const body = + this.codec && !isOffloadedPayloadPointerPayload(payload) + ? await compressMessageBody(jsonBody, this.codec) + : jsonBody // Options are already resolved in publish() before offloading const command = new SendMessageCommand({ QueueUrl: this.queueUrl, - MessageBody: JSON.stringify(payload), + MessageBody: body, MessageAttributes: attributes, ...options, }) diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 0e19409c..b6c36306 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -42,10 +42,10 @@ "@biomejs/biome": "^2.3.8", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/core": "*", - "@message-queue-toolkit/redis-message-deduplication-store": "*", - "@message-queue-toolkit/s3-payload-store": "*", - "@message-queue-toolkit/schemas": "*", + "@message-queue-toolkit/core": "workspace:*", + "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", + "@message-queue-toolkit/s3-payload-store": "workspace:*", + "@message-queue-toolkit/schemas": "workspace:*", "@types/node": "^25.0.2", "@vitest/coverage-v8": "^4.0.15", "awilix": "^13.0.3", diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts new file mode 100644 index 00000000..042050e2 --- /dev/null +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -0,0 +1,123 @@ +import { SendMessageCommand } from '@aws-sdk/client-sqs' +import { compressMessageBody } from '@message-queue-toolkit/core' +import type { AwilixContainer } from 'awilix' +import { asValue } from 'awilix' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher.ts' +import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' +import type { Dependencies } from '../utils/testContext.ts' +import { registerDependencies } from '../utils/testContext.ts' +import { SqsPermissionConsumer } from './SqsPermissionConsumer.ts' +import type { PERMISSIONS_ADD_MESSAGE_TYPE } from './userConsumerSchemas.ts' + +describe('SqsPermissionConsumer - zstd codec', () => { + let diContainer: AwilixContainer + let testAdmin: TestAwsResourceAdmin + let publisher: SqsPermissionPublisher + let consumer: SqsPermissionConsumer + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + testAdmin = diContainer.cradle.testAdmin + }) + + beforeEach(async () => { + await testAdmin.deleteQueues(SqsPermissionConsumer.QUEUE_NAME) + + consumer = new SqsPermissionConsumer(diContainer.cradle, { + codec: 'zstd', + deletionConfig: { deleteIfExists: false }, + }) + publisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + }) + + await consumer.start() + await publisher.init() + }) + + afterEach(async () => { + await publisher.close() + await consumer.close(true) + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('publishes a compressed message and consumer decompresses it correctly', async () => { + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-test-1', + messageType: 'add', + metadata: { info: 'hello zstd' }, + } + + await publisher.publish(message) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + }) + + it('consumer correctly handles multiple compressed messages in sequence', async () => { + const messages: PERMISSIONS_ADD_MESSAGE_TYPE[] = [ + { id: 'codec-seq-1', messageType: 'add' }, + { id: 'codec-seq-2', messageType: 'add' }, + { id: 'codec-seq-3', messageType: 'add' }, + ] + + for (const msg of messages) { + await publisher.publish(msg) + } + + for (const msg of messages) { + const result = await consumer.handlerSpy.waitForMessageWithId(msg.id, 'consumed') + expect(result.message).toMatchObject(msg) + } + }) + + it('consumer decompresses a message compressed externally with zstd', async () => { + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-external-1', + messageType: 'add', + metadata: { source: 'external-compressor' }, + } + + // Simulate a publisher that compressed the message itself + const compressedBody = await compressMessageBody(JSON.stringify(message), 'zstd') + await diContainer.cradle.sqsClient.send( + new SendMessageCommand({ + QueueUrl: consumer.queueProps.url, + MessageBody: compressedBody, + }), + ) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + }) + + it('consumer without codec option still decompresses zstd messages (auto-detection)', async () => { + // Consumer without codec — auto-detects from envelope __codec field + const autoConsumer = new SqsPermissionConsumer(diContainer.cradle, { + locatorConfig: { queueUrl: consumer.queueProps.url }, + deletionConfig: { deleteIfExists: false }, + }) + await autoConsumer.start() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-auto-detect-1', + messageType: 'add', + } + await publisher.publish(message) + + const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await autoConsumer.close(true) + }) +}) diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.ts index 3155d507..cfadac40 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.ts @@ -31,6 +31,7 @@ type SqsPermissionConsumerOptions = Pick< | 'payloadStoreConfig' | 'messageDeduplicationConfig' | 'enableConsumerDeduplication' + | 'codec' > & { addPreHandlerBarrier?: ( message: SupportedMessages, @@ -126,6 +127,7 @@ export class SqsPermissionConsumer extends AbstractSqsConsumer< payloadStoreConfig: options.payloadStoreConfig, messageDeduplicationConfig: options.messageDeduplicationConfig, enableConsumerDeduplication: options.enableConsumerDeduplication, + codec: options.codec, messageDeduplicationIdField: 'deduplicationId', messageDeduplicationOptionsField: 'deduplicationOptions', handlers: new MessageHandlerConfigBuilder< diff --git a/packages/sqs/test/publishers/SqsPermissionPublisher.ts b/packages/sqs/test/publishers/SqsPermissionPublisher.ts index 33ab0c15..5aa73c20 100644 --- a/packages/sqs/test/publishers/SqsPermissionPublisher.ts +++ b/packages/sqs/test/publishers/SqsPermissionPublisher.ts @@ -31,6 +31,7 @@ export class SqsPermissionPublisher extends AbstractSqsPublisher, ) { super(dependencies, { @@ -53,6 +54,7 @@ export class SqsPermissionPublisher extends AbstractSqsPublisher=23.1.0' ioredis: ^5.3.2 - '@message-queue-toolkit/s3-payload-store@3.0.0': - resolution: {integrity: sha512-AX2PI74CN9CBQWHT/nJBhUPR8E6beGodTsuSSlZ/zQvy6ViDcI4gEKxFViqKR2xai7PeLsqw+HWdkXhawwEqYA==} - peerDependencies: - '@aws-sdk/client-s3': ^3.596.0 - '@message-queue-toolkit/core': '>=24.0.0' - '@message-queue-toolkit/schemas@7.1.0': resolution: {integrity: sha512-JAzSQAHouympK/cEDBxsfEuS2Ifu1pv0a/NRvhNWfFlgW0TmsWT7SkYNERA7x89OK7PGk9PyDN88cV9l0gZ22Q==} peerDependencies: zod: '>=3.25.76 <5.0.0' - '@message-queue-toolkit/sqs@24.2.1': - resolution: {integrity: sha512-R3+XSzvBUStsjbjKWtTIb64PD/4+cUM0CrqiATuPj4XUK0WTzCYhHF42okk+6k1Nv5H3U7P1gGXgeotBGRKadg==} - peerDependencies: - '@aws-sdk/client-sqs': ^3.632.0 - '@message-queue-toolkit/core': '>=25.0.0' - zod: '>=3.25.76 <5.0.0' + '@mongodb-js/zstd@7.0.0': + resolution: {integrity: sha512-mQ2s0pYYiav+tzCDR05Zptem8Ey2v8s11lri5RKGhTtL4COVCvVCk5vtyRYNT+9L8qSfyOqqefF9UtnW8mC5jA==} + engines: {node: '>= 20.19.0'} '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} @@ -1567,6 +1561,9 @@ packages: bintrees@1.0.2: resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + bowser@2.14.1: resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} @@ -1584,6 +1581,9 @@ packages: buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bullmq@5.76.8: resolution: {integrity: sha512-v3WTwA8diFtsADaJ8eK2ozyi2CYK9rDZCeoKF+dIPF/MUL8HxAOa3H72Gmu1lC4yKlho6t1PwNr/QpDVqaNEZQ==} engines: {node: '>=12.22.0'} @@ -1596,6 +1596,9 @@ packages: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -1653,6 +1656,14 @@ packages: supports-color: optional: true + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + + deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -1729,6 +1740,10 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -1823,6 +1838,9 @@ packages: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1859,6 +1877,9 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -1945,9 +1966,15 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + ioredis@5.10.1: resolution: {integrity: sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==} engines: {node: '>=12.22.0'} @@ -2168,6 +2195,10 @@ packages: engines: {node: '>=10.0.0'} hasBin: true + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + minimatch@10.2.5: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} @@ -2183,6 +2214,9 @@ packages: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + mnemonist@0.40.4: resolution: {integrity: sha512-ZAv+KNavneRVzu4tUeOgzkScI3W5BGwZ3rkxIpKtzzVgfTtWQFN1CgX0U72cyvyh3iTuHL3SiSmrQxTlryEIcw==} @@ -2201,9 +2235,20 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + + node-abi@3.92.0: + resolution: {integrity: sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==} + engines: {node: '>=10'} + node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-addon-api@8.7.0: + resolution: {integrity: sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==} + engines: {node: ^18 || ^20 || >= 21} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2302,6 +2347,12 @@ packages: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. + hasBin: true + process-warning@4.0.1: resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} @@ -2333,6 +2384,10 @@ packages: quick-format-unescaped@4.0.4: resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} @@ -2449,6 +2504,12 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + sonic-boom@4.2.1: resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} @@ -2460,12 +2521,6 @@ packages: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} - sqs-consumer@14.2.8: - resolution: {integrity: sha512-Iq3AdASfsxY2s3KL88aPRGgvST3gviZphL5teTKxUqDZzXRIB8s6SWATft+v7YaGMspCs8BwrOKo9vD6ltcJmw==} - engines: {node: '>=20.0.0'} - peerDependencies: - '@aws-sdk/client-sqs': ^3.1018.0 - sqs-consumer@15.0.1: resolution: {integrity: sha512-cDcSkekXxhLMn3u+FSnAG3Ryh6jwHMPna1/oqs2dUL0gvUXOLLMa2Yzu0K4QqwVyaH0MDZw1IU7+W+W618/xcw==} engines: {node: '>=22.0.0'} @@ -2506,6 +2561,10 @@ packages: resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} + strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + strip-json-comments@5.0.3: resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} engines: {node: '>=14.16'} @@ -2524,6 +2583,13 @@ packages: resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} engines: {node: '>=20'} + tar-fs@2.1.4: + resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} @@ -2579,6 +2645,9 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + turbo@2.9.14: resolution: {integrity: sha512-BQqXRr4UoWI3UPFrtznCLykYHxwxWh53iCB57x092jPMjIlW1wnm3N895g5irpiXmnxUhREBB0n6+y8BHhs4nw==} hasBin: true @@ -3378,24 +3447,14 @@ snapshots: - supports-color - zod - '@message-queue-toolkit/s3-payload-store@3.0.0(@aws-sdk/client-s3@3.1048.0)(@message-queue-toolkit/core@25.5.0(zod@4.4.3))': - dependencies: - '@aws-sdk/client-s3': 3.1048.0 - '@message-queue-toolkit/core': 25.5.0(zod@4.4.3) - '@message-queue-toolkit/schemas@7.1.0(zod@4.4.3)': dependencies: zod: 4.4.3 - '@message-queue-toolkit/sqs@24.2.1(@aws-sdk/client-sqs@3.1048.0)(@message-queue-toolkit/core@25.5.0(zod@4.4.3))(zod@4.4.3)': + '@mongodb-js/zstd@7.0.0': dependencies: - '@aws-sdk/client-sqs': 3.1048.0 - '@lokalise/node-core': 14.8.1(zod@4.4.3) - '@message-queue-toolkit/core': 25.5.0(zod@4.4.3) - sqs-consumer: 14.2.8(@aws-sdk/client-sqs@3.1048.0) - zod: 4.4.3 - transitivePeerDependencies: - - supports-color + node-addon-api: 8.7.0 + prebuild-install: 7.1.3 '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true @@ -3861,6 +3920,12 @@ snapshots: bintrees@1.0.2: {} + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + bowser@2.14.1: {} brace-expansion@2.1.0: @@ -3877,6 +3942,11 @@ snapshots: buffer-equal-constant-time@1.0.1: {} + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + bullmq@5.76.8: dependencies: cron-parser: 4.9.0 @@ -3895,6 +3965,8 @@ snapshots: chai@6.2.2: {} + chownr@1.1.4: {} + cliui@8.0.1: dependencies: string-width: 4.2.3 @@ -3939,6 +4011,12 @@ snapshots: dependencies: ms: 2.1.3 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + + deep-extend@0.6.0: {} + delayed-stream@1.0.0: {} denque@2.1.0: {} @@ -4007,6 +4085,8 @@ snapshots: event-target-shim@5.0.1: {} + expand-template@2.0.3: {} + expect-type@1.3.0: {} extend@3.0.2: {} @@ -4138,6 +4218,8 @@ snapshots: dependencies: fetch-blob: 3.2.0 + fs-constants@1.0.0: {} + fsevents@2.3.3: optional: true @@ -4199,6 +4281,8 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + github-from-package@0.0.0: {} + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -4320,8 +4404,12 @@ snapshots: transitivePeerDependencies: - supports-color + ieee754@1.2.1: {} + inherits@2.0.4: {} + ini@1.3.8: {} + ioredis@5.10.1: dependencies: '@ioredis/commands': 1.5.1 @@ -4506,6 +4594,8 @@ snapshots: mime@3.0.0: {} + mimic-response@3.1.0: {} + minimatch@10.2.5: dependencies: brace-expansion: 5.0.6 @@ -4518,6 +4608,8 @@ snapshots: minipass@7.1.3: {} + mkdirp-classic@0.5.3: {} + mnemonist@0.40.4: dependencies: obliterator: 2.0.5 @@ -4542,8 +4634,16 @@ snapshots: nanoid@3.3.12: {} + napi-build-utils@2.0.0: {} + + node-abi@3.92.0: + dependencies: + semver: 7.8.0 + node-abort-controller@3.1.1: {} + node-addon-api@8.7.0: {} + node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -4645,6 +4745,21 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.1.2 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.92.0 + pump: 3.0.4 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.4 + tunnel-agent: 0.6.0 + process-warning@4.0.1: {} process-warning@5.0.0: {} @@ -4687,6 +4802,13 @@ snapshots: quick-format-unescaped@4.0.4: {} + rc@1.2.8: + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + readable-stream@3.6.2: dependencies: inherits: 2.0.4 @@ -4799,6 +4921,14 @@ snapshots: signal-exit@4.1.0: {} + simple-concat@1.0.1: {} + + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 @@ -4807,13 +4937,6 @@ snapshots: split2@4.2.0: {} - sqs-consumer@14.2.8(@aws-sdk/client-sqs@3.1048.0): - dependencies: - '@aws-sdk/client-sqs': 3.1048.0 - debug: 4.4.3 - transitivePeerDependencies: - - supports-color - sqs-consumer@15.0.1(@aws-sdk/client-sqs@3.1048.0): dependencies: '@aws-sdk/client-sqs': 3.1048.0 @@ -4857,6 +4980,8 @@ snapshots: dependencies: ansi-regex: 6.2.2 + strip-json-comments@2.0.1: {} + strip-json-comments@5.0.3: {} strnum@2.3.0: {} @@ -4869,6 +4994,21 @@ snapshots: tagged-tag@1.0.0: {} + tar-fs@2.1.4: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.4 + tar-stream: 2.2.0 + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + tdigest@0.1.2: dependencies: bintrees: 1.0.2 @@ -4926,6 +5066,10 @@ snapshots: tslib@2.8.1: {} + tunnel-agent@0.6.0: + dependencies: + safe-buffer: 5.2.1 + turbo@2.9.14: optionalDependencies: '@turbo/darwin-64': 2.9.14 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5918a4e3..35668904 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ packages: # pnpm 11 auto-appends new entries here whenever a dep has scripts, prompting # an explicit decision per package. allowBuilds: + '@mongodb-js/zstd': true msgpackr-extract: false protobufjs: false From b3b0fcfd66c23d0884b61984796a364ea0c70590 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Mon, 18 May 2026 18:28:35 +0200 Subject: [PATCH 02/23] refactor: move zstd implementation out of core, harden codec guardrails, fix test races - Move @mongodb-js/zstd out of core into sqs; core now only defines MessageCodecHandler interface + pure envelope types (CodecEnvelope, isCodecEnvelope, MessageCodec) with no native dependencies - Add packages/sqs/lib/codec/sqsCodecHandler.ts: ZstdCodecHandler, resolveCodecHandler, and the concrete compressMessageBody / decompressMessageBody helpers - AbstractSqsPublisher and AbstractSqsConsumer import from local codec; AbstractSnsPublisher imports compressMessageBody from @message-queue-toolkit/sqs - Strengthen isCodecEnvelope to assert typeof __data === 'string' so Buffer.from downstream is guaranteed a string - Fix race condition in SQS codec auto-detection test: use a dedicated queue (user_permissions_multi-auto-detect) instead of sharing the beforeEach consumer's queue, eliminating both the steal-race and the localstack long-poll timing issue - Fix race condition in SNS codec auto-detection test: stop the original consumer before starting autoConsumer, reassign consumer = autoConsumer so afterEach handles cleanup without a double-close Co-Authored-By: Claude Sonnet 4.6 --- packages/core/lib/codec/messageCodec.ts | 32 +++++------------ packages/core/lib/index.ts | 3 +- packages/core/package.json | 1 - packages/sns/lib/sns/AbstractSnsPublisher.ts | 3 +- .../SnsSqsPermissionConsumer.codec.spec.ts | 7 ++-- packages/sqs/lib/codec/sqsCodecHandler.ts | 36 +++++++++++++++++++ packages/sqs/lib/index.ts | 6 ++++ packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 6 ++-- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 2 +- packages/sqs/package.json | 1 + .../SqsPermissionConsumer.codec.spec.ts | 20 ++++++++--- pnpm-lock.yaml | 6 ++-- 12 files changed, 81 insertions(+), 42 deletions(-) create mode 100644 packages/sqs/lib/codec/sqsCodecHandler.ts diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 4709a8de..8b6f0253 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -1,5 +1,3 @@ -import { compress, decompress } from '@mongodb-js/zstd' - export const SUPPORTED_CODECS = ['zstd'] as const export type MessageCodec = (typeof SUPPORTED_CODECS)[number] @@ -11,33 +9,19 @@ export type CodecEnvelope = { [DATA_FIELD]: string } +export interface MessageCodecHandler { + compress(data: Buffer): Promise + decompress(data: Buffer): Promise +} + export function isCodecEnvelope(value: unknown): value is CodecEnvelope { + const record = value as Record return ( typeof value === 'object' && value !== null && CODEC_FIELD in value && DATA_FIELD in value && - SUPPORTED_CODECS.includes((value as Record)[CODEC_FIELD] as MessageCodec) + SUPPORTED_CODECS.includes(record[CODEC_FIELD] as MessageCodec) && + typeof record[DATA_FIELD] === 'string' ) } - -export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { - if (codec === 'zstd') { - const compressed = await compress(Buffer.from(jsonBody, 'utf8')) - const envelope: CodecEnvelope = { - [CODEC_FIELD]: codec, - [DATA_FIELD]: compressed.toString('base64'), - } - return JSON.stringify(envelope) - } - throw new Error(`Unsupported codec: ${codec}`) -} - -export async function decompressMessageBody(envelope: CodecEnvelope): Promise { - if (envelope[CODEC_FIELD] === 'zstd') { - const compressed = Buffer.from(envelope[DATA_FIELD], 'base64') - const decompressed = await decompress(compressed) - return JSON.parse(decompressed.toString('utf8')) - } - throw new Error(`Unsupported codec: ${envelope[CODEC_FIELD]}`) -} diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index 97855fb7..c3334659 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,9 +1,8 @@ export { type CodecEnvelope, - compressMessageBody, - decompressMessageBody, isCodecEnvelope, type MessageCodec, + type MessageCodecHandler, SUPPORTED_CODECS, } from './codec/messageCodec.ts' export { DoNotProcessMessageError } from './errors/DoNotProcessError.ts' diff --git a/packages/core/package.json b/packages/core/package.json index 12b5e3be..041db97b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -28,7 +28,6 @@ "@lokalise/node-core": "^14.2.0", "@lokalise/universal-ts-utils": "^4.5.1", "@message-queue-toolkit/schemas": "^7.0.0", - "@mongodb-js/zstd": "^7.0.0", "dot-prop": "^10.1.0", "fast-equals": "^6.0.0", "json-stream-stringify": "^3.1.6", diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 6e9f95f7..81c98269 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -5,7 +5,6 @@ import { InternalError } from '@lokalise/node-core' import { type AsyncPublisher, type BarrierResult, - compressMessageBody, DeduplicationRequesterEnum, isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, @@ -15,7 +14,7 @@ import { type QueuePublisherOptions, type ResolvedMessage, } from '@message-queue-toolkit/core' -import { resolveOutgoingMessageAttributes } from '@message-queue-toolkit/sqs' +import { compressMessageBody, resolveOutgoingMessageAttributes } from '@message-queue-toolkit/sqs' import { calculateOutgoingMessageSize, validateFifoTopicName } from '../utils/snsUtils.ts' diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts index 72611d4a..e859c317 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts @@ -80,6 +80,9 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { }, 15000) it('consumer without codec option auto-detects and decompresses zstd messages from SNS', async () => { + // Stop the beforeEach consumer so it cannot steal messages from the shared queue + await consumer.close() + // Consumer without explicit codec — decompression is auto-detected from envelope __codec field const autoConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { locatorConfig: { @@ -89,6 +92,8 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { }, }) await autoConsumer.start() + // Reassign so afterEach closes autoConsumer instead of the already-closed consumer + consumer = autoConsumer const message: PERMISSIONS_ADD_MESSAGE_TYPE = { id: 'sns-codec-auto-1', @@ -98,7 +103,5 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') expect(result.message).toMatchObject(message) - - await autoConsumer.close() }, 15000) }) diff --git a/packages/sqs/lib/codec/sqsCodecHandler.ts b/packages/sqs/lib/codec/sqsCodecHandler.ts new file mode 100644 index 00000000..4a095e21 --- /dev/null +++ b/packages/sqs/lib/codec/sqsCodecHandler.ts @@ -0,0 +1,36 @@ +import type { CodecEnvelope, MessageCodec, MessageCodecHandler } from '@message-queue-toolkit/core' +import { compress, decompress } from '@mongodb-js/zstd' + +export class ZstdCodecHandler implements MessageCodecHandler { + compress(data: Buffer): Promise { + return compress(data) + } + + decompress(data: Buffer): Promise { + return decompress(data) + } +} + +const ZSTD_HANDLER = new ZstdCodecHandler() + +export function resolveCodecHandler(codec: MessageCodec): MessageCodecHandler { + if (codec === 'zstd') return ZSTD_HANDLER + throw new Error(`Unsupported codec: ${codec}`) +} + +export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { + const handler = resolveCodecHandler(codec) + const compressed = await handler.compress(Buffer.from(jsonBody, 'utf8')) + const envelope: CodecEnvelope = { + __codec: codec, + __data: compressed.toString('base64'), + } + return JSON.stringify(envelope) +} + +export async function decompressMessageBody(envelope: CodecEnvelope): Promise { + const handler = resolveCodecHandler(envelope.__codec) + const compressed = Buffer.from(envelope.__data, 'base64') + const decompressed = await handler.decompress(compressed) + return JSON.parse(decompressed.toString('utf8')) +} diff --git a/packages/sqs/lib/index.ts b/packages/sqs/lib/index.ts index 49d3803c..7705888f 100644 --- a/packages/sqs/lib/index.ts +++ b/packages/sqs/lib/index.ts @@ -1,3 +1,9 @@ +export { + compressMessageBody, + decompressMessageBody, + resolveCodecHandler, + ZstdCodecHandler, +} from './codec/sqsCodecHandler.ts' export { SqsConsumerErrorResolver } from './errors/SqsConsumerErrorResolver.ts' export { FakeConsumerErrorResolver } from './fakes/FakeConsumerErrorResolver.ts' export { TestSqsPublisher, type TestSqsPublishOptions } from './fakes/TestSqsPublisher.ts' diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index 7ce99a76..e45e94f9 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -10,7 +10,6 @@ import { type BarrierResult, type DeadLetterQueueOptions, DeduplicationRequesterEnum, - decompressMessageBody, HandlerContainer, isCodecEnvelope, isMessageError, @@ -28,6 +27,7 @@ import { import type { ConsumerOptions } from 'sqs-consumer' import { Consumer } from 'sqs-consumer' import type { ZodSchema } from 'zod/v4' +import { decompressMessageBody } from '../codec/sqsCodecHandler.ts' import type { SQSMessage } from '../types/MessageTypes.ts' import { hasOffloadedPayload } from '../utils/messageUtils.ts' import { deleteSqs, initSqs } from '../utils/sqsInitter.ts' @@ -880,7 +880,7 @@ export abstract class AbstractSqsConsumer< } // Empty content for whatever reason - if (!resolveMessageResult.result || !resolveMessageResult.result.body) { + if (!resolveMessageResult.result?.body) { return ABORT_EARLY_EITHER } @@ -916,7 +916,7 @@ export abstract class AbstractSqsConsumer< const resolvedMessage = resolveMessageResult.result // Empty content for whatever reason - if (!resolvedMessage || !resolvedMessage.body) return ABORT_EARLY_EITHER + if (!resolvedMessage?.body) return ABORT_EARLY_EITHER // @ts-expect-error if (this.messageIdField in resolvedMessage.body) { diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index 54a2e43e..f6cc3f7f 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -5,7 +5,6 @@ import { InternalError } from '@lokalise/node-core' import { type AsyncPublisher, type BarrierResult, - compressMessageBody, DeduplicationRequesterEnum, isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, @@ -16,6 +15,7 @@ import { type ResolvedMessage, } from '@message-queue-toolkit/core' import type { ZodSchema } from 'zod/v4' +import { compressMessageBody } from '../codec/sqsCodecHandler.ts' import type { SQSMessage } from '../types/MessageTypes.ts' import { resolveOutgoingMessageAttributes } from '../utils/messageUtils.ts' diff --git a/packages/sqs/package.json b/packages/sqs/package.json index b6c36306..823da8db 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -29,6 +29,7 @@ }, "dependencies": { "@lokalise/node-core": "^14.6.1", + "@mongodb-js/zstd": "^7.0.0", "sqs-consumer": "^15.0.1" }, "peerDependencies": { diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index 042050e2..b7229361 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,8 +1,8 @@ import { SendMessageCommand } from '@aws-sdk/client-sqs' -import { compressMessageBody } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { compressMessageBody } from '../../lib/codec/sqsCodecHandler.ts' import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher.ts' import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' @@ -102,9 +102,20 @@ describe('SqsPermissionConsumer - zstd codec', () => { }) it('consumer without codec option still decompresses zstd messages (auto-detection)', async () => { + // Use a dedicated queue so only autoConsumer polls it — avoids both the race + // condition (shared queue) and localstack long-poll timing issues (abort + restart) + const autoQueueName = `${SqsPermissionConsumer.QUEUE_NAME}-auto-detect` + await testAdmin.deleteQueues(autoQueueName) + + const autoPublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: autoQueueName } }, + }) + await autoPublisher.init() + // Consumer without codec — auto-detects from envelope __codec field const autoConsumer = new SqsPermissionConsumer(diContainer.cradle, { - locatorConfig: { queueUrl: consumer.queueProps.url }, + creationConfig: { queue: { QueueName: autoQueueName } }, deletionConfig: { deleteIfExists: false }, }) await autoConsumer.start() @@ -113,11 +124,12 @@ describe('SqsPermissionConsumer - zstd codec', () => { id: 'codec-auto-detect-1', messageType: 'add', } - await publisher.publish(message) + await autoPublisher.publish(message) const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') expect(result.message).toMatchObject(message) + await autoPublisher.close() await autoConsumer.close(true) - }) + }, 15000) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01681c80..53770fcc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -75,9 +75,6 @@ importers: '@message-queue-toolkit/schemas': specifier: ^7.0.0 version: 7.1.0(zod@4.4.3) - '@mongodb-js/zstd': - specifier: ^7.0.0 - version: 7.0.0 dot-prop: specifier: ^10.1.0 version: 10.1.0 @@ -548,6 +545,9 @@ importers: '@lokalise/node-core': specifier: ^14.6.1 version: 14.8.1(zod@4.4.3) + '@mongodb-js/zstd': + specifier: ^7.0.0 + version: 7.0.0 sqs-consumer: specifier: ^15.0.1 version: 15.0.1(@aws-sdk/client-sqs@3.1048.0) From 9950db2fba16696d31c516cb4cb5681727899962 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 09:16:40 +0200 Subject: [PATCH 03/23] ci: rebuild @mongodb-js/zstd native binary after --ignore-scripts install The native addon requires node-gyp compilation. pnpm install runs with --ignore-scripts in CI, so the binary is never built. pnpm rebuild explicitly compiles it regardless of that flag. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.common.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.common.yml b/.github/workflows/ci.common.yml index 4875f728..94f3d65c 100644 --- a/.github/workflows/ci.common.yml +++ b/.github/workflows/ci.common.yml @@ -36,6 +36,9 @@ jobs: - name: Install run: pnpm install --frozen-lockfile --ignore-scripts + - name: Build native dependencies + run: pnpm rebuild @mongodb-js/zstd + - name: Build TS env: PACKAGE_NAME: ${{ inputs.package_name }} From 2dfdb6e334511fd1da3c0a4690fe0fb14cf7db82 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 09:35:30 +0200 Subject: [PATCH 04/23] feat(sqs): add codec benchmarks for publish and consume throughput Measures wall-clock time and msg/s for 50 messages with and without zstd compression across small (~80 B) and large (~6 KB) payloads. Each run deletes its queues before and after so no resources are left behind. Run with: pnpm --filter @message-queue-toolkit/sqs bench Co-Authored-By: Claude Sonnet 4.6 --- packages/sqs/bench/codec.bench.ts | 206 ++++++++++++++++++++++++++++ packages/sqs/package.json | 1 + packages/sqs/vitest.bench.config.ts | 14 ++ 3 files changed, 221 insertions(+) create mode 100644 packages/sqs/bench/codec.bench.ts create mode 100644 packages/sqs/vitest.bench.config.ts diff --git a/packages/sqs/bench/codec.bench.ts b/packages/sqs/bench/codec.bench.ts new file mode 100644 index 00000000..12735c67 --- /dev/null +++ b/packages/sqs/bench/codec.bench.ts @@ -0,0 +1,206 @@ +/** + * Codec benchmarks — publish and consume throughput with vs without zstd compression. + * + * Run: pnpm --filter @message-queue-toolkit/sqs bench + * + * Each benchmark pre-fills queues (consume) or sends N messages (publish) and + * measures wall-clock time, reporting msg/s and the overhead percentage. + * All queues are deleted before and after each case. + */ +import type { AwilixContainer } from 'awilix' +import { asValue } from 'awilix' +import { afterAll, beforeAll, describe, it } from 'vitest' + +import { SqsPermissionConsumer } from '../test/consumers/SqsPermissionConsumer.ts' +import type { PERMISSIONS_ADD_MESSAGE_TYPE } from '../test/consumers/userConsumerSchemas.ts' +import { SqsPermissionPublisher } from '../test/publishers/SqsPermissionPublisher.ts' +import type { TestAwsResourceAdmin } from '../test/utils/testAdmin.ts' +import type { Dependencies } from '../test/utils/testContext.ts' +import { registerDependencies } from '../test/utils/testContext.ts' + +// ─── Configuration ──────────────────────────────────────────────────────────── + +const N = 50 + +/** Small message with minimal payload (~80 B serialised). */ +const SMALL_META: undefined = undefined + +/** + * Large message with repetitive text (~6 KB serialised). + * Repetitive content compresses very well, showing the realistic best case. + */ +const LARGE_META: Record = { + description: 'The quick brown fox jumps over the lazy dog. '.repeat(60), + items: Array.from({ length: 80 }, (_, i) => ({ + id: `item-${i}`, + value: `value-number-${i}`, + enabled: i % 2 === 0, + })), +} + +const CASES = [ + { label: 'small payload (~80 B) ', suffix: 'sm', meta: SMALL_META }, + { label: 'large payload (~6 KB) ', suffix: 'lg', meta: LARGE_META }, +] as const + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function makeMessages( + prefix: string, + count: number, + meta: Record | undefined, +): PERMISSIONS_ADD_MESSAGE_TYPE[] { + return Array.from({ length: count }, (_, i) => ({ + id: `${prefix}-${i}`, + messageType: 'add' as const, + ...(meta !== undefined ? { metadata: meta } : {}), + })) +} + +function printRow(label: string, count: number, plainMs: number, codecMs: number): void { + const tps = (ms: number) => ((count / ms) * 1000).toFixed(0).padStart(6) + const diff = codecMs - plainMs + const pct = ((diff / plainMs) * 100).toFixed(1) + const sign = diff >= 0 ? '+' : '' + console.log( + ` ${label}` + + ` plain: ${String(plainMs.toFixed(0)).padStart(5)} ms (${tps(plainMs)} msg/s)` + + ` zstd: ${String(codecMs.toFixed(0)).padStart(5)} ms (${tps(codecMs)} msg/s)` + + ` overhead: ${sign}${pct}%`, + ) +} + +// ─── Suite ──────────────────────────────────────────────────────────────────── + +describe('SQS codec benchmarks', () => { + let diContainer: AwilixContainer + let testAdmin: TestAwsResourceAdmin + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + testAdmin = diContainer.cradle.testAdmin + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + // ── Publish ──────────────────────────────────────────────────────────────── + + it( + `publish: with vs without zstd (${N} messages)`, + async () => { + console.log(`\n${'─'.repeat(72)}`) + console.log(` PUBLISH BENCHMARK — ${N} messages per run`) + console.log('─'.repeat(72)) + + for (const { label, suffix, meta } of CASES) { + const plainQ = `bench-pub-plain-${suffix}` + const codecQ = `bench-pub-codec-${suffix}` + await testAdmin.deleteQueues(plainQ, codecQ) + + const plainMsgs = makeMessages('bpp', N, meta) + const codecMsgs = makeMessages('bcp', N, meta) + + // ── Plain publish ── + const plainPub = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + }) + await plainPub.init() + const t0 = performance.now() + for (const msg of plainMsgs) await plainPub.publish(msg) + const plainMs = performance.now() - t0 + await plainPub.close() + + // ── Codec publish ── + const codecPub = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + }) + await codecPub.init() + const t1 = performance.now() + for (const msg of codecMsgs) await codecPub.publish(msg) + const codecMs = performance.now() - t1 + await codecPub.close() + + await testAdmin.deleteQueues(plainQ, codecQ) + printRow(label, N, plainMs, codecMs) + } + }, + 120_000, + ) + + // ── Consume ──────────────────────────────────────────────────────────────── + + it( + `consume: with vs without zstd (${N} messages)`, + async () => { + console.log(`\n${'─'.repeat(72)}`) + console.log(` CONSUME BENCHMARK — ${N} messages per run`) + console.log('─'.repeat(72)) + + for (const { label, suffix, meta } of CASES) { + const plainQ = `bench-con-plain-${suffix}` + const codecQ = `bench-con-codec-${suffix}` + await testAdmin.deleteQueues(plainQ, codecQ) + + const plainMsgs = makeMessages('bpc', N, meta) + const codecMsgs = makeMessages('bcc', N, meta) + + // ── Pre-fill plain queue ── + const plainPub = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + }) + await plainPub.init() + for (const msg of plainMsgs) await plainPub.publish(msg) + await plainPub.close() + + // ── Pre-fill codec queue ── + const codecPub = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + }) + await codecPub.init() + for (const msg of codecMsgs) await codecPub.publish(msg) + await codecPub.close() + + // ── Measure plain consume ── + // deletionConfig: { deleteIfExists: false } preserves the pre-filled queue + const plainCon = new SqsPermissionConsumer(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + deletionConfig: { deleteIfExists: false }, + }) + await plainCon.start() + const t2 = performance.now() + await Promise.all( + plainMsgs.map((m) => plainCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), + ) + const plainMs = performance.now() - t2 + await plainCon.close(true) + + // ── Measure codec consume ── + const codecCon = new SqsPermissionConsumer(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + deletionConfig: { deleteIfExists: false }, + }) + await codecCon.start() + const t3 = performance.now() + await Promise.all( + codecMsgs.map((m) => codecCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), + ) + const codecMs = performance.now() - t3 + await codecCon.close(true) + + await testAdmin.deleteQueues(plainQ, codecQ) + printRow(label, N, plainMs, codecMs) + } + }, + 120_000, + ) +}) diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 823da8db..acf138f3 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -19,6 +19,7 @@ "scripts": { "build": "pnpm run clean && tsc --project tsconfig.build.json", "clean": "rimraf dist", + "bench": "vitest run --config vitest.bench.config.ts --reporter=verbose", "test": "vitest", "test:coverage": "pnpm run test --coverage", "lint": "biome check && tsc", diff --git a/packages/sqs/vitest.bench.config.ts b/packages/sqs/vitest.bench.config.ts new file mode 100644 index 00000000..fd4e580e --- /dev/null +++ b/packages/sqs/vitest.bench.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vitest/config' + +// biome-ignore lint/style/noDefaultExport: vite expects default export +export default defineConfig({ + test: { + globals: true, + watch: false, + mockReset: true, + pool: 'threads', + maxWorkers: 1, + setupFiles: ['test/utils/vitest.setup.ts'], + include: ['bench/**/*.ts'], + }, +}) From c1d83e7b6e47fadb8eba448c504a0843ebaeecd1 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 10:12:02 +0200 Subject: [PATCH 05/23] feat(sqs/core): replace @mongodb-js/zstd with Node.js built-in zlib and add codec documentation Switch from @mongodb-js/zstd (native node-gyp addon requiring Python and a C++ toolchain) to zlib.zstdCompress/zstdDecompress built into Node.js 22+. This removes 24 transitive packages, drops the pnpm rebuild CI step, and eliminates native build requirements for end users of the package. Refactor MessageCodec to use MessageCodecEnum object pattern, enabling MessageCodecEnum.ZSTD usage alongside the plain string literal. Add JSDoc to MessageCodecEnum, MessageCodecHandler, and the codec option in queueOptionsTypes. Add a Message Compression section to the SQS README with publisher/consumer examples and auto-detection behaviour, and reference it from the SNS README. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.common.yml | 3 - biome.json | 14 +- packages/core/lib/codec/messageCodec.ts | 28 ++- packages/core/lib/index.ts | 2 +- packages/core/lib/types/queueOptionsTypes.ts | 23 +- packages/sns/README.md | 5 + packages/sqs/README.md | 58 ++++++ packages/sqs/bench/codec.bench.ts | 208 +++++++++---------- packages/sqs/lib/codec/sqsCodecHandler.ts | 13 +- packages/sqs/package.json | 1 - pnpm-lock.yaml | 185 ----------------- 11 files changed, 229 insertions(+), 311 deletions(-) diff --git a/.github/workflows/ci.common.yml b/.github/workflows/ci.common.yml index 94f3d65c..4875f728 100644 --- a/.github/workflows/ci.common.yml +++ b/.github/workflows/ci.common.yml @@ -36,9 +36,6 @@ jobs: - name: Install run: pnpm install --frozen-lockfile --ignore-scripts - - name: Build native dependencies - run: pnpm rebuild @mongodb-js/zstd - - name: Build TS env: PACKAGE_NAME: ${{ inputs.package_name }} diff --git a/biome.json b/biome.json index e55fa69d..b0390747 100644 --- a/biome.json +++ b/biome.json @@ -15,5 +15,17 @@ "noUnusedPrivateClassMembers": "off" } } - } + }, + "overrides": [ + { + "includes": ["**/bench/**"], + "linter": { + "rules": { + "suspicious": { + "noConsole": "off" + } + } + } + } + ] } diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 8b6f0253..cc7fab0f 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -1,5 +1,20 @@ -export const SUPPORTED_CODECS = ['zstd'] as const -export type MessageCodec = (typeof SUPPORTED_CODECS)[number] +type ObjectValues = T[keyof T] + +/** + * Supported message compression codecs. + * + * Use the enum values instead of raw strings so that adding a new codec in + * the future is a single-place change and consumers benefit from IDE + * auto-complete. + * + * @example + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) + */ +export const MessageCodecEnum = { + /** zstd compression via Node.js built-in `zlib` (requires Node.js 22+). */ + ZSTD: 'zstd', +} as const +export type MessageCodec = ObjectValues const CODEC_FIELD = '__codec' const DATA_FIELD = '__data' @@ -9,6 +24,13 @@ export type CodecEnvelope = { [DATA_FIELD]: string } +/** + * Low-level interface for a compression codec. + * + * Implement this interface to plug in a custom compression algorithm. + * The built-in implementation (`ZstdCodecHandler` in `@message-queue-toolkit/sqs`) + * uses Node.js built-in `zlib` zstd support. + */ export interface MessageCodecHandler { compress(data: Buffer): Promise decompress(data: Buffer): Promise @@ -21,7 +43,7 @@ export function isCodecEnvelope(value: unknown): value is CodecEnvelope { value !== null && CODEC_FIELD in value && DATA_FIELD in value && - SUPPORTED_CODECS.includes(record[CODEC_FIELD] as MessageCodec) && + (Object.values(MessageCodecEnum) as string[]).includes(record[CODEC_FIELD] as string) && typeof record[DATA_FIELD] === 'string' ) } diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index c3334659..6d23f0ca 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -2,8 +2,8 @@ export { type CodecEnvelope, isCodecEnvelope, type MessageCodec, + MessageCodecEnum, type MessageCodecHandler, - SUPPORTED_CODECS, } from './codec/messageCodec.ts' export { DoNotProcessMessageError } from './errors/DoNotProcessError.ts' export { diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index 3310ca5b..c8ae6d7c 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -141,11 +141,24 @@ export type CommonQueueOptions = { payloadStoreConfig?: PayloadStoreConfig messageDeduplicationConfig?: MessageDeduplicationConfig /** - * Codec to use for compressing outgoing messages and decompressing incoming messages. - * When set on a publisher, messages are compressed before sending. - * When set on a consumer, it signals that incoming messages may be compressed. - * Compressed messages are self-describing (the codec is embedded in the message envelope), - * so consumers can decompress even without this option explicitly set. + * Compression codec applied to message bodies. + * + * - **Publisher**: every outgoing message body is compressed and wrapped in a + * self-describing envelope `{ __codec: 'zstd', __data: '' }`. + * - **Consumer**: when set, the consumer expects compressed messages. + * Even without this option, consumers auto-detect and decompress any message + * that carries a codec envelope, so mixed queues work transparently. + * + * Uses Node.js built-in `zlib` zstd support — **requires Node.js 22+**. + * + * @example + * import { MessageCodecEnum } from '@message-queue-toolkit/core' + * + * // Publisher + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) + * + * // Consumer (optional — auto-detection handles it even without this) + * new MyConsumer(deps, { codec: MessageCodecEnum.ZSTD }) */ codec?: MessageCodec } diff --git a/packages/sns/README.md b/packages/sns/README.md index ed6b7fb6..34112836 100644 --- a/packages/sns/README.md +++ b/packages/sns/README.md @@ -58,6 +58,7 @@ npm install @message-queue-toolkit/sns @message-queue-toolkit/sqs @message-queue - ✅ **Handler spies** for testing - ✅ **Pre-handlers and barriers** for complex message processing - ✅ **Cross-account and cross-region publishing** +- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js 22+ required) ## Core Concepts @@ -683,6 +684,9 @@ await consumer.start() // Optional - Payload Offloading (same as SQS) payloadStoreConfig: { /* ... */ }, + // Optional - Compression (Node.js 22+ required) + codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd + // Optional - Deletion deletionConfig: { /* ... */ }, } @@ -1000,6 +1004,7 @@ SNS consumers inherit all advanced features from SQS consumers. See the SQS READ - **[Message Retry Logic](../sqs/README.md#message-retry-logic)** - Exponential backoff and retry limits - **[Message Deduplication](../sqs/README.md#message-deduplication)** - Publisher and consumer-level deduplication - **[Payload Offloading](../sqs/README.md#payload-offloading)** - S3 storage for large messages +- **[Message Compression](../sqs/README.md#message-compression)** - zstd compression via Node.js built-in `zlib` - **[Message Handlers](../sqs/README.md#message-handlers)** - Type-safe handler configuration - **[Pre-handlers and Barriers](../sqs/README.md#pre-handlers-and-barriers)** - Middleware and message dependencies - **[Handler Spies](../sqs/README.md#handler-spies)** - Testing async message flows diff --git a/packages/sqs/README.md b/packages/sqs/README.md index 94bdaf57..f37edd79 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -24,6 +24,7 @@ for publishing and consuming messages from both standard and FIFO SQS queues. - [Message Retry Logic](#message-retry-logic) - [Message Deduplication](#message-deduplication) - [Payload Offloading](#payload-offloading) + - [Message Compression](#message-compression) - [Message Handlers](#message-handlers) - [Pre-handlers and Barriers](#pre-handlers-and-barriers) - [Handler Spies](#handler-spies) @@ -62,6 +63,7 @@ npm install @message-queue-toolkit/sqs @message-queue-toolkit/core - ✅ **Handler spies** for testing - ✅ **Pre-handlers and barriers** for complex message processing - ✅ **Automatic queue creation** with validation +- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js 22+ required) ## Core Concepts @@ -460,6 +462,9 @@ When using `locatorConfig`, you connect to an existing queue without creating it maxPayloadSize: 1024 * 1024, // 1 MiB }, + // Optional - Compression (Node.js 22+ required) + codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd + // Optional - Deletion deletionConfig: { deleteIfExists: false, // Delete queue on init @@ -531,6 +536,11 @@ When using `locatorConfig`, you connect to an existing queue without creating it payloadStore: s3Store, }, + // Optional - Compression (Node.js 22+ required) + // Auto-detection is always active: consumers decompress codec envelopes + // even without this option set. + codec: MessageCodecEnum.ZSTD, + // Optional - Other logMessages: false, handlerSpy: true, @@ -791,6 +801,54 @@ await publisher.publish({ **Note:** Payload cleanup is the responsibility of the store (e.g., S3 lifecycle policies). +### Message Compression + +Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js 22+**. + +Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __codec: 'zstd', __data: '' }`), so a consumer without `codec` set will still decompress automatically via envelope detection. This allows a gradual rollout — enable compression on the publisher first, consumers adapt without configuration changes. + +#### Publisher + +```typescript +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +class MyPublisher extends AbstractSqsPublisher { + constructor(deps: SQSDependencies) { + super(deps, { + codec: MessageCodecEnum.ZSTD, // compress every outgoing message + creationConfig: { queue: { QueueName: 'my-queue' } }, + // ... + }) + } +} +``` + +#### Consumer + +```typescript +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +class MyConsumer extends AbstractSqsConsumer { + constructor(deps: SQSConsumerDependencies) { + super(deps, { + // Optional: explicitly declare that messages are compressed. + // Without this, consumers still auto-detect and decompress codec envelopes. + codec: MessageCodecEnum.ZSTD, + creationConfig: { queue: { QueueName: 'my-queue' } }, + handlers: new MessageHandlerConfigBuilder() + .addConfig(MySchema, myHandler) + .build(), + }, executionContext) + } +} +``` + +#### Notes + +- Compression is applied **after** schema validation and **before** the SQS `SendMessage` call. +- Compressed payloads are still subject to the SQS 256 KB message size limit. For large messages that remain oversized after compression, combine with [Payload Offloading](#payload-offloading). +- Uses `MessageCodecEnum.ZSTD` (value `'zstd'`). You can use the string literal or the enum — both satisfy the `MessageCodec` type. + ### Message Handlers Handlers process messages based on their type. Messages are routed to the appropriate handler using the discriminator field (configurable via `messageTypeResolver`): diff --git a/packages/sqs/bench/codec.bench.ts b/packages/sqs/bench/codec.bench.ts index 12735c67..600a7ca9 100644 --- a/packages/sqs/bench/codec.bench.ts +++ b/packages/sqs/bench/codec.bench.ts @@ -92,115 +92,107 @@ describe('SQS codec benchmarks', () => { // ── Publish ──────────────────────────────────────────────────────────────── - it( - `publish: with vs without zstd (${N} messages)`, - async () => { - console.log(`\n${'─'.repeat(72)}`) - console.log(` PUBLISH BENCHMARK — ${N} messages per run`) - console.log('─'.repeat(72)) - - for (const { label, suffix, meta } of CASES) { - const plainQ = `bench-pub-plain-${suffix}` - const codecQ = `bench-pub-codec-${suffix}` - await testAdmin.deleteQueues(plainQ, codecQ) - - const plainMsgs = makeMessages('bpp', N, meta) - const codecMsgs = makeMessages('bcp', N, meta) - - // ── Plain publish ── - const plainPub = new SqsPermissionPublisher(diContainer.cradle, { - creationConfig: { queue: { QueueName: plainQ } }, - }) - await plainPub.init() - const t0 = performance.now() - for (const msg of plainMsgs) await plainPub.publish(msg) - const plainMs = performance.now() - t0 - await plainPub.close() - - // ── Codec publish ── - const codecPub = new SqsPermissionPublisher(diContainer.cradle, { - codec: 'zstd', - creationConfig: { queue: { QueueName: codecQ } }, - }) - await codecPub.init() - const t1 = performance.now() - for (const msg of codecMsgs) await codecPub.publish(msg) - const codecMs = performance.now() - t1 - await codecPub.close() - - await testAdmin.deleteQueues(plainQ, codecQ) - printRow(label, N, plainMs, codecMs) - } - }, - 120_000, - ) + it(`publish: with vs without zstd (${N} messages)`, async () => { + console.log(`\n${'─'.repeat(72)}`) + console.log(` PUBLISH BENCHMARK — ${N} messages per run`) + console.log('─'.repeat(72)) + + for (const { label, suffix, meta } of CASES) { + const plainQ = `bench-pub-plain-${suffix}` + const codecQ = `bench-pub-codec-${suffix}` + await testAdmin.deleteQueues(plainQ, codecQ) + + const plainMsgs = makeMessages('bpp', N, meta) + const codecMsgs = makeMessages('bcp', N, meta) + + // ── Plain publish ── + const plainPub = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + }) + await plainPub.init() + const t0 = performance.now() + for (const msg of plainMsgs) await plainPub.publish(msg) + const plainMs = performance.now() - t0 + await plainPub.close() + + // ── Codec publish ── + const codecPub = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + }) + await codecPub.init() + const t1 = performance.now() + for (const msg of codecMsgs) await codecPub.publish(msg) + const codecMs = performance.now() - t1 + await codecPub.close() + + await testAdmin.deleteQueues(plainQ, codecQ) + printRow(label, N, plainMs, codecMs) + } + }, 120_000) // ── Consume ──────────────────────────────────────────────────────────────── - it( - `consume: with vs without zstd (${N} messages)`, - async () => { - console.log(`\n${'─'.repeat(72)}`) - console.log(` CONSUME BENCHMARK — ${N} messages per run`) - console.log('─'.repeat(72)) - - for (const { label, suffix, meta } of CASES) { - const plainQ = `bench-con-plain-${suffix}` - const codecQ = `bench-con-codec-${suffix}` - await testAdmin.deleteQueues(plainQ, codecQ) - - const plainMsgs = makeMessages('bpc', N, meta) - const codecMsgs = makeMessages('bcc', N, meta) - - // ── Pre-fill plain queue ── - const plainPub = new SqsPermissionPublisher(diContainer.cradle, { - creationConfig: { queue: { QueueName: plainQ } }, - }) - await plainPub.init() - for (const msg of plainMsgs) await plainPub.publish(msg) - await plainPub.close() - - // ── Pre-fill codec queue ── - const codecPub = new SqsPermissionPublisher(diContainer.cradle, { - codec: 'zstd', - creationConfig: { queue: { QueueName: codecQ } }, - }) - await codecPub.init() - for (const msg of codecMsgs) await codecPub.publish(msg) - await codecPub.close() - - // ── Measure plain consume ── - // deletionConfig: { deleteIfExists: false } preserves the pre-filled queue - const plainCon = new SqsPermissionConsumer(diContainer.cradle, { - creationConfig: { queue: { QueueName: plainQ } }, - deletionConfig: { deleteIfExists: false }, - }) - await plainCon.start() - const t2 = performance.now() - await Promise.all( - plainMsgs.map((m) => plainCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), - ) - const plainMs = performance.now() - t2 - await plainCon.close(true) - - // ── Measure codec consume ── - const codecCon = new SqsPermissionConsumer(diContainer.cradle, { - codec: 'zstd', - creationConfig: { queue: { QueueName: codecQ } }, - deletionConfig: { deleteIfExists: false }, - }) - await codecCon.start() - const t3 = performance.now() - await Promise.all( - codecMsgs.map((m) => codecCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), - ) - const codecMs = performance.now() - t3 - await codecCon.close(true) - - await testAdmin.deleteQueues(plainQ, codecQ) - printRow(label, N, plainMs, codecMs) - } - }, - 120_000, - ) + it(`consume: with vs without zstd (${N} messages)`, async () => { + console.log(`\n${'─'.repeat(72)}`) + console.log(` CONSUME BENCHMARK — ${N} messages per run`) + console.log('─'.repeat(72)) + + for (const { label, suffix, meta } of CASES) { + const plainQ = `bench-con-plain-${suffix}` + const codecQ = `bench-con-codec-${suffix}` + await testAdmin.deleteQueues(plainQ, codecQ) + + const plainMsgs = makeMessages('bpc', N, meta) + const codecMsgs = makeMessages('bcc', N, meta) + + // ── Pre-fill plain queue ── + const plainPub = new SqsPermissionPublisher(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + }) + await plainPub.init() + for (const msg of plainMsgs) await plainPub.publish(msg) + await plainPub.close() + + // ── Pre-fill codec queue ── + const codecPub = new SqsPermissionPublisher(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + }) + await codecPub.init() + for (const msg of codecMsgs) await codecPub.publish(msg) + await codecPub.close() + + // ── Measure plain consume ── + // deletionConfig: { deleteIfExists: false } preserves the pre-filled queue + const plainCon = new SqsPermissionConsumer(diContainer.cradle, { + creationConfig: { queue: { QueueName: plainQ } }, + deletionConfig: { deleteIfExists: false }, + }) + await plainCon.start() + const t2 = performance.now() + await Promise.all( + plainMsgs.map((m) => plainCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), + ) + const plainMs = performance.now() - t2 + await plainCon.close(true) + + // ── Measure codec consume ── + const codecCon = new SqsPermissionConsumer(diContainer.cradle, { + codec: 'zstd', + creationConfig: { queue: { QueueName: codecQ } }, + deletionConfig: { deleteIfExists: false }, + }) + await codecCon.start() + const t3 = performance.now() + await Promise.all( + codecMsgs.map((m) => codecCon.handlerSpy.waitForMessageWithId(m.id, 'consumed')), + ) + const codecMs = performance.now() - t3 + await codecCon.close(true) + + await testAdmin.deleteQueues(plainQ, codecQ) + printRow(label, N, plainMs, codecMs) + } + }, 120_000) }) diff --git a/packages/sqs/lib/codec/sqsCodecHandler.ts b/packages/sqs/lib/codec/sqsCodecHandler.ts index 4a095e21..b00c18ba 100644 --- a/packages/sqs/lib/codec/sqsCodecHandler.ts +++ b/packages/sqs/lib/codec/sqsCodecHandler.ts @@ -1,20 +1,25 @@ +import { promisify } from 'node:util' +import zlib from 'node:zlib' import type { CodecEnvelope, MessageCodec, MessageCodecHandler } from '@message-queue-toolkit/core' -import { compress, decompress } from '@mongodb-js/zstd' +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +const zstdCompress = promisify(zlib.zstdCompress) +const zstdDecompress = promisify(zlib.zstdDecompress) export class ZstdCodecHandler implements MessageCodecHandler { compress(data: Buffer): Promise { - return compress(data) + return zstdCompress(data) } decompress(data: Buffer): Promise { - return decompress(data) + return zstdDecompress(data) } } const ZSTD_HANDLER = new ZstdCodecHandler() export function resolveCodecHandler(codec: MessageCodec): MessageCodecHandler { - if (codec === 'zstd') return ZSTD_HANDLER + if (codec === MessageCodecEnum.ZSTD) return ZSTD_HANDLER throw new Error(`Unsupported codec: ${codec}`) } diff --git a/packages/sqs/package.json b/packages/sqs/package.json index acf138f3..de529971 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -30,7 +30,6 @@ }, "dependencies": { "@lokalise/node-core": "^14.6.1", - "@mongodb-js/zstd": "^7.0.0", "sqs-consumer": "^15.0.1" }, "peerDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 53770fcc..572f62f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -545,9 +545,6 @@ importers: '@lokalise/node-core': specifier: ^14.6.1 version: 14.8.1(zod@4.4.3) - '@mongodb-js/zstd': - specifier: ^7.0.0 - version: 7.0.0 sqs-consumer: specifier: ^15.0.1 version: 15.0.1(@aws-sdk/client-sqs@3.1048.0) @@ -999,10 +996,6 @@ packages: peerDependencies: zod: '>=3.25.76 <5.0.0' - '@mongodb-js/zstd@7.0.0': - resolution: {integrity: sha512-mQ2s0pYYiav+tzCDR05Zptem8Ey2v8s11lri5RKGhTtL4COVCvVCk5vtyRYNT+9L8qSfyOqqefF9UtnW8mC5jA==} - engines: {node: '>= 20.19.0'} - '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} cpu: [arm64] @@ -1561,9 +1554,6 @@ packages: bintrees@1.0.2: resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} - bl@4.1.0: - resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} - bowser@2.14.1: resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} @@ -1581,9 +1571,6 @@ packages: buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - buffer@5.7.1: - resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} - bullmq@5.76.8: resolution: {integrity: sha512-v3WTwA8diFtsADaJ8eK2ozyi2CYK9rDZCeoKF+dIPF/MUL8HxAOa3H72Gmu1lC4yKlho6t1PwNr/QpDVqaNEZQ==} engines: {node: '>=12.22.0'} @@ -1596,9 +1583,6 @@ packages: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} - chownr@1.1.4: - resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -1656,14 +1640,6 @@ packages: supports-color: optional: true - decompress-response@6.0.0: - resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} - engines: {node: '>=10'} - - deep-extend@0.6.0: - resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} - engines: {node: '>=4.0.0'} - delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -1740,10 +1716,6 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} - expand-template@2.0.3: - resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} - engines: {node: '>=6'} - expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -1838,9 +1810,6 @@ packages: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} - fs-constants@1.0.0: - resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} - fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1877,9 +1846,6 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} - github-from-package@0.0.0: - resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} - glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -1966,15 +1932,9 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} - ieee754@1.2.1: - resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - ini@1.3.8: - resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - ioredis@5.10.1: resolution: {integrity: sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==} engines: {node: '>=12.22.0'} @@ -2195,10 +2155,6 @@ packages: engines: {node: '>=10.0.0'} hasBin: true - mimic-response@3.1.0: - resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} - engines: {node: '>=10'} - minimatch@10.2.5: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} @@ -2214,9 +2170,6 @@ packages: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} - mkdirp-classic@0.5.3: - resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - mnemonist@0.40.4: resolution: {integrity: sha512-ZAv+KNavneRVzu4tUeOgzkScI3W5BGwZ3rkxIpKtzzVgfTtWQFN1CgX0U72cyvyh3iTuHL3SiSmrQxTlryEIcw==} @@ -2235,20 +2188,9 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - napi-build-utils@2.0.0: - resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} - - node-abi@3.92.0: - resolution: {integrity: sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==} - engines: {node: '>=10'} - node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} - node-addon-api@8.7.0: - resolution: {integrity: sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==} - engines: {node: ^18 || ^20 || >= 21} - node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2347,12 +2289,6 @@ packages: resolution: {integrity: sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==} engines: {node: ^10 || ^12 || >=14} - prebuild-install@7.1.3: - resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} - engines: {node: '>=10'} - deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. - hasBin: true - process-warning@4.0.1: resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} @@ -2384,10 +2320,6 @@ packages: quick-format-unescaped@4.0.4: resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} - rc@1.2.8: - resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} - hasBin: true - readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} @@ -2504,12 +2436,6 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} - simple-concat@1.0.1: - resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} - - simple-get@4.0.1: - resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} - sonic-boom@4.2.1: resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} @@ -2561,10 +2487,6 @@ packages: resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} - strip-json-comments@2.0.1: - resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} - engines: {node: '>=0.10.0'} - strip-json-comments@5.0.3: resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} engines: {node: '>=14.16'} @@ -2583,13 +2505,6 @@ packages: resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} engines: {node: '>=20'} - tar-fs@2.1.4: - resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} - - tar-stream@2.2.0: - resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} - engines: {node: '>=6'} - tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} @@ -2645,9 +2560,6 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - tunnel-agent@0.6.0: - resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - turbo@2.9.14: resolution: {integrity: sha512-BQqXRr4UoWI3UPFrtznCLykYHxwxWh53iCB57x092jPMjIlW1wnm3N895g5irpiXmnxUhREBB0n6+y8BHhs4nw==} hasBin: true @@ -3451,11 +3363,6 @@ snapshots: dependencies: zod: 4.4.3 - '@mongodb-js/zstd@7.0.0': - dependencies: - node-addon-api: 8.7.0 - prebuild-install: 7.1.3 - '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true @@ -3920,12 +3827,6 @@ snapshots: bintrees@1.0.2: {} - bl@4.1.0: - dependencies: - buffer: 5.7.1 - inherits: 2.0.4 - readable-stream: 3.6.2 - bowser@2.14.1: {} brace-expansion@2.1.0: @@ -3942,11 +3843,6 @@ snapshots: buffer-equal-constant-time@1.0.1: {} - buffer@5.7.1: - dependencies: - base64-js: 1.5.1 - ieee754: 1.2.1 - bullmq@5.76.8: dependencies: cron-parser: 4.9.0 @@ -3965,8 +3861,6 @@ snapshots: chai@6.2.2: {} - chownr@1.1.4: {} - cliui@8.0.1: dependencies: string-width: 4.2.3 @@ -4011,12 +3905,6 @@ snapshots: dependencies: ms: 2.1.3 - decompress-response@6.0.0: - dependencies: - mimic-response: 3.1.0 - - deep-extend@0.6.0: {} - delayed-stream@1.0.0: {} denque@2.1.0: {} @@ -4085,8 +3973,6 @@ snapshots: event-target-shim@5.0.1: {} - expand-template@2.0.3: {} - expect-type@1.3.0: {} extend@3.0.2: {} @@ -4218,8 +4104,6 @@ snapshots: dependencies: fetch-blob: 3.2.0 - fs-constants@1.0.0: {} - fsevents@2.3.3: optional: true @@ -4281,8 +4165,6 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 - github-from-package@0.0.0: {} - glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -4404,12 +4286,8 @@ snapshots: transitivePeerDependencies: - supports-color - ieee754@1.2.1: {} - inherits@2.0.4: {} - ini@1.3.8: {} - ioredis@5.10.1: dependencies: '@ioredis/commands': 1.5.1 @@ -4594,8 +4472,6 @@ snapshots: mime@3.0.0: {} - mimic-response@3.1.0: {} - minimatch@10.2.5: dependencies: brace-expansion: 5.0.6 @@ -4608,8 +4484,6 @@ snapshots: minipass@7.1.3: {} - mkdirp-classic@0.5.3: {} - mnemonist@0.40.4: dependencies: obliterator: 2.0.5 @@ -4634,16 +4508,8 @@ snapshots: nanoid@3.3.12: {} - napi-build-utils@2.0.0: {} - - node-abi@3.92.0: - dependencies: - semver: 7.8.0 - node-abort-controller@3.1.1: {} - node-addon-api@8.7.0: {} - node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -4745,21 +4611,6 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - prebuild-install@7.1.3: - dependencies: - detect-libc: 2.1.2 - expand-template: 2.0.3 - github-from-package: 0.0.0 - minimist: 1.2.8 - mkdirp-classic: 0.5.3 - napi-build-utils: 2.0.0 - node-abi: 3.92.0 - pump: 3.0.4 - rc: 1.2.8 - simple-get: 4.0.1 - tar-fs: 2.1.4 - tunnel-agent: 0.6.0 - process-warning@4.0.1: {} process-warning@5.0.0: {} @@ -4802,13 +4653,6 @@ snapshots: quick-format-unescaped@4.0.4: {} - rc@1.2.8: - dependencies: - deep-extend: 0.6.0 - ini: 1.3.8 - minimist: 1.2.8 - strip-json-comments: 2.0.1 - readable-stream@3.6.2: dependencies: inherits: 2.0.4 @@ -4921,14 +4765,6 @@ snapshots: signal-exit@4.1.0: {} - simple-concat@1.0.1: {} - - simple-get@4.0.1: - dependencies: - decompress-response: 6.0.0 - once: 1.4.0 - simple-concat: 1.0.1 - sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 @@ -4980,8 +4816,6 @@ snapshots: dependencies: ansi-regex: 6.2.2 - strip-json-comments@2.0.1: {} - strip-json-comments@5.0.3: {} strnum@2.3.0: {} @@ -4994,21 +4828,6 @@ snapshots: tagged-tag@1.0.0: {} - tar-fs@2.1.4: - dependencies: - chownr: 1.1.4 - mkdirp-classic: 0.5.3 - pump: 3.0.4 - tar-stream: 2.2.0 - - tar-stream@2.2.0: - dependencies: - bl: 4.1.0 - end-of-stream: 1.4.5 - fs-constants: 1.0.0 - inherits: 2.0.4 - readable-stream: 3.6.2 - tdigest@0.1.2: dependencies: bintrees: 1.0.2 @@ -5066,10 +4885,6 @@ snapshots: tslib@2.8.1: {} - tunnel-agent@0.6.0: - dependencies: - safe-buffer: 5.2.1 - turbo@2.9.14: optionalDependencies: '@turbo/darwin-64': 2.9.14 From 12897ca6037e1edb4852835d3111322589288b39 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 10:49:59 +0200 Subject: [PATCH 06/23] feat: extract codec into standalone @message-queue-toolkit/codec package Move the zstd codec implementation (ZstdCodecHandler, compressMessageBody, decompressMessageBody, resolveCodecHandler) from packages/sqs into a new dedicated packages/codec package so any adapter can use compression without depending on @message-queue-toolkit/sqs. - Create packages/codec with package.json, tsconfigs, and lib/codec/codecHandler.ts - Delete packages/sqs/lib/codec/sqsCodecHandler.ts - Update sqs and sns to import from @message-queue-toolkit/codec - Re-export codec functions from @message-queue-toolkit/sqs for backwards compatibility - Add @message-queue-toolkit/codec as peer dependency in sqs and sns packages - Remove @mongodb-js/zstd from pnpm-workspace.yaml allowBuilds (no longer used) - Register packages/codec in CI PATH_TO_NAME map - Update SQS and SNS READMEs to document codec as a separate peer dependency Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 1 + packages/codec/README.md | 58 +++++++++++++++++++ .../lib/codec/codecHandler.ts} | 0 packages/codec/lib/index.ts | 6 ++ packages/codec/package.json | 55 ++++++++++++++++++ packages/codec/tsconfig.build.json | 4 ++ packages/codec/tsconfig.json | 4 ++ packages/sns/README.md | 1 + packages/sns/lib/sns/AbstractSnsPublisher.ts | 3 +- packages/sns/package.json | 2 + packages/sqs/README.md | 7 +++ packages/sqs/lib/index.ts | 2 +- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 2 +- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 2 +- packages/sqs/package.json | 2 + .../SqsPermissionConsumer.codec.spec.ts | 2 +- pnpm-lock.yaml | 30 ++++++++++ pnpm-workspace.yaml | 1 - 18 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 packages/codec/README.md rename packages/{sqs/lib/codec/sqsCodecHandler.ts => codec/lib/codec/codecHandler.ts} (100%) create mode 100644 packages/codec/lib/index.ts create mode 100644 packages/codec/package.json create mode 100644 packages/codec/tsconfig.build.json create mode 100644 packages/codec/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1796fc95..9c6cc4bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: run: | declare -A PATH_TO_NAME=( ["packages/amqp"]="@message-queue-toolkit/amqp" + ["packages/codec"]="@message-queue-toolkit/codec" ["packages/core"]="@message-queue-toolkit/core" ["packages/gcp-pubsub"]="@message-queue-toolkit/gcp-pubsub" ["packages/gcs-payload-store"]="@message-queue-toolkit/gcs-payload-store" diff --git a/packages/codec/README.md b/packages/codec/README.md new file mode 100644 index 00000000..0b34c725 --- /dev/null +++ b/packages/codec/README.md @@ -0,0 +1,58 @@ +# @message-queue-toolkit/codec + +Message compression codec implementations for [message-queue-toolkit](https://github.com/kibertoad/message-queue-toolkit). + +This package provides the concrete codec implementations (e.g. zstd) used by the SQS and SNS adapters. The codec interfaces and types (`MessageCodecEnum`, `MessageCodecHandler`, `CodecEnvelope`) live in `@message-queue-toolkit/core`. + +## Installation + +```sh +npm install @message-queue-toolkit/codec @message-queue-toolkit/core +``` + +> **Requirements:** Node.js 22+ (uses the built-in `zlib` zstd support). + +## Usage + +Codec options are typically set on the publisher/consumer constructor in the SQS or SNS adapter packages. You do not need to interact with this package directly unless you are building a custom adapter. + +### Compress / decompress a message body + +```typescript +import { compressMessageBody, decompressMessageBody } from '@message-queue-toolkit/codec' +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +// Compress (returns a JSON string containing the codec envelope) +const compressed = await compressMessageBody(JSON.stringify(payload), MessageCodecEnum.ZSTD) + +// Decompress (parses the envelope and returns the original object) +const original = await decompressMessageBody(JSON.parse(compressed)) +``` + +### Custom codec handler + +```typescript +import type { MessageCodecHandler } from '@message-queue-toolkit/core' + +class MyCodecHandler implements MessageCodecHandler { + compress(data: Buffer): Promise { /* ... */ } + decompress(data: Buffer): Promise { /* ... */ } +} +``` + +## Codec envelope format + +Compressed messages are wrapped in a self-describing JSON envelope: + +```json +{ + "__codec": "zstd", + "__data": "" +} +``` + +Consumers auto-detect this envelope and decompress transparently, even if the `codec` option is not set on the consumer. + +## License + +MIT diff --git a/packages/sqs/lib/codec/sqsCodecHandler.ts b/packages/codec/lib/codec/codecHandler.ts similarity index 100% rename from packages/sqs/lib/codec/sqsCodecHandler.ts rename to packages/codec/lib/codec/codecHandler.ts diff --git a/packages/codec/lib/index.ts b/packages/codec/lib/index.ts new file mode 100644 index 00000000..97ca1042 --- /dev/null +++ b/packages/codec/lib/index.ts @@ -0,0 +1,6 @@ +export { + compressMessageBody, + decompressMessageBody, + resolveCodecHandler, + ZstdCodecHandler, +} from './codec/codecHandler.ts' diff --git a/packages/codec/package.json b/packages/codec/package.json new file mode 100644 index 00000000..811a191d --- /dev/null +++ b/packages/codec/package.json @@ -0,0 +1,55 @@ +{ + "name": "@message-queue-toolkit/codec", + "version": "1.0.0", + "private": false, + "license": "MIT", + "description": "Message compression codec implementations for message-queue-toolkit", + "maintainers": [ + { + "name": "Igor Savin", + "email": "kibertoad@gmail.com" + } + ], + "type": "module", + "main": "./dist/index.js", + "exports": { + ".": "./dist/index.js", + "./package.json": "./package.json" + }, + "scripts": { + "build": "pnpm run clean && tsc --project tsconfig.build.json", + "clean": "rimraf dist", + "lint": "biome check . && tsc", + "lint:fix": "biome check --write .", + "prepublishOnly": "pnpm run lint && pnpm run build" + }, + "peerDependencies": { + "@message-queue-toolkit/core": ">=25.0.0" + }, + "devDependencies": { + "@biomejs/biome": "^2.3.8", + "@lokalise/biome-config": "^3.1.0", + "@lokalise/tsconfig": "^3.0.0", + "@message-queue-toolkit/core": "workspace:*", + "@types/node": "^25.0.2", + "rimraf": "^6.0.1", + "typescript": "^5.9.3" + }, + "homepage": "https://github.com/kibertoad/message-queue-toolkit", + "repository": { + "type": "git", + "url": "git://github.com/kibertoad/message-queue-toolkit.git" + }, + "keywords": [ + "message", + "queue", + "codec", + "compression", + "zstd" + ], + "files": [ + "README.md", + "LICENSE", + "dist/*" + ] +} diff --git a/packages/codec/tsconfig.build.json b/packages/codec/tsconfig.build.json new file mode 100644 index 00000000..198dcfd5 --- /dev/null +++ b/packages/codec/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": ["./tsconfig.json", "@lokalise/tsconfig/build-public-lib"], + "include": ["lib/**/*"] +} diff --git a/packages/codec/tsconfig.json b/packages/codec/tsconfig.json new file mode 100644 index 00000000..8dca583f --- /dev/null +++ b/packages/codec/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@lokalise/tsconfig/tsc", + "include": ["lib/**/*"] +} diff --git a/packages/sns/README.md b/packages/sns/README.md index 34112836..4e38a4e2 100644 --- a/packages/sns/README.md +++ b/packages/sns/README.md @@ -43,6 +43,7 @@ npm install @message-queue-toolkit/sns @message-queue-toolkit/sqs @message-queue - `@aws-sdk/client-sqs` - AWS SDK for SQS (required for consumers) - `@aws-sdk/client-sts` - AWS SDK for STS (for ARN resolution) - `zod` - Schema validation +- `@message-queue-toolkit/codec` - Required when using message compression ## Features diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 81c98269..34713ced 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -2,6 +2,7 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sns' import { PublishCommand } from '@aws-sdk/client-sns' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' +import { compressMessageBody } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -14,7 +15,7 @@ import { type QueuePublisherOptions, type ResolvedMessage, } from '@message-queue-toolkit/core' -import { compressMessageBody, resolveOutgoingMessageAttributes } from '@message-queue-toolkit/sqs' +import { resolveOutgoingMessageAttributes } from '@message-queue-toolkit/sqs' import { calculateOutgoingMessageSize, validateFifoTopicName } from '../utils/snsUtils.ts' diff --git a/packages/sns/package.json b/packages/sns/package.json index f47833f4..f6dfe5e1 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -35,6 +35,7 @@ "@aws-sdk/client-sns": "^3.632.0", "@aws-sdk/client-sqs": "^3.632.0", "@aws-sdk/client-sts": "^3.632.0", + "@message-queue-toolkit/codec": ">=1.0.0", "@message-queue-toolkit/core": ">=24.0.0", "@message-queue-toolkit/schemas": ">=7.0.0", "@message-queue-toolkit/sqs": ">=23.0.0", @@ -47,6 +48,7 @@ "@biomejs/biome": "^2.3.6", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", + "@message-queue-toolkit/codec": "workspace:*", "@message-queue-toolkit/core": "workspace:*", "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", "@message-queue-toolkit/s3-payload-store": "workspace:*", diff --git a/packages/sqs/README.md b/packages/sqs/README.md index f37edd79..e06234eb 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -49,6 +49,7 @@ npm install @message-queue-toolkit/sqs @message-queue-toolkit/core **Peer Dependencies:** - `@aws-sdk/client-sqs` - AWS SDK for SQS - `zod` - Schema validation +- `@message-queue-toolkit/codec` - Required when using message compression ## Features @@ -805,6 +806,12 @@ await publisher.publish({ Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js 22+**. +The codec implementation lives in the separate [`@message-queue-toolkit/codec`](../codec/README.md) package, which must be installed alongside this package when using compression. + +```bash +npm install @message-queue-toolkit/codec +``` + Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __codec: 'zstd', __data: '' }`), so a consumer without `codec` set will still decompress automatically via envelope detection. This allows a gradual rollout — enable compression on the publisher first, consumers adapt without configuration changes. #### Publisher diff --git a/packages/sqs/lib/index.ts b/packages/sqs/lib/index.ts index 7705888f..40828db2 100644 --- a/packages/sqs/lib/index.ts +++ b/packages/sqs/lib/index.ts @@ -3,7 +3,7 @@ export { decompressMessageBody, resolveCodecHandler, ZstdCodecHandler, -} from './codec/sqsCodecHandler.ts' +} from '@message-queue-toolkit/codec' export { SqsConsumerErrorResolver } from './errors/SqsConsumerErrorResolver.ts' export { FakeConsumerErrorResolver } from './fakes/FakeConsumerErrorResolver.ts' export { TestSqsPublisher, type TestSqsPublishOptions } from './fakes/TestSqsPublisher.ts' diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index e45e94f9..9bd8cf7f 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -5,6 +5,7 @@ import { SetQueueAttributesCommand, } from '@aws-sdk/client-sqs' import type { Either, ErrorResolver } from '@lokalise/node-core' +import { decompressMessageBody } from '@message-queue-toolkit/codec' import type { ProcessedMessageMetadata } from '@message-queue-toolkit/core' import { type BarrierResult, @@ -27,7 +28,6 @@ import { import type { ConsumerOptions } from 'sqs-consumer' import { Consumer } from 'sqs-consumer' import type { ZodSchema } from 'zod/v4' -import { decompressMessageBody } from '../codec/sqsCodecHandler.ts' import type { SQSMessage } from '../types/MessageTypes.ts' import { hasOffloadedPayload } from '../utils/messageUtils.ts' import { deleteSqs, initSqs } from '../utils/sqsInitter.ts' diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index f6cc3f7f..9c02ef52 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -2,6 +2,7 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sqs' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' +import { compressMessageBody } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -15,7 +16,6 @@ import { type ResolvedMessage, } from '@message-queue-toolkit/core' import type { ZodSchema } from 'zod/v4' -import { compressMessageBody } from '../codec/sqsCodecHandler.ts' import type { SQSMessage } from '../types/MessageTypes.ts' import { resolveOutgoingMessageAttributes } from '../utils/messageUtils.ts' diff --git a/packages/sqs/package.json b/packages/sqs/package.json index de529971..5eb68739 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -34,6 +34,7 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.632.0", + "@message-queue-toolkit/codec": ">=1.0.0", "@message-queue-toolkit/core": ">=25.0.0", "zod": ">=3.25.76 <5.0.0" }, @@ -43,6 +44,7 @@ "@biomejs/biome": "^2.3.8", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", + "@message-queue-toolkit/codec": "workspace:*", "@message-queue-toolkit/core": "workspace:*", "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", "@message-queue-toolkit/s3-payload-store": "workspace:*", diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index b7229361..de4cf018 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,8 +1,8 @@ import { SendMessageCommand } from '@aws-sdk/client-sqs' +import { compressMessageBody } from '@message-queue-toolkit/codec' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' -import { compressMessageBody } from '../../lib/codec/sqsCodecHandler.ts' import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher.ts' import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 572f62f2..895502f9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -64,6 +64,30 @@ importers: specifier: ^4.1.13 version: 4.4.3 + packages/codec: + devDependencies: + '@biomejs/biome': + specifier: ^2.3.8 + version: 2.4.15 + '@lokalise/biome-config': + specifier: ^3.1.0 + version: 3.1.1 + '@lokalise/tsconfig': + specifier: ^3.0.0 + version: 3.1.0 + '@message-queue-toolkit/core': + specifier: workspace:* + version: link:../core + '@types/node': + specifier: ^25.0.2 + version: 25.8.0 + rimraf: + specifier: ^6.0.1 + version: 6.1.3 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + packages/core: dependencies: '@lokalise/node-core': @@ -497,6 +521,9 @@ importers: '@lokalise/tsconfig': specifier: ^3.0.0 version: 3.1.0 + '@message-queue-toolkit/codec': + specifier: workspace:* + version: link:../codec '@message-queue-toolkit/core': specifier: workspace:* version: link:../core @@ -564,6 +591,9 @@ importers: '@lokalise/tsconfig': specifier: ^3.0.0 version: 3.1.0 + '@message-queue-toolkit/codec': + specifier: workspace:* + version: link:../codec '@message-queue-toolkit/core': specifier: workspace:* version: link:../core diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 35668904..5918a4e3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,7 +6,6 @@ packages: # pnpm 11 auto-appends new entries here whenever a dep has scripts, prompting # an explicit decision per package. allowBuilds: - '@mongodb-js/zstd': true msgpackr-extract: false protobufjs: false From 094dc5df0de39aac3fe07595bda13100bfd92c72 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 10:57:27 +0200 Subject: [PATCH 07/23] fix(codec): tighten Node.js version requirement to >=22.15.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit zlib.zstdCompress/zstdDecompress were added in Node.js v22.15.0 and v23.8.0, not v22.0.0. The previous "Node.js 22+" claim was incorrect and would cause a cryptic TypeError at import time on v22.0.0-v22.14.x. - Add runtime guard in codecHandler.ts that throws a clear error if zstd functions are missing, before promisify() is called - Add engines: { node: ">=22.15.0" } to packages/codec/package.json - Update all JSDoc and README references from "Node.js 22+" to ">=22.15.0" CI matrix (22.x, 24.x) resolves to latest patches which are >=22.15.0 — no change needed. Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/README.md | 2 +- packages/codec/lib/codec/codecHandler.ts | 7 +++++++ packages/codec/package.json | 3 +++ packages/core/lib/codec/messageCodec.ts | 2 +- packages/core/lib/types/queueOptionsTypes.ts | 2 +- packages/sns/README.md | 4 ++-- packages/sqs/README.md | 8 ++++---- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/codec/README.md b/packages/codec/README.md index 0b34c725..6460bc51 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -10,7 +10,7 @@ This package provides the concrete codec implementations (e.g. zstd) used by the npm install @message-queue-toolkit/codec @message-queue-toolkit/core ``` -> **Requirements:** Node.js 22+ (uses the built-in `zlib` zstd support). +> **Requirements:** Node.js >=22.15.0 (uses the built-in `zlib` zstd support). ## Usage diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index b00c18ba..bcdffb3e 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -3,6 +3,13 @@ import zlib from 'node:zlib' import type { CodecEnvelope, MessageCodec, MessageCodecHandler } from '@message-queue-toolkit/core' import { MessageCodecEnum } from '@message-queue-toolkit/core' +if (typeof zlib.zstdCompress !== 'function' || typeof zlib.zstdDecompress !== 'function') { + throw new Error( + 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + + '@message-queue-toolkit/codec requires Node.js >=22.15.0 or >=23.8.0.', + ) +} + const zstdCompress = promisify(zlib.zstdCompress) const zstdDecompress = promisify(zlib.zstdDecompress) diff --git a/packages/codec/package.json b/packages/codec/package.json index 811a191d..aca4b8cf 100644 --- a/packages/codec/package.json +++ b/packages/codec/package.json @@ -23,6 +23,9 @@ "lint:fix": "biome check --write .", "prepublishOnly": "pnpm run lint && pnpm run build" }, + "engines": { + "node": ">=22.15.0" + }, "peerDependencies": { "@message-queue-toolkit/core": ">=25.0.0" }, diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index cc7fab0f..6ecc46dd 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -11,7 +11,7 @@ type ObjectValues = T[keyof T] * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) */ export const MessageCodecEnum = { - /** zstd compression via Node.js built-in `zlib` (requires Node.js 22+). */ + /** zstd compression via Node.js built-in `zlib` (requires Node.js >=22.15.0). */ ZSTD: 'zstd', } as const export type MessageCodec = ObjectValues diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index c8ae6d7c..204f37a8 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -149,7 +149,7 @@ export type CommonQueueOptions = { * Even without this option, consumers auto-detect and decompress any message * that carries a codec envelope, so mixed queues work transparently. * - * Uses Node.js built-in `zlib` zstd support — **requires Node.js 22+**. + * Uses Node.js built-in `zlib` zstd support — **requires Node.js >=22.15.0** (or >=23.8.0). * * @example * import { MessageCodecEnum } from '@message-queue-toolkit/core' diff --git a/packages/sns/README.md b/packages/sns/README.md index 4e38a4e2..1b58d404 100644 --- a/packages/sns/README.md +++ b/packages/sns/README.md @@ -59,7 +59,7 @@ npm install @message-queue-toolkit/sns @message-queue-toolkit/sqs @message-queue - ✅ **Handler spies** for testing - ✅ **Pre-handlers and barriers** for complex message processing - ✅ **Cross-account and cross-region publishing** -- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js 22+ required) +- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js >=22.15.0 required) ## Core Concepts @@ -685,7 +685,7 @@ await consumer.start() // Optional - Payload Offloading (same as SQS) payloadStoreConfig: { /* ... */ }, - // Optional - Compression (Node.js 22+ required) + // Optional - Compression (Node.js >=22.15.0 required) codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd // Optional - Deletion diff --git a/packages/sqs/README.md b/packages/sqs/README.md index e06234eb..1bafa729 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -64,7 +64,7 @@ npm install @message-queue-toolkit/sqs @message-queue-toolkit/core - ✅ **Handler spies** for testing - ✅ **Pre-handlers and barriers** for complex message processing - ✅ **Automatic queue creation** with validation -- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js 22+ required) +- ✅ **Message compression** with zstd via Node.js built-in `zlib` (Node.js >=22.15.0 required) ## Core Concepts @@ -463,7 +463,7 @@ When using `locatorConfig`, you connect to an existing queue without creating it maxPayloadSize: 1024 * 1024, // 1 MiB }, - // Optional - Compression (Node.js 22+ required) + // Optional - Compression (Node.js >=22.15.0 required) codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd // Optional - Deletion @@ -537,7 +537,7 @@ When using `locatorConfig`, you connect to an existing queue without creating it payloadStore: s3Store, }, - // Optional - Compression (Node.js 22+ required) + // Optional - Compression (Node.js >=22.15.0 required) // Auto-detection is always active: consumers decompress codec envelopes // even without this option set. codec: MessageCodecEnum.ZSTD, @@ -804,7 +804,7 @@ await publisher.publish({ ### Message Compression -Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js 22+**. +Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js >=22.15.0**. The codec implementation lives in the separate [`@message-queue-toolkit/codec`](../codec/README.md) package, which must be installed alongside this package when using compression. From ff779b5076cd8298b8d195600f69e1baeec1aa50 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 14:49:50 +0200 Subject: [PATCH 08/23] refactor: compress-once in publish, split offload into focused helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes double compression: previously publish() delegated to offloadMessagePayloadIfNeeded which compressed the payload to check the size threshold, then returned the original message, and sendMessage() compressed again. Now when a codec is set, the message is compressed exactly once at publish() entry point, regardless of whether offloading is also configured. The same compressed Buffer is then either: - stored in S3 and replaced with a pointer (if compressed size exceeds messageSizeThreshold), or - wrapped in a codec envelope and sent inline (if it fits). The payload is never compressed twice. Key changes: - codec: add buildCodecEnvelope(compressed, codec) to wrap pre-compressed bytes without re-compressing - core: replace offloadMessagePayloadIfNeeded with three focused methods: - private buildPointer() — shared pointer construction logic - protected offloadPayload() — no-codec path, returns null if fits - protected offloadCompressedPayload() — codec path, always stores - sqs/sns: restructure publish() via private prepareOutgoingPayload() that compresses once and branches; sendMessage() accepts preBuiltBody to skip re-serialization - gcp-pubsub: migrate to offloadPayload(), pin core to workspace:* - docs: update SQS, SNS, core, and codec READMEs to explain the single compression pass and how codec interacts with payload offloading Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/README.md | 26 ++++ packages/codec/lib/codec/codecHandler.ts | 8 + packages/codec/lib/index.ts | 1 + packages/core/README.md | 11 ++ .../offloadedPayloadMessageSchemas.ts | 6 + .../core/lib/queues/AbstractQueueService.ts | 146 ++++++++++++------ packages/core/lib/utils/streamUtils.ts | 9 +- .../AbstractQueueService.offload.spec.ts | 6 +- .../lib/pubsub/AbstractPubSubPublisher.ts | 11 +- packages/gcp-pubsub/package.json | 2 +- packages/sns/lib/sns/AbstractSnsPublisher.ts | 53 +++++-- packages/sqs/README.md | 23 ++- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 6 +- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 54 +++++-- .../SqsPermissionConsumer.codec.spec.ts | 9 +- ...rmissionConsumer.payloadOffloading.spec.ts | 109 ++++++++++++- pnpm-lock.yaml | 16 +- 17 files changed, 396 insertions(+), 100 deletions(-) diff --git a/packages/codec/README.md b/packages/codec/README.md index 6460bc51..22b082dc 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -16,6 +16,16 @@ npm install @message-queue-toolkit/codec @message-queue-toolkit/core Codec options are typically set on the publisher/consumer constructor in the SQS or SNS adapter packages. You do not need to interact with this package directly unless you are building a custom adapter. +### How compression works during publish + +When `codec` is set on a publisher, compression happens **exactly once** at the start of `publish()`, before any other processing: + +1. The message JSON is compressed to a raw `Buffer`. +2. If a payload store is configured **and** the compressed size exceeds `messageSizeThreshold`, the compressed bytes are stored in S3 and only a lightweight pointer is sent. The codec name is recorded in `payloadRef.codec` so the consumer can decompress after retrieval. +3. If the compressed size fits within the threshold (or no store is configured), the message is sent inline as a self-describing codec envelope. + +The payload is never compressed twice. The same compressed `Buffer` from step 1 is either uploaded to S3 or wrapped in the envelope — whichever path is taken. + ### Compress / decompress a message body ```typescript @@ -29,6 +39,22 @@ const compressed = await compressMessageBody(JSON.stringify(payload), MessageCod const original = await decompressMessageBody(JSON.parse(compressed)) ``` +### Build a codec envelope from already-compressed bytes + +When you have pre-compressed bytes (e.g., from `resolveCodecHandler(codec).compress(...)`) and want to produce the envelope string without compressing again: + +```typescript +import { buildCodecEnvelope } from '@message-queue-toolkit/codec' +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +const handler = resolveCodecHandler(MessageCodecEnum.ZSTD) +const compressed: Buffer = await handler.compress(Buffer.from(JSON.stringify(payload), 'utf8')) + +// Build envelope without a second compression pass +const envelopeString = buildCodecEnvelope(compressed, MessageCodecEnum.ZSTD) +// → '{"__codec":"zstd","__data":""}' +``` + ### Custom codec handler ```typescript diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index bcdffb3e..e39eebc0 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -33,6 +33,14 @@ export function resolveCodecHandler(codec: MessageCodec): MessageCodecHandler { export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { const handler = resolveCodecHandler(codec) const compressed = await handler.compress(Buffer.from(jsonBody, 'utf8')) + return buildCodecEnvelope(compressed, codec) +} + +/** + * Wraps an already-compressed buffer in a codec envelope string. + * Use this when you have pre-compressed bytes and want to avoid compressing twice. + */ +export function buildCodecEnvelope(compressed: Buffer, codec: MessageCodec): string { const envelope: CodecEnvelope = { __codec: codec, __data: compressed.toString('base64'), diff --git a/packages/codec/lib/index.ts b/packages/codec/lib/index.ts index 97ca1042..ed36e669 100644 --- a/packages/codec/lib/index.ts +++ b/packages/codec/lib/index.ts @@ -1,4 +1,5 @@ export { + buildCodecEnvelope, compressMessageBody, decompressMessageBody, resolveCodecHandler, diff --git a/packages/core/README.md b/packages/core/README.md index 125a1285..ce1139be 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -641,6 +641,17 @@ class MyPayloadStore implements PayloadStore { } ``` +#### Interaction with codec (compression) + +When both `codec` and `payloadStoreConfig` are set on a publisher, compression and offloading work together with a single compression pass: + +1. The message is compressed **once** at publish time. +2. The **compressed** size is compared against `messageSizeThreshold`. +3. If the compressed size exceeds the threshold, the raw compressed bytes are stored in the payload store. The codec name is written to `payloadRef.codec` so the consumer knows how to decompress after retrieval. +4. If the compressed size fits within the threshold, the message is sent inline as a self-describing codec envelope — S3 is never touched. + +This means compression can prevent offloading entirely for messages that are large before compression but small after. + ## API Reference ### Types diff --git a/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts b/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts index fa62f183..29bc770f 100644 --- a/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts +++ b/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts @@ -11,6 +11,12 @@ export const PAYLOAD_REF_SCHEMA = z.object({ store: z.string().min(1), /** Size of the payload in bytes */ size: z.number().int().positive(), + /** + * Codec used to compress the stored payload. + * When set, the stored bytes are raw compressed binary (not base64 JSON). + * The consumer must decompress using this codec before parsing. + */ + codec: z.string().optional(), }) export type PayloadRef = z.output diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index ca305d0c..c42c926a 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -1,3 +1,4 @@ +import { Readable } from 'node:stream' import { types } from 'node:util' import { type CommonLogger, @@ -49,7 +50,7 @@ import type { QueueOptions, } from '../types/queueOptionsTypes.ts' import { isRetryDateExceeded } from '../utils/dateUtils.ts' -import { streamWithKnownSizeToString } from '../utils/streamUtils.ts' +import { streamWithKnownSizeToBuffer, streamWithKnownSizeToString } from '../utils/streamUtils.ts' import { toDatePreprocessor } from '../utils/toDateProcessor.ts' import type { BarrierCallback, @@ -660,49 +661,30 @@ export abstract class AbstractQueueService< } /** - * Offload message payload to an external store if it exceeds the threshold. - * Returns a special type that contains a pointer to the offloaded payload or the original payload if it was not offloaded. - * Requires message size as only the implementation knows how to calculate it. + * Builds an OffloadedPayloadPointerPayload from the given message and storage metadata. + * Copies identity fields and preserves the message type field through offloading. * - * For multi-store configuration, uses the configured outgoingStore. - * For single-store configuration, uses the single store. - * - * The returned payload includes both the new payloadRef format and legacy fields for backward compatibility. + * We default to the conventional top-level `type` path so that routing/identity fields are + * handled consistently with `messageIdField`/`messageTimestampField`/etc. Without this + * fallback, `messageTypeResolver` modes that don't specify a body path silently strip `type` + * from the offloaded body, breaking downstream SNS subscription FilterPolicy filters. */ - protected async offloadMessagePayloadIfNeeded( + private buildPointer( message: MessagePayloadSchemas, - messageSizeFn: () => number, - ): Promise { - if ( - !this.payloadStoreConfig || - messageSizeFn() <= this.payloadStoreConfig.messageSizeThreshold - ) { - return message - } - - const { store, storeName } = this.resolveOutgoingStore() - const serializedPayload = await this.payloadStoreConfig.serializer.serialize(message) - - let payloadId: string - try { - payloadId = await store.storePayload(serializedPayload) - } finally { - if (isDestroyable(serializedPayload)) { - await serializedPayload.destroy() - } - } - - // Return message with both new and legacy formats for backward compatibility + payloadId: string, + storeName: string, + size: number, + codec?: MessageCodec, + ): OffloadedPayloadPointerPayload { const result: OffloadedPayloadPointerPayload = { - // Extended payload reference format payloadRef: { id: payloadId, store: storeName, - size: serializedPayload.size, + size, + ...(codec ? { codec } : {}), }, - // Legacy format for backward compatibility offloadedPayloadPointer: payloadId, - offloadedPayloadSize: serializedPayload.size, + offloadedPayloadSize: size, // @ts-expect-error [this.messageIdField]: message[this.messageIdField], // @ts-expect-error @@ -713,14 +695,6 @@ export abstract class AbstractQueueService< [this.messageDeduplicationOptionsField]: message[this.messageDeduplicationOptionsField], } - // Preserve the message type field through offloading. We default to the conventional - // top-level `type` path so that routing/identity fields are handled consistently with - // `messageIdField`/`messageTimestampField`/etc., which have defaulted names ('id', - // 'timestamp', ...) and are always copied across when present. Without this fallback, - // `messageTypeResolver` modes that don't specify a body path (no resolver, `literal`, - // or `resolver`) silently strip `type` from the offloaded SNS body, which then breaks - // any downstream subscription whose FilterPolicy filters on `type` - // (FilterPolicyScope: 'MessageBody'). const typePath = this.messageTypeResolver && isMessageTypePathConfig(this.messageTypeResolver) ? this.messageTypeResolver.messageTypePath @@ -733,14 +707,82 @@ export abstract class AbstractQueueService< return result } + /** + * Offloads the message payload to the configured store if it exceeds the size threshold. + * Returns null if no offloading is needed (store not configured or message fits within threshold). + * + * For multi-store configuration, uses the configured outgoingStore. + * For single-store configuration, uses the single store. + * + * The returned pointer includes both the new payloadRef format and legacy fields for backward + * compatibility. The message type field is always preserved through offloading. + */ + protected async offloadPayload( + message: MessagePayloadSchemas, + messageSizeFn: () => number, + ): Promise { + if (!this.payloadStoreConfig) { + return null + } + + if (messageSizeFn() <= this.payloadStoreConfig.messageSizeThreshold) { + return null + } + + const { store, storeName } = this.resolveOutgoingStore() + const serializedPayload = await this.payloadStoreConfig.serializer.serialize(message) + + let payloadId: string + try { + payloadId = await store.storePayload(serializedPayload) + } finally { + if (isDestroyable(serializedPayload)) { + await serializedPayload.destroy() + } + } + + return this.buildPointer(message, payloadId, storeName, serializedPayload.size) + } + + /** + * Stores an already-compressed payload in the configured store. + * The `codec` name is recorded in payloadRef so the consumer can decompress after retrieval. + * + * The threshold check is NOT performed here — callers must decide whether to offload. + * Use this when compression has already been done and the compressed size exceeds the threshold. + * + * @throws Error if payload store is not configured + */ + protected async offloadCompressedPayload( + message: MessagePayloadSchemas, + compressed: Buffer, + codec: MessageCodec, + ): Promise { + if (!this.payloadStoreConfig) { + throw new Error('Payload store is not configured') + } + + const { store, storeName } = this.resolveOutgoingStore() + const payloadId = await store.storePayload({ + value: Readable.from(compressed), + size: compressed.byteLength, + }) + + return this.buildPointer(message, payloadId, storeName, compressed.byteLength, codec) + } + /** * Retrieve previously offloaded message payload using provided pointer payload. * Returns the original payload or an error if the payload was not found or could not be parsed. * * Supports both new multi-store format (payloadRef) and legacy format (offloadedPayloadPointer). + * + * When `decompress` is provided and the pointer's `payloadRef.codec` matches, the fetched bytes + * are treated as raw compressed binary and decompressed before JSON parsing. */ protected async retrieveOffloadedMessagePayload( maybeOffloadedPayloadPointerPayload: unknown, + decompress?: (codec: string, data: Buffer) => Promise, ): Promise> { if (!this.payloadStoreConfig) { return { @@ -790,6 +832,24 @@ export abstract class AbstractQueueService< } } + const codec = parsedPayload.payloadRef?.codec + if (codec && decompress) { + try { + const compressedBuffer = await streamWithKnownSizeToBuffer( + serializedOffloadedPayloadReadable, + payloadSize, + ) + const decompressed = await decompress(codec, compressedBuffer) + return { result: JSON.parse(decompressed.toString('utf8')) } + } catch (e) { + return { + error: new Error(`Failed to decompress offloaded payload with codec "${codec}"`, { + cause: e, + }), + } + } + } + const serializedOffloadedPayloadString = await streamWithKnownSizeToString( serializedOffloadedPayloadReadable, payloadSize, diff --git a/packages/core/lib/utils/streamUtils.ts b/packages/core/lib/utils/streamUtils.ts index 7ea2155f..7e69617b 100644 --- a/packages/core/lib/utils/streamUtils.ts +++ b/packages/core/lib/utils/streamUtils.ts @@ -1,6 +1,6 @@ import type { Readable } from 'node:stream' -export async function streamWithKnownSizeToString(stream: Readable, size: number): Promise { +export async function streamWithKnownSizeToBuffer(stream: Readable, size: number): Promise { const buffer = Buffer.alloc(size) let offset = 0 @@ -14,5 +14,10 @@ export async function streamWithKnownSizeToString(stream: Readable, size: number offset += chunkBuffer.length } - return buffer.toString('utf8', 0, offset) + return buffer.subarray(0, offset) +} + +export async function streamWithKnownSizeToString(stream: Readable, size: number): Promise { + const buffer = await streamWithKnownSizeToBuffer(stream, size) + return buffer.toString('utf8') } diff --git a/packages/core/test/queues/AbstractQueueService.offload.spec.ts b/packages/core/test/queues/AbstractQueueService.offload.spec.ts index 2d6142ac..32167f30 100644 --- a/packages/core/test/queues/AbstractQueueService.offload.spec.ts +++ b/packages/core/test/queues/AbstractQueueService.offload.spec.ts @@ -1,5 +1,5 @@ /** - * Regression tests for `AbstractQueueService.offloadMessagePayloadIfNeeded`. + * Regression tests for `AbstractQueueService.offloadPayload`. * * Identity fields (`messageIdField`, `messageTimestampField`, `messageDeduplicationIdField`, * `messageDeduplicationOptionsField`) all have defaulted names ('id', 'timestamp', ...) and @@ -58,7 +58,7 @@ class TestQueueService extends AbstractQueueService< // Expose protected method for direct testing. public callOffload(message: TestMessage, sizeFn: () => number) { - return this.offloadMessagePayloadIfNeeded(message, sizeFn) + return this.offloadPayload(message, sizeFn) } } @@ -102,7 +102,7 @@ const baseMessage: TestMessage = { payload: { large: 'data' }, } -describe('AbstractQueueService.offloadMessagePayloadIfNeeded — `type` preservation', () => { +describe('AbstractQueueService.offloadPayload — `type` preservation', () => { it('preserves `type` when no messageTypeResolver is configured', async () => { const svc = buildService(undefined) const result = (await svc.callOffload( diff --git a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts index 5927afa0..067fdc58 100644 --- a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts +++ b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts @@ -69,11 +69,12 @@ export abstract class AbstractPubSubPublisher const parsedMessage = messageSchemaResult.result.parse(message) message = this.updateInternalProperties(message) - const maybeOffloadedPayloadMessage = await this.offloadMessagePayloadIfNeeded(message, () => { - // Calculate message size for PubSub - const messageData = Buffer.from(JSON.stringify(message)) - return messageData.length - }) + const maybeOffloadedPayloadMessage = + (await this.offloadPayload(message, () => { + // Calculate message size for PubSub + const messageData = Buffer.from(JSON.stringify(message)) + return messageData.length + })) ?? message if ( this.isDeduplicationEnabledForMessage(parsedMessage) && diff --git a/packages/gcp-pubsub/package.json b/packages/gcp-pubsub/package.json index e731daef..b43d284d 100644 --- a/packages/gcp-pubsub/package.json +++ b/packages/gcp-pubsub/package.json @@ -40,7 +40,7 @@ "@biomejs/biome": "^2.3.8", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/core": "*", + "@message-queue-toolkit/core": "workspace:*", "@message-queue-toolkit/gcs-payload-store": "*", "@message-queue-toolkit/redis-message-deduplication-store": "*", "@message-queue-toolkit/schemas": "*", diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 34713ced..090f27b1 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -2,12 +2,11 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sns' import { PublishCommand } from '@aws-sdk/client-sns' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { compressMessageBody } from '@message-queue-toolkit/codec' +import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, DeduplicationRequesterEnum, - isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, type MessageSchemaContainer, type MessageValidationError, @@ -133,10 +132,7 @@ export abstract class AbstractSnsPublisher // (offloaded payload won't have user fields needed for messageGroupIdField) const resolvedOptions = this.resolveFifoOptions(updatedMessage, options) - const maybeOffloadedPayloadMessage = await this.offloadMessagePayloadIfNeeded( - updatedMessage, - () => calculateOutgoingMessageSize(updatedMessage), - ) + const { payload, preBuiltBody } = await this.prepareOutgoingPayload(updatedMessage) if ( this.isDeduplicationEnabledForMessage(parsedMessage) && @@ -152,7 +148,7 @@ export abstract class AbstractSnsPublisher return } - await this.sendMessage(maybeOffloadedPayloadMessage, resolvedOptions) + await this.sendMessage(payload, resolvedOptions, preBuiltBody) this.handleMessageProcessed({ message: parsedMessage, @@ -208,16 +204,49 @@ export abstract class AbstractSnsPublisher return this.isDeduplicationEnabled && super.isDeduplicationEnabledForMessage(message) } + /** + * Compresses (when codec is set) or offloads (when store is configured) the message. + * Returns the payload to send and an optional pre-built body string. + * When preBuiltBody is set, it is a ready-to-send codec envelope — sendMessage must use it as-is. + */ + private async prepareOutgoingPayload(message: MessagePayloadType): Promise<{ + payload: MessagePayloadType | OffloadedPayloadPointerPayload + preBuiltBody?: string + }> { + const codec = this.codec + + if (codec) { + // Compress once up-front, then decide: offload the compressed bytes or send inline. + const compressed = await resolveCodecHandler(codec).compress( + Buffer.from(JSON.stringify(message), 'utf8'), + ) + + if ( + this.payloadStoreConfig && + compressed.byteLength > this.payloadStoreConfig.messageSizeThreshold + ) { + return { payload: await this.offloadCompressedPayload(message, compressed, codec) } + } + + return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codec) } + } + + return { + payload: + (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? + message, + } + } + protected async sendMessage( payload: MessagePayloadType | OffloadedPayloadPointerPayload, options: SNSMessageOptions, + preBuiltBody?: string, ): Promise { const attributes = resolveOutgoingMessageAttributes(payload) - const jsonBody = JSON.stringify(payload) - const body = - this.codec && !isOffloadedPayloadPointerPayload(payload) - ? await compressMessageBody(jsonBody, this.codec) - : jsonBody + // preBuiltBody is set when codec is active and the payload was not offloaded — + // it contains the already-compressed codec envelope, so we skip re-serialization. + const body = preBuiltBody ?? JSON.stringify(payload) const command = new PublishCommand({ Message: body, MessageAttributes: attributes, diff --git a/packages/sqs/README.md b/packages/sqs/README.md index bac33c86..9eebaa7b 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -799,13 +799,20 @@ await publisher.publish({ }) ``` -**How it works:** +**How it works (without codec):** 1. Publisher checks message size before sending -2. If size exceeds `maxPayloadSize`, stores payload in S3 -3. Replaces payload with pointer: `{ _offloadedPayload: { bucketName, key, size } }` -4. Sends pointer message to SQS -5. Consumer detects pointer, fetches payload from S3 -6. Processes message with full payload +2. If size exceeds `messageSizeThreshold`, serializes and stores payload in S3 +3. Sends a lightweight pointer message to SQS instead +4. Consumer detects the pointer, fetches payload from S3 +5. Processes message with full payload + +**How it works (with codec — compress + offload):** +1. Publisher compresses the serialized message with zstd **once**, up-front +2. If the **compressed** size exceeds `messageSizeThreshold`, stores the compressed bytes in S3 and sends a pointer +3. If the compressed size fits within the threshold, sends the message inline as a codec envelope +4. Consumer fetches the pointer payload as raw bytes, decompresses, then processes as normal + +The codec embedded in `payloadRef.codec` tells the consumer which algorithm to use — no `codec` option is needed on the consumer. **Note:** Payload cleanup is the responsibility of the store (e.g., S3 lifecycle policies). @@ -860,7 +867,9 @@ class MyConsumer extends AbstractSqsConsumer { + const handler = resolveCodecHandler(codec as Parameters[0]) + return handler.decompress(data) + }, ) if (retrieveOffloadedMessagePayloadResult.error) { this.handleError(retrieveOffloadedMessagePayloadResult.error) diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index 9c02ef52..29d8b9c8 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -2,12 +2,11 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sqs' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { compressMessageBody } from '@message-queue-toolkit/codec' +import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, DeduplicationRequesterEnum, - isOffloadedPayloadPointerPayload, type MessageInvalidFormatError, type MessageSchemaContainer, type MessageValidationError, @@ -125,9 +124,7 @@ export abstract class AbstractSqsPublisher // (offloaded payload won't have user fields needed for messageGroupIdField) const resolvedOptions = this.resolveFifoOptions(message, options) - const maybeOffloadedPayloadMessage = await this.offloadMessagePayloadIfNeeded(message, () => - calculateOutgoingMessageSize(message), - ) + const { payload, preBuiltBody } = await this.prepareOutgoingPayload(message) if ( this.isDeduplicationEnabledForMessage(parsedMessage) && @@ -143,7 +140,7 @@ export abstract class AbstractSqsPublisher return } - await this.sendMessage(maybeOffloadedPayloadMessage, resolvedOptions) + await this.sendMessage(payload, resolvedOptions, preBuiltBody) this.handleMessageProcessed({ message: parsedMessage, processingResult: { status: 'published' }, @@ -201,18 +198,49 @@ export abstract class AbstractSqsPublisher return this.messageSchemaContainer.resolveSchema(message) } + /** + * Compresses (when codec is set) or offloads (when store is configured) the message. + * Returns the payload to send and an optional pre-built body string. + * When preBuiltBody is set, it is a ready-to-send codec envelope — sendMessage must use it as-is. + */ + private async prepareOutgoingPayload(message: MessagePayloadType): Promise<{ + payload: MessagePayloadType | OffloadedPayloadPointerPayload + preBuiltBody?: string + }> { + const codec = this.codec + + if (codec) { + // Compress once up-front, then decide: offload the compressed bytes or send inline. + const compressed = await resolveCodecHandler(codec).compress( + Buffer.from(JSON.stringify(message), 'utf8'), + ) + + if ( + this.payloadStoreConfig && + compressed.byteLength > this.payloadStoreConfig.messageSizeThreshold + ) { + return { payload: await this.offloadCompressedPayload(message, compressed, codec) } + } + + return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codec) } + } + + return { + payload: + (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? + message, + } + } + protected async sendMessage( payload: MessagePayloadType | OffloadedPayloadPointerPayload, options: SQSMessageOptions, + preBuiltBody?: string, ): Promise { const attributes = resolveOutgoingMessageAttributes(payload) - const jsonBody = JSON.stringify(payload) - const body = - this.codec && !isOffloadedPayloadPointerPayload(payload) - ? await compressMessageBody(jsonBody, this.codec) - : jsonBody - - // Options are already resolved in publish() before offloading + // preBuiltBody is set when codec is active and the payload was not offloaded — + // it contains the already-compressed codec envelope, so we skip re-serialization. + const body = preBuiltBody ?? JSON.stringify(payload) const command = new SendMessageCommand({ QueueUrl: this.queueUrl, MessageBody: body, diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index de4cf018..9ac1b3a8 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,5 +1,6 @@ import { SendMessageCommand } from '@aws-sdk/client-sqs' import { compressMessageBody } from '@message-queue-toolkit/codec' +import { MessageCodecEnum } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' @@ -29,11 +30,11 @@ describe('SqsPermissionConsumer - zstd codec', () => { await testAdmin.deleteQueues(SqsPermissionConsumer.QUEUE_NAME) consumer = new SqsPermissionConsumer(diContainer.cradle, { - codec: 'zstd', + codec: MessageCodecEnum.ZSTD, deletionConfig: { deleteIfExists: false }, }) publisher = new SqsPermissionPublisher(diContainer.cradle, { - codec: 'zstd', + codec: MessageCodecEnum.ZSTD, }) await consumer.start() @@ -89,7 +90,7 @@ describe('SqsPermissionConsumer - zstd codec', () => { } // Simulate a publisher that compressed the message itself - const compressedBody = await compressMessageBody(JSON.stringify(message), 'zstd') + const compressedBody = await compressMessageBody(JSON.stringify(message), MessageCodecEnum.ZSTD) await diContainer.cradle.sqsClient.send( new SendMessageCommand({ QueueUrl: consumer.queueProps.url, @@ -108,7 +109,7 @@ describe('SqsPermissionConsumer - zstd codec', () => { await testAdmin.deleteQueues(autoQueueName) const autoPublisher = new SqsPermissionPublisher(diContainer.cradle, { - codec: 'zstd', + codec: MessageCodecEnum.ZSTD, creationConfig: { queue: { QueueName: autoQueueName } }, }) await autoPublisher.init() diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts index c53ff481..370901a0 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts @@ -1,7 +1,7 @@ import type { S3 } from '@aws-sdk/client-s3' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { SinglePayloadStoreConfig } from '@message-queue-toolkit/core' -import { MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' +import { MessageCodecEnum, MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' import { S3PayloadStore } from '@message-queue-toolkit/s3-payload-store' import { OFFLOADED_PAYLOAD_SIZE_ATTRIBUTE } from '@message-queue-toolkit/sqs' import type { AwilixContainer } from 'awilix' @@ -401,3 +401,110 @@ describe('SqsPermissionConsumer - nested messageTypePath with payload offloading }) }) }) + +describe('SqsPermissionConsumer - codec + payload offloading', () => { + const s3BucketName = 'test-bucket-codec' + // Threshold low enough that even a small compressed payload triggers offloading + const smallThreshold = 10 + + let diContainer: AwilixContainer + let s3: S3 + let testAdmin: TestAwsResourceAdmin + let payloadStoreConfig: SinglePayloadStoreConfig + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + s3 = diContainer.cradle.s3 + testAdmin = diContainer.cradle.testAdmin + + await testAdmin.createBucket(s3BucketName) + payloadStoreConfig = { + messageSizeThreshold: smallThreshold, + store: new S3PayloadStore(diContainer.cradle, { bucketName: s3BucketName }), + storeName: 's3', + } + }) + + afterAll(async () => { + await testAdmin.emptyBuckets(s3BucketName) + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('compresses payload, offloads to S3 as raw binary, and consumer decompresses correctly', async () => { + const queueName = 'codec-offload-roundtrip' + await testAdmin.deleteQueues(queueName) + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-offload-1', + messageType: 'add', + metadata: { info: 'compressed and offloaded' }, + } + + const publisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + payloadStoreConfig, + creationConfig: { queue: { QueueName: queueName } }, + }) + // No codec on consumer — codec is read from payloadRef.codec in the pointer + const consumer = new SqsPermissionConsumer(diContainer.cradle, { + payloadStoreConfig, + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + }) + + await publisher.init() + await consumer.start() + + await publisher.publish(message) + + // Verify payload was offloaded to S3 + const s3Keys = await waitForS3Objects(s3, s3BucketName, 1, 5000) + expect(s3Keys.length).toBeGreaterThan(0) + + // Verify consumer receives the correct decompressed payload + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await publisher.close() + await consumer.close(true) + }, 30_000) + + it('consumer without explicit codec still decompresses codec-offloaded payload', async () => { + const queueName = 'codec-offload-auto-detect' + await testAdmin.deleteQueues(queueName) + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-offload-auto-1', + messageType: 'add', + metadata: { info: 'auto-detect codec from pointer' }, + } + + const publisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + payloadStoreConfig, + creationConfig: { queue: { QueueName: queueName } }, + }) + // Consumer has no explicit codec — should still work because codec comes from payloadRef.codec + const consumer = new SqsPermissionConsumer(diContainer.cradle, { + payloadStoreConfig, + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + }) + + await publisher.init() + await consumer.start() + + await publisher.publish(message) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await publisher.close() + await consumer.close(true) + }, 30_000) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eeda1f46..cf6ea3d5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -174,14 +174,14 @@ importers: specifier: ^3.0.0 version: 3.1.0 '@message-queue-toolkit/core': - specifier: '*' - version: 25.5.0(zod@4.4.3) + specifier: workspace:* + version: link:../core '@message-queue-toolkit/gcs-payload-store': specifier: '*' - version: 1.0.0(@google-cloud/storage@7.19.0)(@message-queue-toolkit/core@25.5.0(zod@4.4.3)) + version: 1.0.0(@google-cloud/storage@7.19.0)(@message-queue-toolkit/core@packages+core) '@message-queue-toolkit/redis-message-deduplication-store': specifier: '*' - version: 2.0.2(@message-queue-toolkit/core@25.5.0(zod@4.4.3))(ioredis@5.10.1)(zod@4.4.3) + version: 2.0.2(@message-queue-toolkit/core@packages+core)(ioredis@5.10.1)(zod@4.4.3) '@message-queue-toolkit/schemas': specifier: '*' version: 7.1.0(zod@4.4.3) @@ -3374,15 +3374,15 @@ snapshots: toad-cache: 3.7.0 zod: 4.4.3 - '@message-queue-toolkit/gcs-payload-store@1.0.0(@google-cloud/storage@7.19.0)(@message-queue-toolkit/core@25.5.0(zod@4.4.3))': + '@message-queue-toolkit/gcs-payload-store@1.0.0(@google-cloud/storage@7.19.0)(@message-queue-toolkit/core@packages+core)': dependencies: '@google-cloud/storage': 7.19.0 - '@message-queue-toolkit/core': 25.5.0(zod@4.4.3) + '@message-queue-toolkit/core': link:packages/core - '@message-queue-toolkit/redis-message-deduplication-store@2.0.2(@message-queue-toolkit/core@25.5.0(zod@4.4.3))(ioredis@5.10.1)(zod@4.4.3)': + '@message-queue-toolkit/redis-message-deduplication-store@2.0.2(@message-queue-toolkit/core@packages+core)(ioredis@5.10.1)(zod@4.4.3)': dependencies: '@lokalise/node-core': 14.8.1(zod@4.4.3) - '@message-queue-toolkit/core': 25.5.0(zod@4.4.3) + '@message-queue-toolkit/core': link:packages/core ioredis: 5.10.1 redis-semaphore: 5.7.0(ioredis@5.10.1) transitivePeerDependencies: From 989165f392d19468f3541192fed68f59c1c7e975 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Tue, 19 May 2026 15:31:40 +0200 Subject: [PATCH 09/23] test: verify wire format of compressed messages in integration tests For the inline codec path, read the raw SQS message body from an isolated queue (no consumer) using ReceiveMessageCommand and assert: - body is a JSON codec envelope with __codec === 'zstd' - __data decodes from base64 to a valid zstd frame (magic bytes 28 B5 2F FD) For the codec + payload offloading path, assert: - SQS message body is a plain JSON pointer (no __codec field), with payloadRef.codec === 'zstd' confirming which algorithm was used - S3 object contains raw compressed binary, not a JSON envelope (first 4 bytes match the zstd magic number 0xFD2FB528) Also add getObjectBuffer() to s3Utils for reading S3 objects as raw Buffer without UTF-8 decoding, and fix missing resolveCodecHandler import in codec README example snippet. Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/README.md | 2 +- .../SqsPermissionConsumer.codec.spec.ts | 42 ++++++++++++++- ...rmissionConsumer.payloadOffloading.spec.ts | 53 +++++++++++++++++-- packages/sqs/test/utils/s3Utils.ts | 7 +++ 4 files changed, 99 insertions(+), 5 deletions(-) diff --git a/packages/codec/README.md b/packages/codec/README.md index 22b082dc..d915544f 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -44,7 +44,7 @@ const original = await decompressMessageBody(JSON.parse(compressed)) When you have pre-compressed bytes (e.g., from `resolveCodecHandler(codec).compress(...)`) and want to produce the envelope string without compressing again: ```typescript -import { buildCodecEnvelope } from '@message-queue-toolkit/codec' +import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' import { MessageCodecEnum } from '@message-queue-toolkit/core' const handler = resolveCodecHandler(MessageCodecEnum.ZSTD) diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index 9ac1b3a8..43f44493 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,4 +1,4 @@ -import { SendMessageCommand } from '@aws-sdk/client-sqs' +import { ReceiveMessageCommand, SendMessageCommand } from '@aws-sdk/client-sqs' import { compressMessageBody } from '@message-queue-toolkit/codec' import { MessageCodecEnum } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' @@ -65,6 +65,46 @@ describe('SqsPermissionConsumer - zstd codec', () => { expect(result.message).toMatchObject(message) }) + it('published SQS message body is a codec envelope containing valid zstd bytes', async () => { + // Use an isolated queue with no consumer so we can read the raw message without a race + const wireQueueName = `${SqsPermissionConsumer.QUEUE_NAME}-wire-check` + await testAdmin.deleteQueues(wireQueueName) + + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + creationConfig: { queue: { QueueName: wireQueueName } }, + }) + await wirePublisher.init() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-wire-1', + messageType: 'add', + } + await wirePublisher.publish(message) + + // Read the raw message directly from SQS — no consumer is running on this queue + const { Messages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + expect(Messages, 'Expected a message to be in the queue').toBeDefined() + expect(Messages!.length).toBe(1) + + // Body must be a self-describing codec envelope, not raw message JSON + const envelope = JSON.parse(Messages![0]!.Body!) as Record + expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__data).toBe('string') + + // __data must decode to a valid zstd frame: magic number 0xFD2FB528 (LE → 28 B5 2F FD) + const compressed = Buffer.from(envelope.__data as string, 'base64') + expect(compressed.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) + + await wirePublisher.close() + }) + it('consumer correctly handles multiple compressed messages in sequence', async () => { const messages: PERMISSIONS_ADD_MESSAGE_TYPE[] = [ { id: 'codec-seq-1', messageType: 'add' }, diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts index 370901a0..d41bad0c 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts @@ -1,5 +1,5 @@ import type { S3 } from '@aws-sdk/client-s3' -import { SendMessageCommand } from '@aws-sdk/client-sqs' +import { ReceiveMessageCommand, SendMessageCommand } from '@aws-sdk/client-sqs' import type { SinglePayloadStoreConfig } from '@message-queue-toolkit/core' import { MessageCodecEnum, MessageHandlerConfigBuilder } from '@message-queue-toolkit/core' import { S3PayloadStore } from '@message-queue-toolkit/s3-payload-store' @@ -13,7 +13,7 @@ import { AbstractSqsConsumer } from '../../lib/sqs/AbstractSqsConsumer.ts' import { AbstractSqsPublisher } from '../../lib/sqs/AbstractSqsPublisher.ts' import { SQS_MESSAGE_MAX_SIZE } from '../../lib/sqs/AbstractSqsService.ts' import { SqsPermissionPublisher } from '../publishers/SqsPermissionPublisher.ts' -import { putObjectContent, waitForS3Objects } from '../utils/s3Utils.ts' +import { getObjectBuffer, putObjectContent, waitForS3Objects } from '../utils/s3Utils.ts' import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts' import type { Dependencies } from '../utils/testContext.ts' import { registerDependencies } from '../utils/testContext.ts' @@ -435,6 +435,54 @@ describe('SqsPermissionConsumer - codec + payload offloading', () => { await diContainer.dispose() }) + it('S3 object is raw zstd binary and SQS message carries a plain pointer (not a codec envelope)', async () => { + // Use an isolated queue with no consumer so we can read the raw SQS message without a race + const wireQueueName = 'codec-offload-wire-check' + await testAdmin.deleteQueues(wireQueueName) + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-offload-wire-1', + messageType: 'add', + metadata: { info: 'wire format check' }, + } + + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + payloadStoreConfig, + creationConfig: { queue: { QueueName: wireQueueName } }, + }) + await wirePublisher.init() + await wirePublisher.publish(message) + + // Read the raw SQS message before any consumer touches it + const { Messages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + expect(Messages, 'Expected a message to be in the queue').toBeDefined() + expect(Messages!.length).toBe(1) + + // SQS body must be a plain JSON pointer — not a codec envelope. + // Compressed bytes live in S3; only the pointer is sent inline. + const sqsBody = JSON.parse(Messages![0]!.Body!) as Record + expect(sqsBody.__codec, 'SQS body must not be a codec envelope when offloading').toBeUndefined() + expect(sqsBody.payloadRef, 'SQS body must contain a payloadRef pointer').toBeDefined() + const payloadRef = sqsBody.payloadRef as Record + expect(payloadRef.codec).toBe(MessageCodecEnum.ZSTD) + + // S3 object must be raw compressed binary, not a JSON codec envelope. + // zstd frames start with magic number 0xFD2FB528 (little-endian: 28 B5 2F FD). + const s3Keys = await waitForS3Objects(s3, s3BucketName, 1, 5000) + expect(s3Keys.length).toBeGreaterThan(0) + const s3Bytes = await getObjectBuffer(s3, s3BucketName, s3Keys[0]!) + expect(s3Bytes.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) + + await wirePublisher.close() + }, 30_000) + it('compresses payload, offloads to S3 as raw binary, and consumer decompresses correctly', async () => { const queueName = 'codec-offload-roundtrip' await testAdmin.deleteQueues(queueName) @@ -459,7 +507,6 @@ describe('SqsPermissionConsumer - codec + payload offloading', () => { await publisher.init() await consumer.start() - await publisher.publish(message) // Verify payload was offloaded to S3 diff --git a/packages/sqs/test/utils/s3Utils.ts b/packages/sqs/test/utils/s3Utils.ts index d453eae3..01f0e5af 100644 --- a/packages/sqs/test/utils/s3Utils.ts +++ b/packages/sqs/test/utils/s3Utils.ts @@ -34,6 +34,13 @@ export async function getObjectContent(s3: S3, bucket: string, key: string) { return result.Body?.transformToString() } +export async function getObjectBuffer(s3: S3, bucket: string, key: string): Promise { + const result = await s3.getObject({ Bucket: bucket, Key: key }) + const bytes = await result.Body?.transformToByteArray() + if (!bytes) throw new Error(`No body for S3 object ${key}`) + return Buffer.from(bytes) +} + export async function putObjectContent(s3: S3, bucket: string, key: string, content: string) { await s3.putObject({ Bucket: bucket, Key: key, Body: content }) } From 17252f923ce82263d987a8533677e5afac07aff8 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Thu, 21 May 2026 12:16:27 +0200 Subject: [PATCH 10/23] feat: add skipCompressionBelow option with default of 512 bytes Introduce `skipCompressionBelow` on publishers (default: 512 bytes). When a message's serialized JSON is smaller than this threshold, compression is skipped and the message is sent as plain JSON instead. Small messages often expand when compressed due to zstd framing overhead, so skipping compression avoids that cost by default. Set to 0 to compress every message regardless of size. Renames the earlier `minCompressionSize` option before it shipped. Updates docs in core, SQS and SNS READMEs and strengthens codec integration tests with wire-format assertions and padding to ensure messages exceed the default threshold when compression is expected. Co-Authored-By: Claude Sonnet 4.6 --- packages/core/README.md | 17 ++ .../core/lib/queues/AbstractQueueService.ts | 2 + packages/core/lib/types/queueOptionsTypes.ts | 19 ++ packages/sns/README.md | 1 + packages/sns/lib/sns/AbstractSnsPublisher.ts | 17 +- packages/sqs/README.md | 1 + packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 17 +- .../SqsPermissionConsumer.codec.spec.ts | 276 +++++++++++++++++- .../test/publishers/SqsPermissionPublisher.ts | 2 + 9 files changed, 335 insertions(+), 17 deletions(-) diff --git a/packages/core/README.md b/packages/core/README.md index ce1139be..9f950658 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -641,6 +641,23 @@ class MyPayloadStore implements PayloadStore { } ``` +#### Message compression (codec) + +Publishers can compress outgoing messages with zstd by setting `codec` in their options. Requires **Node.js >=22.15.0** and the [`@message-queue-toolkit/codec`](../codec/README.md) package. + +```typescript +import { MessageCodecEnum } from '@message-queue-toolkit/core' + +new MyPublisher(deps, { + codec: MessageCodecEnum.ZSTD, + // Optional: skip compression for messages below this byte threshold (default: 512). + // Small messages often expand when compressed; set to 0 to always compress. + skipCompressionBelow: 512, +}) +``` + +Compressed messages are wrapped in a self-describing envelope `{ __codec: 'zstd', __data: '' }`. Consumers detect this envelope automatically and decompress transparently — `codec` does not need to be set on the consumer side. + #### Interaction with codec (compression) When both `codec` and `payloadStoreConfig` are set on a publisher, compression and offloading work together with a single compression pass: diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index c42c926a..4c88a52c 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -138,6 +138,7 @@ export abstract class AbstractQueueService< protected readonly messageMetricsManager?: MessageMetricsManager protected readonly _handlerSpy?: HandlerSpy protected readonly codec?: MessageCodec + protected readonly skipCompressionBelow: number protected isInitted: boolean @@ -176,6 +177,7 @@ export abstract class AbstractQueueService< : undefined this.messageDeduplicationConfig = options.messageDeduplicationConfig this.codec = options.codec + this.skipCompressionBelow = options.skipCompressionBelow ?? 512 this.logMessages = options.logMessages ?? false this._handlerSpy = resolveHandlerSpy(options) diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index 204f37a8..a1ca48f4 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -161,6 +161,25 @@ export type CommonQueueOptions = { * new MyConsumer(deps, { codec: MessageCodecEnum.ZSTD }) */ codec?: MessageCodec + /** + * Minimum serialized size in bytes a message must reach before compression is applied. + * Only meaningful when `codec` is set. Defaults to `512`. + * + * Small messages often expand rather than shrink when compressed due to algorithm + * framing overhead. When the UTF-8 JSON representation of a message is strictly + * smaller than this value, the message is sent as plain JSON instead of a codec + * envelope, avoiding the compression cost with no loss of correctness. + * + * Set to `0` to compress every message regardless of size. + * + * @example + * // Compress only messages ≥ 1 KB + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD, skipCompressionBelow: 1024 }) + * + * // Always compress (disable the floor) + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD, skipCompressionBelow: 0 }) + */ + skipCompressionBelow?: number } export type CommonCreationConfigType = { diff --git a/packages/sns/README.md b/packages/sns/README.md index 29f07ac1..a9f5c4bb 100644 --- a/packages/sns/README.md +++ b/packages/sns/README.md @@ -687,6 +687,7 @@ await consumer.start() // Optional - Compression (Node.js >=22.15.0 required) codec: MessageCodecEnum.ZSTD, // Compress every outgoing message with zstd + skipCompressionBelow: 512, // Skip compression for messages smaller than 512 bytes (default: 512) // Optional - Deletion deletionConfig: { /* ... */ }, diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 090f27b1..cb174f86 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -216,10 +216,21 @@ export abstract class AbstractSnsPublisher const codec = this.codec if (codec) { + // Serialize once so we can check the raw size before deciding whether to compress. + const jsonBuffer = Buffer.from(JSON.stringify(message), 'utf8') + + // Skip compression for messages below the configured floor — small payloads + // often grow when compressed, so we send them as plain JSON instead. + if (jsonBuffer.byteLength < this.skipCompressionBelow) { + return { + payload: + (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? + message, + } + } + // Compress once up-front, then decide: offload the compressed bytes or send inline. - const compressed = await resolveCodecHandler(codec).compress( - Buffer.from(JSON.stringify(message), 'utf8'), - ) + const compressed = await resolveCodecHandler(codec).compress(jsonBuffer) if ( this.payloadStoreConfig && diff --git a/packages/sqs/README.md b/packages/sqs/README.md index 9eebaa7b..83c6cb60 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -871,6 +871,7 @@ class MyConsumer extends AbstractSqsConsumer const codec = this.codec if (codec) { + // Serialize once so we can check the raw size before deciding whether to compress. + const jsonBuffer = Buffer.from(JSON.stringify(message), 'utf8') + + // Skip compression for messages below the configured floor — small payloads + // often grow when compressed, so we send them as plain JSON instead. + if (jsonBuffer.byteLength < this.skipCompressionBelow) { + return { + payload: + (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? + message, + } + } + // Compress once up-front, then decide: offload the compressed bytes or send inline. - const compressed = await resolveCodecHandler(codec).compress( - Buffer.from(JSON.stringify(message), 'utf8'), - ) + const compressed = await resolveCodecHandler(codec).compress(jsonBuffer) if ( this.payloadStoreConfig && diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index 43f44493..86c74cc7 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -12,6 +12,10 @@ import { registerDependencies } from '../utils/testContext.ts' import { SqsPermissionConsumer } from './SqsPermissionConsumer.ts' import type { PERMISSIONS_ADD_MESSAGE_TYPE } from './userConsumerSchemas.ts' +// Padding that pushes any test message's JSON representation above the default +// skipCompressionBelow threshold (512 bytes), ensuring compression is actually applied. +const LARGE_PADDING = 'x'.repeat(450) + describe('SqsPermissionConsumer - zstd codec', () => { let diContainer: AwilixContainer let testAdmin: TestAwsResourceAdmin @@ -30,7 +34,6 @@ describe('SqsPermissionConsumer - zstd codec', () => { await testAdmin.deleteQueues(SqsPermissionConsumer.QUEUE_NAME) consumer = new SqsPermissionConsumer(diContainer.cradle, { - codec: MessageCodecEnum.ZSTD, deletionConfig: { deleteIfExists: false }, }) publisher = new SqsPermissionPublisher(diContainer.cradle, { @@ -53,14 +56,42 @@ describe('SqsPermissionConsumer - zstd codec', () => { }) it('publishes a compressed message and consumer decompresses it correctly', async () => { + // Message is padded to exceed the default skipCompressionBelow (512 bytes) so that + // compression actually fires. Without the padding the payload would be sent as plain + // JSON and the wire assertion below would fail. const message: PERMISSIONS_ADD_MESSAGE_TYPE = { id: 'codec-test-1', messageType: 'add', - metadata: { info: 'hello zstd' }, + metadata: { info: 'hello zstd', padding: LARGE_PADDING }, } - await publisher.publish(message) + // Wire assertion: verify the message is actually sent as a codec envelope. + // Uses an isolated queue with no consumer to avoid a race on the raw SQS body. + const wireQueueName = `${SqsPermissionConsumer.QUEUE_NAME}-roundtrip-wire` + await testAdmin.deleteQueues(wireQueueName) + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + creationConfig: { queue: { QueueName: wireQueueName } }, + }) + await wirePublisher.init() + await wirePublisher.publish(message) + const { Messages: wireMessages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + const envelope = JSON.parse(wireMessages![0]!.Body!) as Record + expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__data).toBe('string') + const compressedBytes = Buffer.from(envelope.__data as string, 'base64') + expect(compressedBytes.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) + await wirePublisher.close() + + // Round-trip assertion: consumer receives and decompresses the message correctly. + await publisher.publish(message) const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') expect(result.message).toMatchObject(message) }) @@ -76,9 +107,11 @@ describe('SqsPermissionConsumer - zstd codec', () => { }) await wirePublisher.init() + // Message must exceed the default skipCompressionBelow (512 bytes) for compression to fire. const message: PERMISSIONS_ADD_MESSAGE_TYPE = { id: 'codec-wire-1', messageType: 'add', + metadata: { padding: LARGE_PADDING }, } await wirePublisher.publish(message) @@ -106,12 +139,45 @@ describe('SqsPermissionConsumer - zstd codec', () => { }) it('consumer correctly handles multiple compressed messages in sequence', async () => { + // Messages are padded to exceed the default skipCompressionBelow (512 bytes). const messages: PERMISSIONS_ADD_MESSAGE_TYPE[] = [ - { id: 'codec-seq-1', messageType: 'add' }, - { id: 'codec-seq-2', messageType: 'add' }, - { id: 'codec-seq-3', messageType: 'add' }, + { id: 'codec-seq-1', messageType: 'add', metadata: { padding: LARGE_PADDING } }, + { id: 'codec-seq-2', messageType: 'add', metadata: { padding: LARGE_PADDING } }, + { id: 'codec-seq-3', messageType: 'add', metadata: { padding: LARGE_PADDING } }, ] + // Wire assertion: verify each message is actually compressed on the wire. + // Uses an isolated queue with no consumer to avoid a race on the raw SQS body. + const wireQueueName = `${SqsPermissionConsumer.QUEUE_NAME}-seq-wire` + await testAdmin.deleteQueues(wireQueueName) + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + creationConfig: { queue: { QueueName: wireQueueName } }, + }) + await wirePublisher.init() + + for (const msg of messages) { + await wirePublisher.publish(msg) + } + + const { Messages: wireMessages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 10, + WaitTimeSeconds: 5, + }), + ) + expect(wireMessages).toHaveLength(messages.length) + for (const raw of wireMessages!) { + const envelope = JSON.parse(raw.Body!) as Record + expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__data).toBe('string') + const compressedBytes = Buffer.from(envelope.__data as string, 'base64') + expect(compressedBytes.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) + } + await wirePublisher.close() + + // Round-trip assertion: consumer receives and decompresses all messages correctly. for (const msg of messages) { await publisher.publish(msg) } @@ -143,8 +209,43 @@ describe('SqsPermissionConsumer - zstd codec', () => { }) it('consumer without codec option still decompresses zstd messages (auto-detection)', async () => { + // Message is padded to exceed the default skipCompressionBelow (512 bytes) so that + // the publisher actually compresses it. Without padding the message would be sent as + // plain JSON and the auto-detect consumer would succeed trivially via normal parsing — + // which would not prove decompression is working. + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'codec-auto-detect-1', + messageType: 'add', + metadata: { padding: LARGE_PADDING }, + } + + // Wire assertion: verify the publisher actually sends a codec envelope. + // The auto-detect consumer would succeed on plain JSON too, so we need this + // to prove decompression is actually happening rather than plain JSON parsing. + const wireQueueName = `${SqsPermissionConsumer.QUEUE_NAME}-auto-detect-wire` + await testAdmin.deleteQueues(wireQueueName) + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + creationConfig: { queue: { QueueName: wireQueueName } }, + }) + await wirePublisher.init() + await wirePublisher.publish(message) + + const { Messages: wireMessages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + const envelope = JSON.parse(wireMessages![0]!.Body!) as Record + expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__data).toBe('string') + await wirePublisher.close() + + // Round-trip assertion: consumer WITHOUT codec auto-detects the envelope and decompresses. // Use a dedicated queue so only autoConsumer polls it — avoids both the race - // condition (shared queue) and localstack long-poll timing issues (abort + restart) + // condition (shared queue) and localstack long-poll timing issues (abort + restart). const autoQueueName = `${SqsPermissionConsumer.QUEUE_NAME}-auto-detect` await testAdmin.deleteQueues(autoQueueName) @@ -161,10 +262,6 @@ describe('SqsPermissionConsumer - zstd codec', () => { }) await autoConsumer.start() - const message: PERMISSIONS_ADD_MESSAGE_TYPE = { - id: 'codec-auto-detect-1', - messageType: 'add', - } await autoPublisher.publish(message) const result = await autoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') @@ -174,3 +271,160 @@ describe('SqsPermissionConsumer - zstd codec', () => { await autoConsumer.close(true) }, 15000) }) + +describe('SqsPermissionConsumer - skipCompressionBelow', () => { + let diContainer: AwilixContainer + let testAdmin: TestAwsResourceAdmin + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + testAdmin = diContainer.cradle.testAdmin + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('sends plain JSON for small messages by default (no skipCompressionBelow set)', async () => { + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-default-skip` + await testAdmin.deleteQueues(queueName) + + // No skipCompressionBelow — default of 512 applies. + // The small message (well under 512 bytes) must be sent as plain JSON. + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + creationConfig: { queue: { QueueName: queueName } }, + }) + await wirePublisher.init() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'default-skip-1', + messageType: 'add', + } + await wirePublisher.publish(message) + + const { Messages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + expect(Messages, 'Expected a message to be in the queue').toBeDefined() + expect(Messages!.length).toBe(1) + + const body = JSON.parse(Messages![0]!.Body!) as Record + expect(body.__codec).toBeUndefined() + expect(body.__data).toBeUndefined() + expect(body.id).toBe(message.id) + + await wirePublisher.close() + }) + + it('sends plain JSON when message is smaller than skipCompressionBelow', async () => { + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-skip-below` + await testAdmin.deleteQueues(queueName) + + // skipCompressionBelow set very high — small message is never compressed + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + skipCompressionBelow: 99_999, + creationConfig: { queue: { QueueName: queueName } }, + }) + await wirePublisher.init() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'skip-below-1', + messageType: 'add', + } + await wirePublisher.publish(message) + + const { Messages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + expect(Messages, 'Expected a message to be in the queue').toBeDefined() + expect(Messages!.length).toBe(1) + + const body = JSON.parse(Messages![0]!.Body!) as Record + expect(body.__codec).toBeUndefined() + expect(body.__data).toBeUndefined() + expect(body.id).toBe(message.id) + + await wirePublisher.close() + }) + + it('compresses when skipCompressionBelow is 0 (always compress)', async () => { + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-always-compress` + await testAdmin.deleteQueues(queueName) + + // skipCompressionBelow: 0 disables the floor — every message is compressed + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + skipCompressionBelow: 0, + creationConfig: { queue: { QueueName: queueName } }, + }) + await wirePublisher.init() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'always-compress-1', + messageType: 'add', + } + await wirePublisher.publish(message) + + const { Messages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + expect(Messages, 'Expected a message to be in the queue').toBeDefined() + expect(Messages!.length).toBe(1) + + const envelope = JSON.parse(Messages![0]!.Body!) as Record + expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__data).toBe('string') + + await wirePublisher.close() + }) + + it('consumer receives and processes a message sent as plain JSON due to skipCompressionBelow', async () => { + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-skip-consumer` + await testAdmin.deleteQueues(queueName) + + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + skipCompressionBelow: 99_999, + creationConfig: { queue: { QueueName: queueName } }, + }) + await wirePublisher.init() + + const wireConsumer = new SqsPermissionConsumer(diContainer.cradle, { + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + }) + await wireConsumer.start() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'skip-consumer-1', + messageType: 'add', + metadata: { info: 'plain json path' }, + } + await wirePublisher.publish(message) + + const result = await wireConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await wirePublisher.close() + await wireConsumer.close(true) + }) +}) diff --git a/packages/sqs/test/publishers/SqsPermissionPublisher.ts b/packages/sqs/test/publishers/SqsPermissionPublisher.ts index 5aa73c20..88010643 100644 --- a/packages/sqs/test/publishers/SqsPermissionPublisher.ts +++ b/packages/sqs/test/publishers/SqsPermissionPublisher.ts @@ -32,6 +32,7 @@ export class SqsPermissionPublisher extends AbstractSqsPublisher, ) { super(dependencies, { @@ -55,6 +56,7 @@ export class SqsPermissionPublisher extends AbstractSqsPublisher Date: Thu, 21 May 2026 13:08:02 +0200 Subject: [PATCH 11/23] =?UTF-8?q?perf:=20stream=20codec+offload=20path=20t?= =?UTF-8?q?o=20avoid=203=C3=97=20buffer=20materialisation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When codec and payloadStoreConfig are both set, the previous code materialised the full payload three times before uploading: JSON.stringify (string) → Buffer.from (buffer) → compress(buffer). For payloads large enough to need streaming offload this could OOM. Adds AbstractQueueService.compressAndOffloadPayload: serialises once via the configured serialiser, pipes the Readable through zlib.createZstdCompress() into a temp file, then either streams the temp file directly to the store (if compressed size exceeds threshold) or reads the small buffer for an inline codec envelope. Temp file is always cleaned up in a finally block. The inline-only path (codec without payloadStoreConfig, bounded by the 256 KB protocol limit) keeps the existing buffer approach unchanged. Co-Authored-By: Claude Sonnet 4.6 --- .../core/lib/queues/AbstractQueueService.ts | 73 +++++++++++++++++++ packages/sns/lib/sns/AbstractSnsPublisher.ts | 32 ++++---- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 32 ++++---- 3 files changed, 109 insertions(+), 28 deletions(-) diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 4c88a52c..7d7175c3 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -1,5 +1,10 @@ +import { randomUUID } from 'node:crypto' +import * as fs from 'node:fs' +import * as os from 'node:os' import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' import { types } from 'node:util' +import * as zlib from 'node:zlib' import { type CommonLogger, type Either, @@ -773,6 +778,74 @@ export abstract class AbstractQueueService< return this.buildPointer(message, payloadId, storeName, compressed.byteLength, codec) } + /** + * Streaming compress-and-offload path for large payloads (used when both `codec` and + * `payloadStoreConfig` are set). + * + * Avoids the 3× memory materialisation that occurs when doing + * JSON.stringify → Buffer → compress(Buffer) before deciding whether to offload. + * Instead serializes the payload once via the configured serializer, pipes the stream + * through a zstd transform into a temp file, then decides based on the compressed size: + * - Uploads the temp file as a stream when compressed size exceeds `messageSizeThreshold` + * - Returns the small compressed buffer for the caller to wrap in an inline codec envelope + * when compressed size fits within the threshold + * + * `skipCompressionBelow` is intentionally NOT checked here: when both `codec` and + * `payloadStoreConfig` are configured the user explicitly opted into compression, and even + * small messages may need to be offloaded (e.g. a very low `messageSizeThreshold`). + * The caller is responsible for applying `skipCompressionBelow` on the inline-only path. + * + * @returns + * - `{ pointer }` — compressed payload was offloaded; use pointer as the message payload + * - `{ compressedBuffer }` — compressed payload fits inline; caller builds the codec envelope + */ + protected async compressAndOffloadPayload( + message: MessagePayloadSchemas, + codec: MessageCodec, + ): Promise< + | { pointer: OffloadedPayloadPointerPayload; compressedBuffer?: never } + | { compressedBuffer: Buffer; pointer?: never } + > { + if (!this.payloadStoreConfig) { + throw new Error('Payload store is not configured') + } + + const serialized = await this.payloadStoreConfig.serializer.serialize(message) + const tmpPath = `${os.tmpdir()}/${randomUUID()}.zst` + + try { + await pipeline( + typeof serialized.value === 'string' ? Readable.from(serialized.value) : serialized.value, + zlib.createZstdCompress(), + fs.createWriteStream(tmpPath), + ) + + const compressedSize = fs.statSync(tmpPath).size + + if (compressedSize > this.payloadStoreConfig.messageSizeThreshold) { + const { store, storeName } = this.resolveOutgoingStore() + const payloadId = await store.storePayload({ + value: fs.createReadStream(tmpPath), + size: compressedSize, + }) + return { pointer: this.buildPointer(message, payloadId, storeName, compressedSize, codec) } + } + + // Compressed payload fits inline — return the buffer; caller wraps it in a codec envelope. + const compressedBuffer = fs.readFileSync(tmpPath) + return { compressedBuffer } + } finally { + try { + fs.unlinkSync(tmpPath) + } catch { + // ignore cleanup errors + } + if (isDestroyable(serialized)) { + await serialized.destroy() + } + } + } + /** * Retrieve previously offloaded message payload using provided pointer payload. * Returns the original payload or an error if the payload was not found or could not be parsed. diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index cb174f86..a7e22203 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -208,6 +208,9 @@ export abstract class AbstractSnsPublisher * Compresses (when codec is set) or offloads (when store is configured) the message. * Returns the payload to send and an optional pre-built body string. * When preBuiltBody is set, it is a ready-to-send codec envelope — sendMessage must use it as-is. + * + * When both codec and payloadStoreConfig are set, uses a streaming pipeline + * (JSON → zstd → temp file → store) to avoid materialising the full payload in memory. */ private async prepareOutgoingPayload(message: MessagePayloadType): Promise<{ payload: MessagePayloadType | OffloadedPayloadPointerPayload @@ -216,29 +219,30 @@ export abstract class AbstractSnsPublisher const codec = this.codec if (codec) { + if (this.payloadStoreConfig) { + // Streaming path: avoids 3× buffer materialisation for large payloads. + // JSON → zstd → temp file → threshold check → offload or inline envelope. + const result = await this.compressAndOffloadPayload(message, codec) + if (result.pointer) { + return { payload: result.pointer } + } + return { + payload: message, + preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codec), + } + } + + // No offload store — bounded by SNS 256 KB limit, safe to buffer. // Serialize once so we can check the raw size before deciding whether to compress. const jsonBuffer = Buffer.from(JSON.stringify(message), 'utf8') // Skip compression for messages below the configured floor — small payloads // often grow when compressed, so we send them as plain JSON instead. if (jsonBuffer.byteLength < this.skipCompressionBelow) { - return { - payload: - (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? - message, - } + return { payload: message } } - // Compress once up-front, then decide: offload the compressed bytes or send inline. const compressed = await resolveCodecHandler(codec).compress(jsonBuffer) - - if ( - this.payloadStoreConfig && - compressed.byteLength > this.payloadStoreConfig.messageSizeThreshold - ) { - return { payload: await this.offloadCompressedPayload(message, compressed, codec) } - } - return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codec) } } diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index a67385e5..13a2ec3f 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -202,6 +202,9 @@ export abstract class AbstractSqsPublisher * Compresses (when codec is set) or offloads (when store is configured) the message. * Returns the payload to send and an optional pre-built body string. * When preBuiltBody is set, it is a ready-to-send codec envelope — sendMessage must use it as-is. + * + * When both codec and payloadStoreConfig are set, uses a streaming pipeline + * (JSON → zstd → temp file → store) to avoid materialising the full payload in memory. */ private async prepareOutgoingPayload(message: MessagePayloadType): Promise<{ payload: MessagePayloadType | OffloadedPayloadPointerPayload @@ -210,29 +213,30 @@ export abstract class AbstractSqsPublisher const codec = this.codec if (codec) { + if (this.payloadStoreConfig) { + // Streaming path: avoids 3× buffer materialisation for large payloads. + // JSON → zstd → temp file → threshold check → offload or inline envelope. + const result = await this.compressAndOffloadPayload(message, codec) + if (result.pointer) { + return { payload: result.pointer } + } + return { + payload: message, + preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codec), + } + } + + // No offload store — bounded by SQS 256 KB limit, safe to buffer. // Serialize once so we can check the raw size before deciding whether to compress. const jsonBuffer = Buffer.from(JSON.stringify(message), 'utf8') // Skip compression for messages below the configured floor — small payloads // often grow when compressed, so we send them as plain JSON instead. if (jsonBuffer.byteLength < this.skipCompressionBelow) { - return { - payload: - (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? - message, - } + return { payload: message } } - // Compress once up-front, then decide: offload the compressed bytes or send inline. const compressed = await resolveCodecHandler(codec).compress(jsonBuffer) - - if ( - this.payloadStoreConfig && - compressed.byteLength > this.payloadStoreConfig.messageSizeThreshold - ) { - return { payload: await this.offloadCompressedPayload(message, compressed, codec) } - } - return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codec) } } From 98acde2afdeecf294ae8a76e8861d3a9ce2a52c2 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Thu, 21 May 2026 13:51:57 +0200 Subject: [PATCH 12/23] fix(codec): rename envelope fields, fix wire-size threshold, skip intermediate object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Rename __codec/__data → __mqtCodec/__mqtData Two-underscore prefixes collide with ORM/framework internals and user schemas; a collision with the loose isCodecEnvelope check would silently mistake a real message for a codec envelope. The mq-toolkit namespace prefix is unambiguous. Breaking change — existing compressed messages on the wire are not readable by the updated consumer. 2. Fix compressAndOffloadPayload threshold comparison (Issue 1) The previous check used raw compressedSize; the actual wire body is a codec envelope where the compressed bytes are base64-encoded (~×4/3). With messageSizeThreshold set near the protocol limit a payload just under the threshold could produce an envelope well over the limit and be rejected at runtime. Now compares estimated envelope size (⌈N×4/3⌉ + 32 + codec.length) against the threshold. 3. Skip intermediate object in buildCodecEnvelope JSON.stringify({ __mqtCodec, __mqtData }) allocated a transient object between the base64 string and the final envelope string. String concatenation avoids that allocation with no observable difference. 4. Document messageSizeThreshold wire-size semantics (Issue 2) Without codec the threshold gates on raw JSON size; with codec it gates on envelope wire size (base64-encoded compressed payload + JSON framing). Enabling codec raises the effective bar for offloading since compression shrinks the payload before comparison. Both SinglePayloadStoreConfig and MultiPayloadStoreConfig JSDoc now explain this explicitly. Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/README.md | 6 +-- packages/codec/lib/codec/codecHandler.ts | 14 +++---- packages/core/README.md | 2 +- packages/core/lib/codec/messageCodec.ts | 4 +- .../lib/payload-store/payloadStoreTypes.ts | 34 ++++++++++++++++- .../core/lib/queues/AbstractQueueService.ts | 7 +++- packages/core/lib/types/queueOptionsTypes.ts | 2 +- .../SnsSqsPermissionConsumer.codec.spec.ts | 2 +- packages/sqs/README.md | 2 +- .../SqsPermissionConsumer.codec.spec.ts | 38 +++++++++---------- ...rmissionConsumer.payloadOffloading.spec.ts | 5 ++- 11 files changed, 77 insertions(+), 39 deletions(-) diff --git a/packages/codec/README.md b/packages/codec/README.md index d915544f..4ac73b8a 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -52,7 +52,7 @@ const compressed: Buffer = await handler.compress(Buffer.from(JSON.stringify(pay // Build envelope without a second compression pass const envelopeString = buildCodecEnvelope(compressed, MessageCodecEnum.ZSTD) -// → '{"__codec":"zstd","__data":""}' +// → '{"__mqtCodec":"zstd","__mqtData":""}' ``` ### Custom codec handler @@ -72,8 +72,8 @@ Compressed messages are wrapped in a self-describing JSON envelope: ```json { - "__codec": "zstd", - "__data": "" + "__mqtCodec": "zstd", + "__mqtData": "" } ``` diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index e39eebc0..a495a771 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -39,18 +39,18 @@ export async function compressMessageBody(jsonBody: string, codec: MessageCodec) /** * Wraps an already-compressed buffer in a codec envelope string. * Use this when you have pre-compressed bytes and want to avoid compressing twice. + * + * Uses string concatenation instead of JSON.stringify to avoid allocating an + * intermediate object — the base64 string and the envelope string are the only + * two allocations on the inline path. */ export function buildCodecEnvelope(compressed: Buffer, codec: MessageCodec): string { - const envelope: CodecEnvelope = { - __codec: codec, - __data: compressed.toString('base64'), - } - return JSON.stringify(envelope) + return '{"__mqtCodec":"' + codec + '","__mqtData":"' + compressed.toString('base64') + '"}' } export async function decompressMessageBody(envelope: CodecEnvelope): Promise { - const handler = resolveCodecHandler(envelope.__codec) - const compressed = Buffer.from(envelope.__data, 'base64') + const handler = resolveCodecHandler(envelope.__mqtCodec) + const compressed = Buffer.from(envelope.__mqtData, 'base64') const decompressed = await handler.decompress(compressed) return JSON.parse(decompressed.toString('utf8')) } diff --git a/packages/core/README.md b/packages/core/README.md index 9f950658..d47465ff 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -656,7 +656,7 @@ new MyPublisher(deps, { }) ``` -Compressed messages are wrapped in a self-describing envelope `{ __codec: 'zstd', __data: '' }`. Consumers detect this envelope automatically and decompress transparently — `codec` does not need to be set on the consumer side. +Compressed messages are wrapped in a self-describing envelope `{ __mqtCodec: 'zstd', __mqtData: '' }`. Consumers detect this envelope automatically and decompress transparently — `codec` does not need to be set on the consumer side. #### Interaction with codec (compression) diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 6ecc46dd..967e7110 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -16,8 +16,8 @@ export const MessageCodecEnum = { } as const export type MessageCodec = ObjectValues -const CODEC_FIELD = '__codec' -const DATA_FIELD = '__data' +const CODEC_FIELD = '__mqtCodec' +const DATA_FIELD = '__mqtData' export type CodecEnvelope = { [CODEC_FIELD]: MessageCodec diff --git a/packages/core/lib/payload-store/payloadStoreTypes.ts b/packages/core/lib/payload-store/payloadStoreTypes.ts index da3d95f9..ba9d56ab 100644 --- a/packages/core/lib/payload-store/payloadStoreTypes.ts +++ b/packages/core/lib/payload-store/payloadStoreTypes.ts @@ -30,7 +30,22 @@ export interface PayloadSerializer { * Use this when you have only one payload store. */ export type SinglePayloadStoreConfig = { - /** Threshold in bytes after which the payload should be stored in the store. */ + /** + * Wire-body size threshold in bytes. Messages whose wire body exceeds this value + * are offloaded to the store; smaller messages are sent inline. + * + * **What counts as "wire body size"** depends on whether a codec is active: + * - Without codec: the UTF-8 byte length of `JSON.stringify(message)`. + * - With codec: the byte length of the codec envelope + * (`{"__mqtCodec":"zstd","__mqtData":""}`) — i.e. the compressed + * payload after base64 encoding and JSON framing (~4/3 of the compressed size). + * + * Because codec reduces payload size before the threshold is applied, enabling + * codec effectively raises the bar for offloading: a 500 KB message that + * compresses to 100 KB will not trigger a 200 KB threshold. + * Size your threshold accordingly, or set it to the protocol's hard limit + * (e.g. `SQS_MESSAGE_MAX_SIZE`) to offload only when strictly necessary. + */ messageSizeThreshold: number /** The store to use for storing the payload. */ @@ -51,7 +66,22 @@ export type SinglePayloadStoreConfig = { * Use this when you need to support multiple payload stores (e.g., for migration). */ export type MultiPayloadStoreConfig = { - /** Threshold in bytes after which the payload should be stored in the store. */ + /** + * Wire-body size threshold in bytes. Messages whose wire body exceeds this value + * are offloaded to the store; smaller messages are sent inline. + * + * **What counts as "wire body size"** depends on whether a codec is active: + * - Without codec: the UTF-8 byte length of `JSON.stringify(message)`. + * - With codec: the byte length of the codec envelope + * (`{"__mqtCodec":"zstd","__mqtData":""}`) — i.e. the compressed + * payload after base64 encoding and JSON framing (~4/3 of the compressed size). + * + * Because codec reduces payload size before the threshold is applied, enabling + * codec effectively raises the bar for offloading: a 500 KB message that + * compresses to 100 KB will not trigger a 200 KB threshold. + * Size your threshold accordingly, or set it to the protocol's hard limit + * (e.g. `SQS_MESSAGE_MAX_SIZE`) to offload only when strictly necessary. + */ messageSizeThreshold: number /** Map of store identifiers to store instances. */ diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 7d7175c3..7899b367 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -822,7 +822,12 @@ export abstract class AbstractQueueService< const compressedSize = fs.statSync(tmpPath).size - if (compressedSize > this.payloadStoreConfig.messageSizeThreshold) { + // Compare the envelope wire size (not raw compressed bytes) against the threshold. + // buildCodecEnvelope produces {"__mqtCodec":"","__mqtData":""}. + // Base64 expands by ⌈N/3⌉×4; the fixed JSON framing adds 32 chars + codec name length. + const envelopeSize = Math.ceil((compressedSize * 4) / 3) + 32 + codec.length + + if (envelopeSize > this.payloadStoreConfig.messageSizeThreshold) { const { store, storeName } = this.resolveOutgoingStore() const payloadId = await store.storePayload({ value: fs.createReadStream(tmpPath), diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index a1ca48f4..979d72ea 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -144,7 +144,7 @@ export type CommonQueueOptions = { * Compression codec applied to message bodies. * * - **Publisher**: every outgoing message body is compressed and wrapped in a - * self-describing envelope `{ __codec: 'zstd', __data: '' }`. + * self-describing envelope `{ __mqtCodec: 'zstd', __mqtData: '' }`. * - **Consumer**: when set, the consumer expects compressed messages. * Even without this option, consumers auto-detect and decompress any message * that carries a codec envelope, so mixed queues work transparently. diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts index e859c317..bdf311df 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts @@ -83,7 +83,7 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { // Stop the beforeEach consumer so it cannot steal messages from the shared queue await consumer.close() - // Consumer without explicit codec — decompression is auto-detected from envelope __codec field + // Consumer without explicit codec — decompression is auto-detected from envelope __mqtCodec field const autoConsumer = new SnsSqsPermissionConsumer(diContainer.cradle, { locatorConfig: { queueUrl: consumer.subscriptionProps.queueUrl, diff --git a/packages/sqs/README.md b/packages/sqs/README.md index 83c6cb60..f6b3961e 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -826,7 +826,7 @@ The codec implementation lives in the separate [`@message-queue-toolkit/codec`]( npm install @message-queue-toolkit/codec ``` -Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __codec: 'zstd', __data: '' }`), so a consumer without `codec` set will still decompress automatically via envelope detection. This allows a gradual rollout — enable compression on the publisher first, consumers adapt without configuration changes. +Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __mqtCodec: 'zstd', __mqtData: '' }`), so a consumer without `codec` set will still decompress automatically via envelope detection. This allows a gradual rollout — enable compression on the publisher first, consumers adapt without configuration changes. #### Publisher diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index 86c74cc7..25e96b9b 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -84,9 +84,9 @@ describe('SqsPermissionConsumer - zstd codec', () => { }), ) const envelope = JSON.parse(wireMessages![0]!.Body!) as Record - expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) - expect(typeof envelope.__data).toBe('string') - const compressedBytes = Buffer.from(envelope.__data as string, 'base64') + expect(envelope.__mqtCodec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__mqtData).toBe('string') + const compressedBytes = Buffer.from(envelope.__mqtData as string, 'base64') expect(compressedBytes.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) await wirePublisher.close() @@ -128,11 +128,11 @@ describe('SqsPermissionConsumer - zstd codec', () => { // Body must be a self-describing codec envelope, not raw message JSON const envelope = JSON.parse(Messages![0]!.Body!) as Record - expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) - expect(typeof envelope.__data).toBe('string') + expect(envelope.__mqtCodec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__mqtData).toBe('string') - // __data must decode to a valid zstd frame: magic number 0xFD2FB528 (LE → 28 B5 2F FD) - const compressed = Buffer.from(envelope.__data as string, 'base64') + // __mqtData must decode to a valid zstd frame: magic number 0xFD2FB528 (LE → 28 B5 2F FD) + const compressed = Buffer.from(envelope.__mqtData as string, 'base64') expect(compressed.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) await wirePublisher.close() @@ -170,9 +170,9 @@ describe('SqsPermissionConsumer - zstd codec', () => { expect(wireMessages).toHaveLength(messages.length) for (const raw of wireMessages!) { const envelope = JSON.parse(raw.Body!) as Record - expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) - expect(typeof envelope.__data).toBe('string') - const compressedBytes = Buffer.from(envelope.__data as string, 'base64') + expect(envelope.__mqtCodec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__mqtData).toBe('string') + const compressedBytes = Buffer.from(envelope.__mqtData as string, 'base64') expect(compressedBytes.subarray(0, 4)).toEqual(Buffer.from([0x28, 0xb5, 0x2f, 0xfd])) } await wirePublisher.close() @@ -239,8 +239,8 @@ describe('SqsPermissionConsumer - zstd codec', () => { }), ) const envelope = JSON.parse(wireMessages![0]!.Body!) as Record - expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) - expect(typeof envelope.__data).toBe('string') + expect(envelope.__mqtCodec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__mqtData).toBe('string') await wirePublisher.close() // Round-trip assertion: consumer WITHOUT codec auto-detects the envelope and decompresses. @@ -255,7 +255,7 @@ describe('SqsPermissionConsumer - zstd codec', () => { }) await autoPublisher.init() - // Consumer without codec — auto-detects from envelope __codec field + // Consumer without codec — auto-detects from envelope __mqtCodec field const autoConsumer = new SqsPermissionConsumer(diContainer.cradle, { creationConfig: { queue: { QueueName: autoQueueName } }, deletionConfig: { deleteIfExists: false }, @@ -319,8 +319,8 @@ describe('SqsPermissionConsumer - skipCompressionBelow', () => { expect(Messages!.length).toBe(1) const body = JSON.parse(Messages![0]!.Body!) as Record - expect(body.__codec).toBeUndefined() - expect(body.__data).toBeUndefined() + expect(body.__mqtCodec).toBeUndefined() + expect(body.__mqtData).toBeUndefined() expect(body.id).toBe(message.id) await wirePublisher.close() @@ -355,8 +355,8 @@ describe('SqsPermissionConsumer - skipCompressionBelow', () => { expect(Messages!.length).toBe(1) const body = JSON.parse(Messages![0]!.Body!) as Record - expect(body.__codec).toBeUndefined() - expect(body.__data).toBeUndefined() + expect(body.__mqtCodec).toBeUndefined() + expect(body.__mqtData).toBeUndefined() expect(body.id).toBe(message.id) await wirePublisher.close() @@ -391,8 +391,8 @@ describe('SqsPermissionConsumer - skipCompressionBelow', () => { expect(Messages!.length).toBe(1) const envelope = JSON.parse(Messages![0]!.Body!) as Record - expect(envelope.__codec).toBe(MessageCodecEnum.ZSTD) - expect(typeof envelope.__data).toBe('string') + expect(envelope.__mqtCodec).toBe(MessageCodecEnum.ZSTD) + expect(typeof envelope.__mqtData).toBe('string') await wirePublisher.close() }) diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts index d41bad0c..22e369e6 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts @@ -468,7 +468,10 @@ describe('SqsPermissionConsumer - codec + payload offloading', () => { // SQS body must be a plain JSON pointer — not a codec envelope. // Compressed bytes live in S3; only the pointer is sent inline. const sqsBody = JSON.parse(Messages![0]!.Body!) as Record - expect(sqsBody.__codec, 'SQS body must not be a codec envelope when offloading').toBeUndefined() + expect( + sqsBody.__mqtCodec, + 'SQS body must not be a codec envelope when offloading', + ).toBeUndefined() expect(sqsBody.payloadRef, 'SQS body must contain a payloadRef pointer').toBeDefined() const payloadRef = sqsBody.payloadRef as Record expect(payloadRef.codec).toBe(MessageCodecEnum.ZSTD) From 8c4a538edb02884ae4219e6d4a54dd684e3f5471 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Thu, 21 May 2026 15:13:52 +0200 Subject: [PATCH 13/23] feat(codec): extensible codec system with consumer registry and custom handler support - Add MessageCodecRegistration union type (built-in string | { name, handler }) so publishers and consumers accept custom codec handlers without forking - Add createCompressStream() to MessageCodecHandler interface, eliminating codec-specific branching in the streaming offload path - Add KNOWN_CODECS module-level Set (built once from MessageCodecEnum values) and expose it from core barrel for hot-path reuse - Add optional knownCodecs param to isCodecEnvelope for per-consumer scoping - Make zstd availability check lazy (throw inside compress/decompress, not at import) - Remove codec/skipCompressionBelow from consumer options; replace with codecs array - Consumer auto-registers all built-in codecs plus any user-supplied ones; throws on unknown codec name in incoming message - Add getCodecName/resolveCodecHandler helpers for object-form registrations - Add unit tests for isCodecEnvelope with custom knownCodecs - Add integration tests for custom codec round-trip and scoped auto-detection - Update README docs for custom codec registration on both publisher and consumer Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/README.md | 34 ++- packages/codec/lib/codec/codecHandler.ts | 59 ++++-- packages/codec/lib/index.ts | 1 + packages/core/README.md | 24 ++- packages/core/lib/codec/messageCodec.ts | 64 +++++- packages/core/lib/index.ts | 2 + .../offloadedPayloadMessageSchemas.ts | 7 +- .../core/lib/queues/AbstractQueueService.ts | 35 ++-- packages/core/lib/types/queueOptionsTypes.ts | 18 +- packages/core/test/codec/messageCodec.spec.ts | 107 ++++++++++ packages/sns/lib/sns/AbstractSnsPublisher.ts | 15 +- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 69 +++++-- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 15 +- .../SqsPermissionConsumer.codec.spec.ts | 195 ++++++++++++++++++ .../test/consumers/SqsPermissionConsumer.ts | 6 +- 15 files changed, 579 insertions(+), 72 deletions(-) create mode 100644 packages/core/test/codec/messageCodec.spec.ts diff --git a/packages/codec/README.md b/packages/codec/README.md index 4ac73b8a..9ab0e6af 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -57,15 +57,43 @@ const envelopeString = buildCodecEnvelope(compressed, MessageCodecEnum.ZSTD) ### Custom codec handler +Implement `MessageCodecHandler` and register it via the `{ name, handler }` form of the `codec` option. The same registration must be provided on both the publisher and the consumer. + ```typescript +import type { Transform } from 'node:stream' import type { MessageCodecHandler } from '@message-queue-toolkit/core' -class MyCodecHandler implements MessageCodecHandler { - compress(data: Buffer): Promise { /* ... */ } - decompress(data: Buffer): Promise { /* ... */ } +class MyLz4Handler implements MessageCodecHandler { + async compress(data: Buffer): Promise { + return lz4.encode(data) // your compression library + } + + async decompress(data: Buffer): Promise { + return lz4.decode(data) + } + + // Required for the streaming offload path (codec + payloadStoreConfig). + // Return a Transform stream that compresses its input chunk-by-chunk. + createCompressStream(): Transform { + return lz4.createEncoderStream() + } } ``` +Register the handler on the publisher and consumer using the `{ name, handler }` object form: + +```typescript +const codec = { name: 'lz4', handler: new MyLz4Handler() } + +// Publisher — wraps each outgoing message in { __mqtCodec: 'lz4', __mqtData: '' } +new MyPublisher(deps, { codec }) + +// Consumer — only auto-detects envelopes whose __mqtCodec matches 'lz4' +new MyConsumer(deps, { codec }) +``` + +**Consumer-side scoping.** A consumer configured with `{ name: 'lz4', handler }` will only decompress envelopes that carry `__mqtCodec: 'lz4'`. A consumer configured with the built-in `MessageCodecEnum.ZSTD` will ignore `lz4` envelopes entirely — they reach schema validation as raw objects and are rejected. This prevents accidental cross-codec decompression. + ## Codec envelope format Compressed messages are wrapped in a self-describing JSON envelope: diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index a495a771..0bb5869a 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -1,39 +1,68 @@ +import type { Transform } from 'node:stream' import { promisify } from 'node:util' import zlib from 'node:zlib' -import type { CodecEnvelope, MessageCodec, MessageCodecHandler } from '@message-queue-toolkit/core' +import type { + CodecEnvelope, + MessageCodecHandler, + MessageCodecRegistration, +} from '@message-queue-toolkit/core' import { MessageCodecEnum } from '@message-queue-toolkit/core' -if (typeof zlib.zstdCompress !== 'function' || typeof zlib.zstdDecompress !== 'function') { - throw new Error( - 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + - '@message-queue-toolkit/codec requires Node.js >=22.15.0 or >=23.8.0.', - ) -} +const ZSTD_UNSUPPORTED_MSG = + 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + + '@message-queue-toolkit/codec requires Node.js >=22.15.0 or >=23.8.0.' -const zstdCompress = promisify(zlib.zstdCompress) -const zstdDecompress = promisify(zlib.zstdDecompress) +// Resolved lazily — undefined on Node versions that lack zstd support. +const zstdCompress = + typeof zlib.zstdCompress === 'function' ? promisify(zlib.zstdCompress) : undefined +const zstdDecompress = + typeof zlib.zstdDecompress === 'function' ? promisify(zlib.zstdDecompress) : undefined export class ZstdCodecHandler implements MessageCodecHandler { compress(data: Buffer): Promise { + if (!zstdCompress) throw new Error(ZSTD_UNSUPPORTED_MSG) return zstdCompress(data) } decompress(data: Buffer): Promise { + if (!zstdDecompress) throw new Error(ZSTD_UNSUPPORTED_MSG) return zstdDecompress(data) } + + createCompressStream(): Transform { + if (typeof zlib.createZstdCompress !== 'function') throw new Error(ZSTD_UNSUPPORTED_MSG) + return zlib.createZstdCompress() + } } const ZSTD_HANDLER = new ZstdCodecHandler() -export function resolveCodecHandler(codec: MessageCodec): MessageCodecHandler { +/** + * Returns the name string that will be written into the `__mqtCodec` field of every envelope. + */ +export function getCodecName(codec: MessageCodecRegistration): string { + return typeof codec === 'string' ? codec : codec.name +} + +/** + * Resolves the {@link MessageCodecHandler} for the given codec registration. + * + * - String form (`MessageCodec`): returns the built-in handler for that codec. + * - Object form (`{ name, handler }`): returns the provided handler directly. + */ +export function resolveCodecHandler(codec: MessageCodecRegistration): MessageCodecHandler { + if (typeof codec === 'object') return codec.handler if (codec === MessageCodecEnum.ZSTD) return ZSTD_HANDLER throw new Error(`Unsupported codec: ${codec}`) } -export async function compressMessageBody(jsonBody: string, codec: MessageCodec): Promise { +export async function compressMessageBody( + jsonBody: string, + codec: MessageCodecRegistration, +): Promise { const handler = resolveCodecHandler(codec) const compressed = await handler.compress(Buffer.from(jsonBody, 'utf8')) - return buildCodecEnvelope(compressed, codec) + return buildCodecEnvelope(compressed, getCodecName(codec)) } /** @@ -44,12 +73,12 @@ export async function compressMessageBody(jsonBody: string, codec: MessageCodec) * intermediate object — the base64 string and the envelope string are the only * two allocations on the inline path. */ -export function buildCodecEnvelope(compressed: Buffer, codec: MessageCodec): string { - return '{"__mqtCodec":"' + codec + '","__mqtData":"' + compressed.toString('base64') + '"}' +export function buildCodecEnvelope(compressed: Buffer, codecName: string): string { + return '{"__mqtCodec":"' + codecName + '","__mqtData":"' + compressed.toString('base64') + '"}' } export async function decompressMessageBody(envelope: CodecEnvelope): Promise { - const handler = resolveCodecHandler(envelope.__mqtCodec) + const handler = resolveCodecHandler(envelope.__mqtCodec as MessageCodecRegistration) const compressed = Buffer.from(envelope.__mqtData, 'base64') const decompressed = await handler.decompress(compressed) return JSON.parse(decompressed.toString('utf8')) diff --git a/packages/codec/lib/index.ts b/packages/codec/lib/index.ts index ed36e669..f723f350 100644 --- a/packages/codec/lib/index.ts +++ b/packages/codec/lib/index.ts @@ -2,6 +2,7 @@ export { buildCodecEnvelope, compressMessageBody, decompressMessageBody, + getCodecName, resolveCodecHandler, ZstdCodecHandler, } from './codec/codecHandler.ts' diff --git a/packages/core/README.md b/packages/core/README.md index d47465ff..6fc1de03 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -643,7 +643,9 @@ class MyPayloadStore implements PayloadStore { #### Message compression (codec) -Publishers can compress outgoing messages with zstd by setting `codec` in their options. Requires **Node.js >=22.15.0** and the [`@message-queue-toolkit/codec`](../codec/README.md) package. +Publishers can compress outgoing messages by setting `codec` in their options. Requires **Node.js >=22.15.0** and the [`@message-queue-toolkit/codec`](../codec/README.md) package. + +**Built-in zstd:** ```typescript import { MessageCodecEnum } from '@message-queue-toolkit/core' @@ -656,7 +658,25 @@ new MyPublisher(deps, { }) ``` -Compressed messages are wrapped in a self-describing envelope `{ __mqtCodec: 'zstd', __mqtData: '' }`. Consumers detect this envelope automatically and decompress transparently — `codec` does not need to be set on the consumer side. +**Custom codec** (bring your own compression library): + +```typescript +import type { MessageCodecHandler } from '@message-queue-toolkit/core' + +class MyLz4Handler implements MessageCodecHandler { + async compress(data: Buffer): Promise { /* ... */ } + async decompress(data: Buffer): Promise { /* ... */ } + createCompressStream(): Transform { /* return a Transform stream */ } +} + +const codec = { name: 'lz4', handler: new MyLz4Handler() } +new MyPublisher(deps, { codec }) +new MyConsumer(deps, { codec }) // same registration required on the consumer +``` + +See the [`@message-queue-toolkit/codec` README](../codec/README.md) for a full custom codec example. + +Compressed messages are wrapped in a self-describing envelope `{ __mqtCodec: '', __mqtData: '' }`. Consumers configured with a matching `codec` registration decompress transparently. Consumers without a matching registration ignore the envelope — `codec` does not need to be set when using the built-in zstd and you are happy with auto-detection. #### Interaction with codec (compression) diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 967e7110..98bcb237 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -20,7 +20,8 @@ const CODEC_FIELD = '__mqtCodec' const DATA_FIELD = '__mqtData' export type CodecEnvelope = { - [CODEC_FIELD]: MessageCodec + // string (not MessageCodec) to accommodate user-supplied codec names. + [CODEC_FIELD]: string [DATA_FIELD]: string } @@ -28,22 +29,75 @@ export type CodecEnvelope = { * Low-level interface for a compression codec. * * Implement this interface to plug in a custom compression algorithm. - * The built-in implementation (`ZstdCodecHandler` in `@message-queue-toolkit/sqs`) + * The built-in implementation (`ZstdCodecHandler` in `@message-queue-toolkit/codec`) * uses Node.js built-in `zlib` zstd support. + * + * All three methods are required: + * - `compress` / `decompress` are used for the inline (non-offloaded) publish path. + * - `createCompressStream` is used by the streaming offload path to pipe serialized + * JSON directly through compression into the payload store without buffering the + * full payload in memory. */ export interface MessageCodecHandler { compress(data: Buffer): Promise decompress(data: Buffer): Promise + /** Returns a Transform stream that compresses its input using this codec. */ + createCompressStream(): import('node:stream').Transform } -export function isCodecEnvelope(value: unknown): value is CodecEnvelope { +/** + * Passed to the `codec` option to select a compression codec. + * + * - **String form** (`MessageCodec`): selects one of the built-in codecs + * (e.g. `MessageCodecEnum.ZSTD`). + * - **Object form** (`{ name, handler }`): plugs in a custom + * `MessageCodecHandler` implementation under a user-chosen name. The name + * is written into the `__mqtCodec` field of every envelope so the consumer + * can identify and route to the correct handler. + * + * @example Built-in zstd + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) + * + * @example Custom codec + * import { LZ4Handler } from './lz4Handler.ts' + * const codec = { name: 'lz4', handler: new LZ4Handler() } + * new MyPublisher(deps, { codec }) + * new MyConsumer(deps, { codec }) // same registration on the consumer side + */ +export type MessageCodecRegistration = MessageCodec | { name: string; handler: MessageCodecHandler } + +/** + * Base64 pattern: groups of 4 chars from the alphabet, with at most 2 trailing `=` pads. + * An empty string (compressed payload of 0 bytes) is also valid. + */ +const BASE64_RE = + /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})?$/ + +/** Built once at module load — avoids a fresh array allocation on every hot-path call. */ +export const KNOWN_CODECS: ReadonlySet = new Set(Object.values(MessageCodecEnum)) + +/** + * Returns true when `value` is a codec envelope that the consumer should decompress. + * + * Pass `knownCodecs` to restrict auto-detection to the codecs your consumer is + * configured to handle (built from the `codec` option). Defaults to the built-in + * codec set — backwards-compatible for consumers that don't configure a codec. + */ +export function isCodecEnvelope( + value: unknown, + knownCodecs: ReadonlySet = KNOWN_CODECS, +): value is CodecEnvelope { const record = value as Record return ( typeof value === 'object' && value !== null && + // Exact two-key shape — extra fields mean this is a real message, not an envelope. + Object.keys(value).length === 2 && CODEC_FIELD in value && DATA_FIELD in value && - (Object.values(MessageCodecEnum) as string[]).includes(record[CODEC_FIELD] as string) && - typeof record[DATA_FIELD] === 'string' + knownCodecs.has(record[CODEC_FIELD] as string) && + typeof record[DATA_FIELD] === 'string' && + // Validate __mqtData is a properly-padded base64 string before handing it to the codec. + BASE64_RE.test(record[DATA_FIELD] as string) ) } diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index 6d23f0ca..b2868728 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,9 +1,11 @@ export { type CodecEnvelope, isCodecEnvelope, + KNOWN_CODECS, type MessageCodec, MessageCodecEnum, type MessageCodecHandler, + type MessageCodecRegistration, } from './codec/messageCodec.ts' export { DoNotProcessMessageError } from './errors/DoNotProcessError.ts' export { diff --git a/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts b/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts index 29bc770f..e6c1dee1 100644 --- a/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts +++ b/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts @@ -15,8 +15,13 @@ export const PAYLOAD_REF_SCHEMA = z.object({ * Codec used to compress the stored payload. * When set, the stored bytes are raw compressed binary (not base64 JSON). * The consumer must decompress using this codec before parsing. + * + * Kept as a plain string to accommodate user-supplied codec names registered + * via the `{ name, handler }` form of the `codec` option — the set of valid + * names is not known statically. Handler resolution validates the name at + * use time and throws if no matching handler is found. */ - codec: z.string().optional(), + codec: z.string().min(1).optional(), }) export type PayloadRef = z.output diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 7899b367..049d6179 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -4,7 +4,6 @@ import * as os from 'node:os' import { Readable } from 'node:stream' import { pipeline } from 'node:stream/promises' import { types } from 'node:util' -import * as zlib from 'node:zlib' import { type CommonLogger, type Either, @@ -20,7 +19,7 @@ import { } from '@message-queue-toolkit/schemas' import { getProperty, setProperty } from 'dot-prop' import type { ZodSchema, ZodType } from 'zod/v4' -import type { MessageCodec } from '../codec/messageCodec.ts' +import type { MessageCodecHandler, MessageCodecRegistration } from '../codec/messageCodec.ts' import type { MessageInvalidFormatError, MessageValidationError } from '../errors/Errors.ts' import { type AcquireLockTimeoutError, @@ -142,8 +141,9 @@ export abstract class AbstractQueueService< protected readonly messageDeduplicationConfig?: MessageDeduplicationConfig protected readonly messageMetricsManager?: MessageMetricsManager protected readonly _handlerSpy?: HandlerSpy - protected readonly codec?: MessageCodec + protected readonly codec?: MessageCodecRegistration protected readonly skipCompressionBelow: number + protected readonly disableCodecAutoDetection: boolean protected isInitted: boolean @@ -183,6 +183,7 @@ export abstract class AbstractQueueService< this.messageDeduplicationConfig = options.messageDeduplicationConfig this.codec = options.codec this.skipCompressionBelow = options.skipCompressionBelow ?? 512 + this.disableCodecAutoDetection = options.disableCodecAutoDetection ?? false this.logMessages = options.logMessages ?? false this._handlerSpy = resolveHandlerSpy(options) @@ -681,14 +682,14 @@ export abstract class AbstractQueueService< payloadId: string, storeName: string, size: number, - codec?: MessageCodec, + codecName?: string, ): OffloadedPayloadPointerPayload { const result: OffloadedPayloadPointerPayload = { payloadRef: { id: payloadId, store: storeName, size, - ...(codec ? { codec } : {}), + ...(codecName ? { codec: codecName } : {}), }, offloadedPayloadPointer: payloadId, offloadedPayloadSize: size, @@ -763,7 +764,7 @@ export abstract class AbstractQueueService< protected async offloadCompressedPayload( message: MessagePayloadSchemas, compressed: Buffer, - codec: MessageCodec, + codecName: string, ): Promise { if (!this.payloadStoreConfig) { throw new Error('Payload store is not configured') @@ -775,7 +776,7 @@ export abstract class AbstractQueueService< size: compressed.byteLength, }) - return this.buildPointer(message, payloadId, storeName, compressed.byteLength, codec) + return this.buildPointer(message, payloadId, storeName, compressed.byteLength, codecName) } /** @@ -801,7 +802,8 @@ export abstract class AbstractQueueService< */ protected async compressAndOffloadPayload( message: MessagePayloadSchemas, - codec: MessageCodec, + handler: MessageCodecHandler, + codecName: string, ): Promise< | { pointer: OffloadedPayloadPointerPayload; compressedBuffer?: never } | { compressedBuffer: Buffer; pointer?: never } @@ -810,22 +812,24 @@ export abstract class AbstractQueueService< throw new Error('Payload store is not configured') } + const tmpPath = `${os.tmpdir()}/${randomUUID()}` const serialized = await this.payloadStoreConfig.serializer.serialize(message) - const tmpPath = `${os.tmpdir()}/${randomUUID()}.zst` try { + // Streaming pipeline: serializer output → codec transform → temp file. + // No full-payload buffer is materialised; each codec supplies its own Transform. await pipeline( typeof serialized.value === 'string' ? Readable.from(serialized.value) : serialized.value, - zlib.createZstdCompress(), + handler.createCompressStream(), fs.createWriteStream(tmpPath), ) const compressedSize = fs.statSync(tmpPath).size // Compare the envelope wire size (not raw compressed bytes) against the threshold. - // buildCodecEnvelope produces {"__mqtCodec":"","__mqtData":""}. + // buildCodecEnvelope produces {"__mqtCodec":"","__mqtData":""}. // Base64 expands by ⌈N/3⌉×4; the fixed JSON framing adds 32 chars + codec name length. - const envelopeSize = Math.ceil((compressedSize * 4) / 3) + 32 + codec.length + const envelopeSize = Math.ceil((compressedSize * 4) / 3) + 32 + codecName.length if (envelopeSize > this.payloadStoreConfig.messageSizeThreshold) { const { store, storeName } = this.resolveOutgoingStore() @@ -833,12 +837,13 @@ export abstract class AbstractQueueService< value: fs.createReadStream(tmpPath), size: compressedSize, }) - return { pointer: this.buildPointer(message, payloadId, storeName, compressedSize, codec) } + return { + pointer: this.buildPointer(message, payloadId, storeName, compressedSize, codecName), + } } // Compressed payload fits inline — return the buffer; caller wraps it in a codec envelope. - const compressedBuffer = fs.readFileSync(tmpPath) - return { compressedBuffer } + return { compressedBuffer: fs.readFileSync(tmpPath) } } finally { try { fs.unlinkSync(tmpPath) diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index 979d72ea..b299caac 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -1,6 +1,6 @@ import type { CommonLogger, ErrorReporter, ErrorResolver } from '@lokalise/node-core' import type { ZodSchema } from 'zod/v4' -import type { MessageCodec } from '../codec/messageCodec.ts' +import type { MessageCodecRegistration } from '../codec/messageCodec.ts' import type { MessageDeduplicationConfig } from '../message-deduplication/messageDeduplicationTypes.ts' import type { PayloadStoreConfig } from '../payload-store/payloadStoreTypes.ts' import type { MessageHandlerConfig } from '../queues/HandlerContainer.ts' @@ -160,7 +160,7 @@ export type CommonQueueOptions = { * // Consumer (optional — auto-detection handles it even without this) * new MyConsumer(deps, { codec: MessageCodecEnum.ZSTD }) */ - codec?: MessageCodec + codec?: MessageCodecRegistration /** * Minimum serialized size in bytes a message must reach before compression is applied. * Only meaningful when `codec` is set. Defaults to `512`. @@ -180,6 +180,20 @@ export type CommonQueueOptions = { * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD, skipCompressionBelow: 0 }) */ skipCompressionBelow?: number + /** + * Disables automatic codec-envelope detection on the consumer. + * + * By default, consumers inspect every incoming message body with `isCodecEnvelope`. + * If the body matches the envelope shape (`__mqtCodec` + `__mqtData` as the only two + * fields), it is treated as compressed and decompressed before schema validation. + * + * Set this to `true` if your message schema legitimately contains fields named + * `__mqtCodec` and `__mqtData` with exactly those two keys, and you do not want + * auto-detection to intercept them. Publisher behaviour is unaffected. + * + * @default false + */ + disableCodecAutoDetection?: boolean } export type CommonCreationConfigType = { diff --git a/packages/core/test/codec/messageCodec.spec.ts b/packages/core/test/codec/messageCodec.spec.ts new file mode 100644 index 00000000..c0b16509 --- /dev/null +++ b/packages/core/test/codec/messageCodec.spec.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest' + +import { isCodecEnvelope, MessageCodecEnum } from '../../lib/codec/messageCodec.ts' + +const VALID_BASE64 = Buffer.from('hello compressed world').toString('base64') + +describe('isCodecEnvelope — custom knownCodecs', () => { + const CUSTOM_CODECS = new Set(['lz4', 'brotli']) + + it('returns true when the envelope codec is in the supplied knownCodecs set', () => { + expect( + isCodecEnvelope({ __mqtCodec: 'lz4', __mqtData: VALID_BASE64 }, CUSTOM_CODECS), + ).toBe(true) + }) + + it('returns false for a built-in codec when it is not in the supplied knownCodecs set', () => { + // zstd is valid with default knownCodecs but must be rejected when not in the custom set + expect( + isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: VALID_BASE64 }, CUSTOM_CODECS), + ).toBe(false) + }) + + it('returns false for a codec that is in neither the default nor the supplied set', () => { + expect( + isCodecEnvelope({ __mqtCodec: 'gzip', __mqtData: VALID_BASE64 }, CUSTOM_CODECS), + ).toBe(false) + }) +}) + +describe('isCodecEnvelope', () => { + describe('valid envelopes', () => { + it('returns true for a well-formed envelope', () => { + expect(isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: VALID_BASE64 })).toBe( + true, + ) + }) + + it('returns true for an envelope with empty base64 data (zero-byte payload)', () => { + expect(isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: '' })).toBe(true) + }) + }) + + describe('extra fields — real messages must not be misclassified', () => { + it('returns false when envelope has an extra field alongside __mqtCodec and __mqtData', () => { + expect( + isCodecEnvelope({ + __mqtCodec: MessageCodecEnum.ZSTD, + __mqtData: VALID_BASE64, + id: 'real-message', + }), + ).toBe(false) + }) + + it('returns false when only __mqtCodec is present (no __mqtData)', () => { + expect(isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD })).toBe(false) + }) + + it('returns false when only __mqtData is present (no __mqtCodec)', () => { + expect(isCodecEnvelope({ __mqtData: VALID_BASE64 })).toBe(false) + }) + }) + + describe('invalid __mqtCodec values', () => { + it('returns false for an unknown codec name', () => { + expect(isCodecEnvelope({ __mqtCodec: 'gzip', __mqtData: VALID_BASE64 })).toBe(false) + }) + + it('returns false when __mqtCodec is not a string', () => { + expect(isCodecEnvelope({ __mqtCodec: 42, __mqtData: VALID_BASE64 })).toBe(false) + }) + }) + + describe('invalid __mqtData values — base64 validation', () => { + it('returns false for a non-base64 string', () => { + expect( + isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: 'not base64!!!' }), + ).toBe(false) + }) + + it('returns false when __mqtData is not a string', () => { + expect(isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: 123 })).toBe(false) + }) + + it('returns false for a base64 string with incorrect padding', () => { + // Valid base64 chars but wrong padding length + expect(isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: 'abc' })).toBe(false) + }) + }) + + describe('non-object inputs', () => { + it('returns false for null', () => { + expect(isCodecEnvelope(null)).toBe(false) + }) + + it('returns false for a string', () => { + expect(isCodecEnvelope('{"__mqtCodec":"zstd","__mqtData":""}')).toBe(false) + }) + + it('returns false for a number', () => { + expect(isCodecEnvelope(42)).toBe(false) + }) + + it('returns false for an empty object', () => { + expect(isCodecEnvelope({})).toBe(false) + }) + }) +}) diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index a7e22203..ac1a4bd5 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -2,7 +2,7 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sns' import { PublishCommand } from '@aws-sdk/client-sns' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' +import { buildCodecEnvelope, getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -219,16 +219,19 @@ export abstract class AbstractSnsPublisher const codec = this.codec if (codec) { + const handler = resolveCodecHandler(codec) + const codecName = getCodecName(codec) + if (this.payloadStoreConfig) { // Streaming path: avoids 3× buffer materialisation for large payloads. - // JSON → zstd → temp file → threshold check → offload or inline envelope. - const result = await this.compressAndOffloadPayload(message, codec) + // JSON → compress → temp file → threshold check → offload or inline envelope. + const result = await this.compressAndOffloadPayload(message, handler, codecName) if (result.pointer) { return { payload: result.pointer } } return { payload: message, - preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codec), + preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codecName), } } @@ -242,8 +245,8 @@ export abstract class AbstractSnsPublisher return { payload: message } } - const compressed = await resolveCodecHandler(codec).compress(jsonBuffer) - return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codec) } + const compressed = await handler.compress(jsonBuffer) + return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codecName) } } return { diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index 9122f464..3cd34268 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -5,7 +5,7 @@ import { SetQueueAttributesCommand, } from '@aws-sdk/client-sqs' import type { Either, ErrorResolver } from '@lokalise/node-core' -import { decompressMessageBody, resolveCodecHandler } from '@message-queue-toolkit/codec' +import { getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' import type { ProcessedMessageMetadata } from '@message-queue-toolkit/core' import { type BarrierResult, @@ -14,6 +14,9 @@ import { HandlerContainer, isCodecEnvelope, isMessageError, + KNOWN_CODECS, + type MessageCodecHandler, + type MessageCodecRegistration, type MessageSchemaContainer, noopReleasableLock, type ParseMessageResult, @@ -77,16 +80,30 @@ type SQSConsumerCommonOptions< PrehandlerOutput, CreationConfigType extends SQSCreationConfig = SQSCreationConfig, QueueLocatorType extends object = SQSQueueLocatorType, -> = QueueConsumerOptions< - CreationConfigType, - QueueLocatorType, - SQSDeadLetterQueueOptions, - MessagePayloadSchemas, - ExecutionContext, - PrehandlerOutput, - SQSCreationConfig, - SQSQueueLocatorType +> = Omit< + QueueConsumerOptions< + CreationConfigType, + QueueLocatorType, + SQSDeadLetterQueueOptions, + MessagePayloadSchemas, + ExecutionContext, + PrehandlerOutput, + SQSCreationConfig, + SQSQueueLocatorType + >, + 'codec' | 'skipCompressionBelow' > & { + /** + * Additional codecs to register on this consumer. + * Built-in codecs (e.g. `MessageCodecEnum.ZSTD`) are always registered automatically. + * Any incoming message whose `__mqtCodec` name is not in the registry causes an error. + * + * Use this to support custom codecs published by a corresponding publisher: + * @example + * const codec = { name: 'lz4', handler: new MyLz4Handler() } + * new MyConsumer(deps, { codecs: [codec] }) + */ + codecs?: MessageCodecRegistration[] /** * Wait time in seconds the consumer passes to SQS ReceiveMessage (long-polling). * AWS allows integer values 0–20; anything else throws a RangeError at @@ -227,6 +244,10 @@ export abstract class AbstractSqsConsumer< private readonly barrierVisibilityExtensionIntervalInMsecs: number private readonly barrierVisibilityTimeoutInSeconds: number private readonly consumerPollingWaitTimeSeconds: number + /** Registry of codec name → handler. Seeded from all built-in codecs + options.codecs. */ + private readonly codecRegistry: ReadonlyMap + /** Precomputed set of codec names in the registry, passed to isCodecEnvelope. */ + private readonly codecKnownNames: ReadonlySet protected deadLetterQueueUrl?: string protected readonly errorResolver: ErrorResolver @@ -278,6 +299,16 @@ export abstract class AbstractSqsConsumer< messageHandlers: options.handlers, }) this.isDeduplicationEnabled = !!options.enableConsumerDeduplication + // Build codec registry: always seed with all built-in codecs, then add user-supplied ones. + const registry = new Map() + for (const builtInName of KNOWN_CODECS) { + registry.set(builtInName, resolveCodecHandler(builtInName as MessageCodecRegistration)) + } + for (const registration of options.codecs ?? []) { + registry.set(getCodecName(registration), resolveCodecHandler(registration)) + } + this.codecRegistry = registry + this.codecKnownNames = new Set(registry.keys()) } override async init(): Promise { @@ -924,8 +955,9 @@ export abstract class AbstractSqsConsumer< if (hasOffloadedPayload(resolveMessageResult.result)) { const retrieveOffloadedMessagePayloadResult = await this.retrieveOffloadedMessagePayload( resolveMessageResult.result.body, - (codec, data) => { - const handler = resolveCodecHandler(codec as Parameters[0]) + (codecName, data) => { + const handler = this.codecRegistry.get(codecName) + if (!handler) throw new Error(`Unknown codec: ${codecName}`) return handler.decompress(data) }, ) @@ -934,10 +966,17 @@ export abstract class AbstractSqsConsumer< return ABORT_EARLY_EITHER } resolveMessageResult.result.body = retrieveOffloadedMessagePayloadResult.result - } else if (isCodecEnvelope(resolveMessageResult.result.body)) { + } else if ( + !this.disableCodecAutoDetection && + isCodecEnvelope(resolveMessageResult.result.body, this.codecKnownNames) + ) { try { - resolveMessageResult.result.body = await decompressMessageBody( - resolveMessageResult.result.body, + const envelope = resolveMessageResult.result.body + const handler = this.codecRegistry.get(envelope.__mqtCodec) + if (!handler) throw new Error(`Unknown codec: ${envelope.__mqtCodec}`) + const compressed = Buffer.from(envelope.__mqtData, 'base64') + resolveMessageResult.result.body = JSON.parse( + (await handler.decompress(compressed)).toString('utf8'), ) } catch (err) { this.handleError(err as Error) diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index 13a2ec3f..1d98bca5 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -2,7 +2,7 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sqs' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' +import { buildCodecEnvelope, getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -213,16 +213,19 @@ export abstract class AbstractSqsPublisher const codec = this.codec if (codec) { + const handler = resolveCodecHandler(codec) + const codecName = getCodecName(codec) + if (this.payloadStoreConfig) { // Streaming path: avoids 3× buffer materialisation for large payloads. - // JSON → zstd → temp file → threshold check → offload or inline envelope. - const result = await this.compressAndOffloadPayload(message, codec) + // JSON → compress → temp file → threshold check → offload or inline envelope. + const result = await this.compressAndOffloadPayload(message, handler, codecName) if (result.pointer) { return { payload: result.pointer } } return { payload: message, - preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codec), + preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codecName), } } @@ -236,8 +239,8 @@ export abstract class AbstractSqsPublisher return { payload: message } } - const compressed = await resolveCodecHandler(codec).compress(jsonBuffer) - return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codec) } + const compressed = await handler.compress(jsonBuffer) + return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codecName) } } return { diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index 25e96b9b..c026c374 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,5 +1,8 @@ +import type { Transform } from 'node:stream' +import { PassThrough } from 'node:stream' import { ReceiveMessageCommand, SendMessageCommand } from '@aws-sdk/client-sqs' import { compressMessageBody } from '@message-queue-toolkit/codec' +import type { MessageCodecHandler } from '@message-queue-toolkit/core' import { MessageCodecEnum } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' @@ -427,4 +430,196 @@ describe('SqsPermissionConsumer - skipCompressionBelow', () => { await wirePublisher.close() await wireConsumer.close(true) }) + + it('consumer with disableCodecAutoDetection passes codec-shaped message through as plain JSON', async () => { + // A publisher with skipCompressionBelow set very high sends plain JSON even though + // codec is configured — the body will NOT be a codec envelope, so this test + // verifies that disableCodecAutoDetection does not break the plain-JSON path. + // More importantly: if the message body were a real envelope, the consumer would + // normally auto-detect and decompress it; with the flag set it must not do so. + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-disable-auto-detect` + await testAdmin.deleteQueues(queueName) + + // Publisher sends plain JSON (skipCompressionBelow prevents compression) + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec: MessageCodecEnum.ZSTD, + skipCompressionBelow: 99_999, + creationConfig: { queue: { QueueName: queueName } }, + }) + await wirePublisher.init() + + // Consumer opts out of auto-detection + const noAutoConsumer = new SqsPermissionConsumer(diContainer.cradle, { + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + disableCodecAutoDetection: true, + }) + await noAutoConsumer.start() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'disable-auto-detect-1', + messageType: 'add', + metadata: { info: 'plain json, no auto-detect' }, + } + await wirePublisher.publish(message) + + const result = await noAutoConsumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await wirePublisher.close() + await noAutoConsumer.close(true) + }) +}) + +// --------------------------------------------------------------------------- +// Custom codec — passthrough handler registered via { name, handler } form +// --------------------------------------------------------------------------- + +/** + * Identity codec: compress/decompress are no-ops, createCompressStream returns a PassThrough. + * Lets us verify the full custom-codec path without a real compression library in tests. + */ +class NoopCodecHandler implements MessageCodecHandler { + compress(data: Buffer): Promise { + return Promise.resolve(data) + } + decompress(data: Buffer): Promise { + return Promise.resolve(data) + } + createCompressStream(): Transform { + return new PassThrough() + } +} + +describe('SqsPermissionConsumer - custom codec registration', () => { + const CUSTOM_CODEC_NAME = 'noop' + + let diContainer: AwilixContainer + let testAdmin: TestAwsResourceAdmin + + beforeAll(async () => { + diContainer = await registerDependencies({ + permissionPublisher: asValue(() => undefined), + permissionConsumer: asValue(() => undefined), + }) + testAdmin = diContainer.cradle.testAdmin + }) + + afterAll(async () => { + const { awilixManager } = diContainer.cradle + await awilixManager.executeDispose() + await diContainer.dispose() + }) + + it('publisher wraps payload in an envelope with the custom codec name', async () => { + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-custom-codec-wire` + await testAdmin.deleteQueues(queueName) + + const codec = { name: CUSTOM_CODEC_NAME, handler: new NoopCodecHandler() } + const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { + codec, + skipCompressionBelow: 0, // always compress so the envelope is always produced + creationConfig: { queue: { QueueName: queueName } }, + }) + await wirePublisher.init() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'custom-codec-wire-1', + messageType: 'add', + } + await wirePublisher.publish(message) + + const { Messages } = await diContainer.cradle.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: wirePublisher.queueProps.url, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 5, + }), + ) + expect(Messages).toBeDefined() + expect(Messages!.length).toBe(1) + + const envelope = JSON.parse(Messages![0]!.Body!) as Record + // Envelope must carry the user-supplied codec name, not 'zstd'. + expect(envelope.__mqtCodec).toBe(CUSTOM_CODEC_NAME) + expect(typeof envelope.__mqtData).toBe('string') + // Since the handler is a no-op, __mqtData decodes to the original JSON. + const decoded = Buffer.from(envelope.__mqtData as string, 'base64').toString('utf8') + expect(JSON.parse(decoded)).toMatchObject(message) + + await wirePublisher.close() + }) + + it('consumer configured with the same custom codec decompresses and processes the message', async () => { + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-custom-codec-roundtrip` + await testAdmin.deleteQueues(queueName) + + const codec = { name: CUSTOM_CODEC_NAME, handler: new NoopCodecHandler() } + + const publisher = new SqsPermissionPublisher(diContainer.cradle, { + codec, + skipCompressionBelow: 0, + creationConfig: { queue: { QueueName: queueName } }, + }) + await publisher.init() + + const consumer = new SqsPermissionConsumer(diContainer.cradle, { + codecs: [codec], + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + }) + await consumer.start() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'custom-codec-roundtrip-1', + messageType: 'add', + metadata: { info: 'custom codec round-trip' }, + } + await publisher.publish(message) + + const result = await consumer.handlerSpy.waitForMessageWithId(message.id, 'consumed') + expect(result.message).toMatchObject(message) + + await publisher.close() + await consumer.close(true) + }) + + it('consumer without the custom codec does not auto-detect envelopes with that codec name', async () => { + // A consumer without extra codecs has codecKnownNames = Set(['zstd']) (built-ins only). + // isCodecEnvelope(body, Set(['zstd'])) returns false for a 'noop' envelope, so the + // raw envelope object reaches schema validation and fails (no messageType field). + // The message is never successfully handled — addCounter stays at 0. + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-custom-codec-no-autodetect` + await testAdmin.deleteQueues(queueName) + + const codec = { name: CUSTOM_CODEC_NAME, handler: new NoopCodecHandler() } + + const publisher = new SqsPermissionPublisher(diContainer.cradle, { + codec, + skipCompressionBelow: 0, + creationConfig: { queue: { QueueName: queueName } }, + }) + await publisher.init() + + // Consumer with no extra codecs — only built-in codecs (e.g. zstd) are registered, not 'noop' + const zstdConsumer = new SqsPermissionConsumer(diContainer.cradle, { + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + }) + await zstdConsumer.start() + + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'custom-codec-no-autodetect-1', + messageType: 'add', + } + await publisher.publish(message) + + // Give the consumer time to attempt processing, then verify no message was consumed. + // The spy can't track by ID because the raw envelope has no top-level `id` field. + await new Promise((resolve) => setTimeout(resolve, 2000)) + expect(zstdConsumer.addCounter).toBe(0) + + await publisher.close() + await zstdConsumer.close(true) + }, 10000) }) diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.ts index e10b38fd..f6415d9e 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.ts @@ -32,7 +32,8 @@ type SqsPermissionConsumerOptions = Pick< | 'payloadStoreConfig' | 'messageDeduplicationConfig' | 'enableConsumerDeduplication' - | 'codec' + | 'codecs' + | 'disableCodecAutoDetection' > & { addPreHandlerBarrier?: ( message: SupportedMessages, @@ -129,7 +130,8 @@ export class SqsPermissionConsumer extends AbstractSqsConsumer< payloadStoreConfig: options.payloadStoreConfig, messageDeduplicationConfig: options.messageDeduplicationConfig, enableConsumerDeduplication: options.enableConsumerDeduplication, - codec: options.codec, + codecs: options.codecs, + disableCodecAutoDetection: options.disableCodecAutoDetection, messageDeduplicationIdField: 'deduplicationId', messageDeduplicationOptionsField: 'deduplicationOptions', handlers: new MessageHandlerConfigBuilder< From 89d37ae26bd2f79588fa19ef7c3569f46316158c Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Thu, 21 May 2026 15:16:00 +0200 Subject: [PATCH 14/23] fix(codec): validate __mqtData is well-formed base64 before decoding in decompressMessageBody Buffer.from(str, 'base64') silently drops non-base64 characters, so a malformed envelope that bypasses isCodecEnvelope would produce garbage bytes and a confusing codec error. Now throws a clear 'not valid base64' error before any decode attempt. Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/lib/codec/codecHandler.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index 0bb5869a..51b5135c 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -8,6 +8,15 @@ import type { } from '@message-queue-toolkit/core' import { MessageCodecEnum } from '@message-queue-toolkit/core' +/** + * Validates that a string is properly-padded base64 before passing it to Buffer.from. + * Buffer.from(str, 'base64') silently ignores non-base64 characters, so without this + * check a malformed __mqtData field produces garbage bytes and a confusing codec error + * instead of a clear "invalid envelope" message. + */ +const BASE64_RE = + /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})?$/ + const ZSTD_UNSUPPORTED_MSG = 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + '@message-queue-toolkit/codec requires Node.js >=22.15.0 or >=23.8.0.' @@ -78,6 +87,11 @@ export function buildCodecEnvelope(compressed: Buffer, codecName: string): strin } export async function decompressMessageBody(envelope: CodecEnvelope): Promise { + if (!BASE64_RE.test(envelope.__mqtData)) { + throw new Error( + `Codec envelope __mqtData is not valid base64 (codec: ${envelope.__mqtCodec})`, + ) + } const handler = resolveCodecHandler(envelope.__mqtCodec as MessageCodecRegistration) const compressed = Buffer.from(envelope.__mqtData, 'base64') const decompressed = await handler.decompress(compressed) From ddf36bec59ec45660751d4eafa6101c2605650dc Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Thu, 21 May 2026 16:53:38 +0200 Subject: [PATCH 15/23] fix(codec): address code-review comments on codec implementation - Guard PubSub and AMQP publishers against unsupported codec option at construction time (throw before super() is called) - Split retrieval vs poison errors in retrieveOffloadedMessagePayload so transient stream failures propagate (retriable) while decompression/ parse failures return an error (DLQ) - Remove dead offloadCompressedPayload method superseded by compressAndOffloadPayload - Fix streamUtils.ts backing-store pin: copy buffer when offset != size - Remove codec re-exports from SQS barrel to avoid dual import-path risk - Pin @types/node to ^22.0.0 in codec package to match engines.node - Add micro-benchmark with CI-assertable timing thresholds - Add tests: base64 validation, PubSub guard, AMQP guard (via minimal pass-through subclass since AmqpPermissionPublisher strips unknown opts) Co-Authored-By: Claude Sonnet 4.6 --- packages/amqp/lib/AbstractAmqpPublisher.ts | 8 ++ .../AmqpPermissionPublisher.spec.ts | 12 +++ packages/codec/lib/codec/codecHandler.ts | 4 +- packages/codec/package.json | 2 +- .../core/lib/queues/AbstractQueueService.ts | 44 +++------- packages/core/lib/utils/streamUtils.ts | 4 +- packages/core/test/codec/messageCodec.spec.ts | 17 ++-- .../lib/pubsub/AbstractPubSubPublisher.ts | 6 ++ .../PubSubPermissionPublisher.spec.ts | 9 +++ packages/sqs/bench/codec.bench.ts | 10 ++- packages/sqs/bench/codecMicro.bench.ts | 80 +++++++++++++++++++ packages/sqs/lib/index.ts | 6 -- packages/sqs/test/codec/codecHandler.spec.ts | 19 +++++ pnpm-lock.yaml | 16 +++- 14 files changed, 184 insertions(+), 53 deletions(-) create mode 100644 packages/sqs/bench/codecMicro.bench.ts create mode 100644 packages/sqs/test/codec/codecHandler.spec.ts diff --git a/packages/amqp/lib/AbstractAmqpPublisher.ts b/packages/amqp/lib/AbstractAmqpPublisher.ts index e9b7aefd..11aaa196 100644 --- a/packages/amqp/lib/AbstractAmqpPublisher.ts +++ b/packages/amqp/lib/AbstractAmqpPublisher.ts @@ -51,6 +51,14 @@ export abstract class AbstractAmqpPublisher< dependencies: AMQPDependencies, options: AMQPPublisherOptions, ) { + // `codec` lives on CommonQueueOptions (added after this package's core peer dep was frozen), + // so it is not present in the local type. The cast guards JS callers and future versions. + if ((options as { codec?: unknown }).codec) { + throw new Error( + 'codec is not supported by AbstractAmqpPublisher. Remove the codec option or use an SQS/SNS publisher.', + ) + } + super(dependencies, options) this.messageSchemaContainer = this.resolvePublisherMessageSchemaContainer(options) diff --git a/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts b/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts index 3822aa82..7962d7e6 100644 --- a/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts +++ b/packages/amqp/test/publishers/AmqpPermissionPublisher.spec.ts @@ -6,6 +6,7 @@ import { asClass, asFunction, Lifetime } from 'awilix' import { asMockFunction } from 'awilix-manager' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' import { ZodError } from 'zod/v4' +import { AbstractAmqpQueuePublisher } from '../../lib/AbstractAmqpQueuePublisher.ts' import { deserializeAmqpMessage } from '../../lib/amqpMessageDeserializer.ts' import { AmqpPermissionConsumer } from '../consumers/AmqpPermissionConsumer.ts' import type { @@ -25,6 +26,17 @@ import { registerDependencies, SINGLETON_CONFIG } from '../utils/testContext.ts' import { AmqpPermissionPublisher } from './AmqpPermissionPublisher.ts' describe('PermissionPublisher', () => { + describe('constructor', () => { + it('throws when codec option is set (codec is not supported by AMQP publishers)', () => { + // AmqpPermissionPublisher strips unknown options before calling super(), so we test + // the guard via a minimal pass-through subclass that mirrors real user code. + class TestPublisher extends AbstractAmqpQueuePublisher<{ messageType: string }> {} + expect(() => new TestPublisher({} as any, { codec: 'zstd' } as any)).toThrow( + 'codec is not supported by AbstractAmqpPublisher', + ) + }) + }) + describe('logging', () => { let logger: FakeLogger let diContainer: AwilixContainer diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index 51b5135c..6794d669 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -88,9 +88,7 @@ export function buildCodecEnvelope(compressed: Buffer, codecName: string): strin export async function decompressMessageBody(envelope: CodecEnvelope): Promise { if (!BASE64_RE.test(envelope.__mqtData)) { - throw new Error( - `Codec envelope __mqtData is not valid base64 (codec: ${envelope.__mqtCodec})`, - ) + throw new Error(`Codec envelope __mqtData is not valid base64 (codec: ${envelope.__mqtCodec})`) } const handler = resolveCodecHandler(envelope.__mqtCodec as MessageCodecRegistration) const compressed = Buffer.from(envelope.__mqtData, 'base64') diff --git a/packages/codec/package.json b/packages/codec/package.json index aca4b8cf..9ab98761 100644 --- a/packages/codec/package.json +++ b/packages/codec/package.json @@ -34,7 +34,7 @@ "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", "@message-queue-toolkit/core": "workspace:*", - "@types/node": "^25.0.2", + "@types/node": "^22.0.0", "rimraf": "^6.0.1", "typescript": "^5.9.3" }, diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 049d6179..967d9033 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -752,33 +752,6 @@ export abstract class AbstractQueueService< return this.buildPointer(message, payloadId, storeName, serializedPayload.size) } - /** - * Stores an already-compressed payload in the configured store. - * The `codec` name is recorded in payloadRef so the consumer can decompress after retrieval. - * - * The threshold check is NOT performed here — callers must decide whether to offload. - * Use this when compression has already been done and the compressed size exceeds the threshold. - * - * @throws Error if payload store is not configured - */ - protected async offloadCompressedPayload( - message: MessagePayloadSchemas, - compressed: Buffer, - codecName: string, - ): Promise { - if (!this.payloadStoreConfig) { - throw new Error('Payload store is not configured') - } - - const { store, storeName } = this.resolveOutgoingStore() - const payloadId = await store.storePayload({ - value: Readable.from(compressed), - size: compressed.byteLength, - }) - - return this.buildPointer(message, payloadId, storeName, compressed.byteLength, codecName) - } - /** * Streaming compress-and-offload path for large payloads (used when both `codec` and * `payloadStoreConfig` are set). @@ -919,11 +892,20 @@ export abstract class AbstractQueueService< const codec = parsedPayload.payloadRef?.codec if (codec && decompress) { + // Stream read is kept outside the try/catch so transient retrieval errors (truncated + // S3 stream, network blip) propagate as thrown exceptions rather than being caught and + // returned as { error }. The caller (consumer) lets unhandled throws bubble up to + // sqs-consumer, which does NOT delete the message — it becomes visible again after the + // visibility timeout and is retried. Only deterministic failures (wrong codec, corrupt + // compressed bytes, invalid JSON after decompression) are caught and returned as + // { error }, which the consumer treats as a poison message and routes to the DLQ. + // This mirrors the non-codec path below, where streamWithKnownSizeToString is also + // outside its try/catch for the same reason. + const compressedBuffer = await streamWithKnownSizeToBuffer( + serializedOffloadedPayloadReadable, + payloadSize, + ) try { - const compressedBuffer = await streamWithKnownSizeToBuffer( - serializedOffloadedPayloadReadable, - payloadSize, - ) const decompressed = await decompress(codec, compressedBuffer) return { result: JSON.parse(decompressed.toString('utf8')) } } catch (e) { diff --git a/packages/core/lib/utils/streamUtils.ts b/packages/core/lib/utils/streamUtils.ts index 7e69617b..b94c79f7 100644 --- a/packages/core/lib/utils/streamUtils.ts +++ b/packages/core/lib/utils/streamUtils.ts @@ -14,7 +14,9 @@ export async function streamWithKnownSizeToBuffer(stream: Readable, size: number offset += chunkBuffer.length } - return buffer.subarray(0, offset) + // Copy only when the stream delivered fewer bytes than expected so the + // full backing allocation is not retained via a shared-memory view. + return offset === size ? buffer : Buffer.from(buffer.subarray(0, offset)) } export async function streamWithKnownSizeToString(stream: Readable, size: number): Promise { diff --git a/packages/core/test/codec/messageCodec.spec.ts b/packages/core/test/codec/messageCodec.spec.ts index c0b16509..c383ffba 100644 --- a/packages/core/test/codec/messageCodec.spec.ts +++ b/packages/core/test/codec/messageCodec.spec.ts @@ -8,22 +8,25 @@ describe('isCodecEnvelope — custom knownCodecs', () => { const CUSTOM_CODECS = new Set(['lz4', 'brotli']) it('returns true when the envelope codec is in the supplied knownCodecs set', () => { - expect( - isCodecEnvelope({ __mqtCodec: 'lz4', __mqtData: VALID_BASE64 }, CUSTOM_CODECS), - ).toBe(true) + expect(isCodecEnvelope({ __mqtCodec: 'lz4', __mqtData: VALID_BASE64 }, CUSTOM_CODECS)).toBe( + true, + ) }) it('returns false for a built-in codec when it is not in the supplied knownCodecs set', () => { // zstd is valid with default knownCodecs but must be rejected when not in the custom set expect( - isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: VALID_BASE64 }, CUSTOM_CODECS), + isCodecEnvelope( + { __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: VALID_BASE64 }, + CUSTOM_CODECS, + ), ).toBe(false) }) it('returns false for a codec that is in neither the default nor the supplied set', () => { - expect( - isCodecEnvelope({ __mqtCodec: 'gzip', __mqtData: VALID_BASE64 }, CUSTOM_CODECS), - ).toBe(false) + expect(isCodecEnvelope({ __mqtCodec: 'gzip', __mqtData: VALID_BASE64 }, CUSTOM_CODECS)).toBe( + false, + ) }) }) diff --git a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts index 067fdc58..aa96a474 100644 --- a/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts +++ b/packages/gcp-pubsub/lib/pubsub/AbstractPubSubPublisher.ts @@ -42,6 +42,12 @@ export abstract class AbstractPubSubPublisher MessagePayloadType >, ) { + if (options.codec) { + throw new Error( + 'codec is not supported by AbstractPubSubPublisher. Remove the codec option or use an SQS/SNS publisher.', + ) + } + super(dependencies, options) this.messageSchemaContainer = this.resolvePublisherMessageSchemaContainer(options) diff --git a/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.spec.ts b/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.spec.ts index 8e99e475..29d3c873 100644 --- a/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.spec.ts +++ b/packages/gcp-pubsub/test/publishers/PubSubPermissionPublisher.spec.ts @@ -23,6 +23,15 @@ describe('PubSubPermissionPublisher', () => { await diContainer.dispose() }) + describe('constructor', () => { + it('throws when codec option is set (codec is not supported by Pub/Sub publishers)', () => { + // The guard fires before super() so real dependencies are not required. + expect(() => new PubSubPermissionPublisher({} as any, { codec: 'zstd' } as any)).toThrow( + 'codec is not supported by AbstractPubSubPublisher', + ) + }) + }) + describe('init', () => { it('creates a new topic', async () => { const newPublisher = diContainer.cradle.permissionPublisher diff --git a/packages/sqs/bench/codec.bench.ts b/packages/sqs/bench/codec.bench.ts index 600a7ca9..fe987433 100644 --- a/packages/sqs/bench/codec.bench.ts +++ b/packages/sqs/bench/codec.bench.ts @@ -1,11 +1,17 @@ /** - * Codec benchmarks — publish and consume throughput with vs without zstd compression. + * Codec integration benchmarks — publish and consume throughput with vs without zstd compression. * * Run: pnpm --filter @message-queue-toolkit/sqs bench * * Each benchmark pre-fills queues (consume) or sends N messages (publish) and * measures wall-clock time, reporting msg/s and the overhead percentage. * All queues are deleted before and after each case. + * + * LIMITATION: N=50 against LocalStack means results are dominated by network + * round-trips (~5–20 ms each), not compression CPU cost (~0.1–1 ms). These + * numbers show end-to-end throughput, not codec overhead. For the codec CPU cost + * in isolation see bench/codecMicro.bench.ts, which is assertable in CI. + * These integration benchmarks print to console only and cannot catch regressions. */ import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' @@ -178,8 +184,8 @@ describe('SQS codec benchmarks', () => { await plainCon.close(true) // ── Measure codec consume ── + // No codec option needed — consumers auto-detect envelopes from __mqtCodec. const codecCon = new SqsPermissionConsumer(diContainer.cradle, { - codec: 'zstd', creationConfig: { queue: { QueueName: codecQ } }, deletionConfig: { deleteIfExists: false }, }) diff --git a/packages/sqs/bench/codecMicro.bench.ts b/packages/sqs/bench/codecMicro.bench.ts new file mode 100644 index 00000000..e41f45ce --- /dev/null +++ b/packages/sqs/bench/codecMicro.bench.ts @@ -0,0 +1,80 @@ +/** + * Codec micro-benchmarks — compress/decompress latency in isolation (no network). + * + * Run: pnpm --filter @message-queue-toolkit/sqs bench + * + * These tests measure the CPU cost of the codec only, free from LocalStack + * network noise. Each case runs ITERATIONS compress+decompress round-trips and + * asserts the total time is below a very conservative ceiling, making them safe + * to run in CI as regression guards. + * + * Expected times on typical developer/CI hardware: + * small payload (~100 B) → ~20–100 ms for 100 iterations + * large payload (~6 KB) → ~50–300 ms for 100 iterations + * + * The ceilings below are set at ~10× the expected worst case so that only a + * genuine algorithmic regression (or a severely starved CI runner) will fail. + */ +import { ZstdCodecHandler } from '@message-queue-toolkit/codec' +import { describe, expect, it } from 'vitest' + +const handler = new ZstdCodecHandler() +const ITERATIONS = 100 + +/** Small message — representative of the "skip compression" boundary (~100 B). */ +const SMALL = Buffer.from(JSON.stringify({ id: 'bench-small', messageType: 'add' }), 'utf8') + +/** Large message — repetitive text that compresses very well (~6 KB). */ +const LARGE = Buffer.from( + JSON.stringify({ + id: 'bench-large', + messageType: 'add', + metadata: { + description: 'The quick brown fox jumps over the lazy dog. '.repeat(60), + items: Array.from({ length: 80 }, (_, i) => ({ + id: `item-${i}`, + value: `value-number-${i}`, + enabled: i % 2 === 0, + })), + }, + }), + 'utf8', +) + +describe('ZstdCodecHandler micro-benchmark', () => { + it(`compress+decompress ${ITERATIONS}× small payload (${SMALL.byteLength} B) in under 2 s`, async () => { + const t0 = performance.now() + for (let i = 0; i < ITERATIONS; i++) { + const compressed = await handler.compress(SMALL) + await handler.decompress(compressed) + } + const elapsed = performance.now() - t0 + const perOp = (elapsed / ITERATIONS).toFixed(2) + console.log( + ` small: ${ITERATIONS} round-trips in ${elapsed.toFixed(0)} ms (${perOp} ms/op, ` + + `${((SMALL.byteLength * ITERATIONS) / elapsed / 1000).toFixed(2)} MB/s input)`, + ) + expect( + elapsed, + `${ITERATIONS} round-trips took ${elapsed.toFixed(0)} ms — possible regression`, + ).toBeLessThan(2000) + }) + + it(`compress+decompress ${ITERATIONS}× large payload (${LARGE.byteLength} B) in under 5 s`, async () => { + const t0 = performance.now() + for (let i = 0; i < ITERATIONS; i++) { + const compressed = await handler.compress(LARGE) + await handler.decompress(compressed) + } + const elapsed = performance.now() - t0 + const perOp = (elapsed / ITERATIONS).toFixed(2) + console.log( + ` large: ${ITERATIONS} round-trips in ${elapsed.toFixed(0)} ms (${perOp} ms/op, ` + + `${((LARGE.byteLength * ITERATIONS) / elapsed / 1000).toFixed(2)} MB/s input)`, + ) + expect( + elapsed, + `${ITERATIONS} round-trips took ${elapsed.toFixed(0)} ms — possible regression`, + ).toBeLessThan(5000) + }) +}) diff --git a/packages/sqs/lib/index.ts b/packages/sqs/lib/index.ts index 40828db2..49d3803c 100644 --- a/packages/sqs/lib/index.ts +++ b/packages/sqs/lib/index.ts @@ -1,9 +1,3 @@ -export { - compressMessageBody, - decompressMessageBody, - resolveCodecHandler, - ZstdCodecHandler, -} from '@message-queue-toolkit/codec' export { SqsConsumerErrorResolver } from './errors/SqsConsumerErrorResolver.ts' export { FakeConsumerErrorResolver } from './fakes/FakeConsumerErrorResolver.ts' export { TestSqsPublisher, type TestSqsPublishOptions } from './fakes/TestSqsPublisher.ts' diff --git a/packages/sqs/test/codec/codecHandler.spec.ts b/packages/sqs/test/codec/codecHandler.spec.ts new file mode 100644 index 00000000..86732f35 --- /dev/null +++ b/packages/sqs/test/codec/codecHandler.spec.ts @@ -0,0 +1,19 @@ +import { decompressMessageBody } from '@message-queue-toolkit/codec' +import { MessageCodecEnum } from '@message-queue-toolkit/core' +import { describe, expect, it } from 'vitest' + +describe('decompressMessageBody', () => { + it('throws a descriptive error when __mqtData is not valid base64', async () => { + await expect( + decompressMessageBody({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: 'not-base64!!!' }), + ).rejects.toThrow('Codec envelope __mqtData is not valid base64 (codec: zstd)') + }) + + it('throws a descriptive error for base64 with incorrect padding', async () => { + // Valid base64 characters but wrong padding — Buffer.from would silently accept this + // and produce garbage bytes; the guard must catch it before the codec is invoked. + await expect( + decompressMessageBody({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: 'abc' }), + ).rejects.toThrow('Codec envelope __mqtData is not valid base64 (codec: zstd)') + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cf6ea3d5..e60ba8a8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -79,8 +79,8 @@ importers: specifier: workspace:* version: link:../core '@types/node': - specifier: ^25.0.2 - version: 25.8.0 + specifier: ^22.0.0 + version: 22.19.19 rimraf: specifier: ^6.0.1 version: 6.1.3 @@ -1434,6 +1434,9 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/node@22.19.19': + resolution: {integrity: sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==} + '@types/node@25.8.0': resolution: {integrity: sha512-TCFSk8IZh+iLX1xtksoBVtdmgL+1IX0fC9BeU4QqFSuNdN/K+HUlhqOzEmSYYpZUVsLYcPqc9KX+60iDuninSQ==} @@ -2610,6 +2613,9 @@ packages: resolution: {integrity: sha512-xY7c8LPyzvhvew0Fn+Ek3wBC9STZAuDI/Y5andCKi9AX6/jvfaX45PhsDX8oxgPL0YFp0Jhr8qWMbS/p9375Xg==} engines: {node: '>=16'} + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} @@ -3707,6 +3713,10 @@ snapshots: '@types/estree@1.0.9': {} + '@types/node@22.19.19': + dependencies: + undici-types: 6.21.0 + '@types/node@25.8.0': dependencies: undici-types: 7.24.6 @@ -4936,6 +4946,8 @@ snapshots: dependencies: layerr: 3.0.0 + undici-types@6.21.0: {} + undici-types@7.24.6: {} util-deprecate@1.0.2: {} From a938f91bd79b3b9422d66e29c5dab3882d71244a Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Thu, 21 May 2026 17:08:18 +0200 Subject: [PATCH 16/23] perf(core): replace blocking fs sync calls with async equivalents fs.statSync, fs.readFileSync, and fs.unlinkSync in compressAndOffloadPayload stall the event loop; replace with fs.promises.stat, fs.promises.readFile, and fs.promises.unlink respectively. Semantics are unchanged. Co-Authored-By: Claude Sonnet 4.6 --- packages/core/lib/queues/AbstractQueueService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 967d9033..7949e984 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -797,7 +797,7 @@ export abstract class AbstractQueueService< fs.createWriteStream(tmpPath), ) - const compressedSize = fs.statSync(tmpPath).size + const compressedSize = (await fs.promises.stat(tmpPath)).size // Compare the envelope wire size (not raw compressed bytes) against the threshold. // buildCodecEnvelope produces {"__mqtCodec":"","__mqtData":""}. @@ -816,10 +816,10 @@ export abstract class AbstractQueueService< } // Compressed payload fits inline — return the buffer; caller wraps it in a codec envelope. - return { compressedBuffer: fs.readFileSync(tmpPath) } + return { compressedBuffer: await fs.promises.readFile(tmpPath) } } finally { try { - fs.unlinkSync(tmpPath) + await fs.promises.unlink(tmpPath) } catch { // ignore cleanup errors } From 1f3476627bd8d348f9cc008a223ce76940e6a2c0 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Thu, 21 May 2026 17:12:38 +0200 Subject: [PATCH 17/23] feat(codec): validate custom codec names at registration time Add SAFE_CODEC_NAME_RE guard in getCodecName() for object-form registrations so names containing JSON-unsafe characters (quotes, backslashes, whitespace, etc.) throw a clear error at startup instead of producing malformed envelope JSON at message-send time. Allowed charset: ASCII letters, digits, hyphens, underscores. Co-Authored-By: Claude Sonnet 4.6 --- packages/codec/lib/codec/codecHandler.ts | 18 +++++++++- packages/sqs/test/codec/codecHandler.spec.ts | 35 +++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index 6794d669..978b2cf8 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -46,11 +46,27 @@ export class ZstdCodecHandler implements MessageCodecHandler { const ZSTD_HANDLER = new ZstdCodecHandler() +/** + * Allowed characters for a custom codec name: ASCII letters, digits, hyphens, underscores. + * This keeps the name JSON-safe without escaping and makes it a recognisable identifier. + */ +const SAFE_CODEC_NAME_RE = /^[A-Za-z0-9_-]+$/ + /** * Returns the name string that will be written into the `__mqtCodec` field of every envelope. + * Throws for custom (object-form) registrations whose name contains characters that would + * produce invalid JSON when interpolated raw into the envelope string. */ export function getCodecName(codec: MessageCodecRegistration): string { - return typeof codec === 'string' ? codec : codec.name + if (typeof codec === 'object') { + if (!SAFE_CODEC_NAME_RE.test(codec.name)) { + throw new Error( + `Invalid codec name "${codec.name}": only ASCII letters, digits, hyphens, and underscores are allowed`, + ) + } + return codec.name + } + return codec } /** diff --git a/packages/sqs/test/codec/codecHandler.spec.ts b/packages/sqs/test/codec/codecHandler.spec.ts index 86732f35..8addbc8d 100644 --- a/packages/sqs/test/codec/codecHandler.spec.ts +++ b/packages/sqs/test/codec/codecHandler.spec.ts @@ -1,7 +1,40 @@ -import { decompressMessageBody } from '@message-queue-toolkit/codec' +import { decompressMessageBody, getCodecName } from '@message-queue-toolkit/codec' import { MessageCodecEnum } from '@message-queue-toolkit/core' import { describe, expect, it } from 'vitest' +describe('getCodecName', () => { + it('returns the string as-is for built-in codec enum values', () => { + expect(getCodecName(MessageCodecEnum.ZSTD)).toBe('zstd') + }) + + it('returns the name for a valid custom codec registration', () => { + expect(getCodecName({ name: 'lz4', handler: {} as any })).toBe('lz4') + expect(getCodecName({ name: 'my-codec_v2', handler: {} as any })).toBe('my-codec_v2') + }) + + it('throws for a custom codec name containing a double-quote', () => { + expect(() => getCodecName({ name: 'lz4"x', handler: {} as any })).toThrow( + 'Invalid codec name "lz4"x"', + ) + }) + + it('throws for a custom codec name containing a backslash', () => { + expect(() => getCodecName({ name: 'lz4\\x', handler: {} as any })).toThrow( + 'Invalid codec name "lz4\\x"', + ) + }) + + it('throws for a custom codec name containing whitespace', () => { + expect(() => getCodecName({ name: 'my codec', handler: {} as any })).toThrow( + 'Invalid codec name "my codec"', + ) + }) + + it('throws for an empty custom codec name', () => { + expect(() => getCodecName({ name: '', handler: {} as any })).toThrow('Invalid codec name ""') + }) +}) + describe('decompressMessageBody', () => { it('throws a descriptive error when __mqtData is not valid base64', async () => { await expect( From abe0c61c7f939f6c8b560f67b78a36c2652ed9ad Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Fri, 22 May 2026 10:48:30 +0200 Subject: [PATCH 18/23] fix(codec): address second round of code-review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 — SNS consumer re-exposed codec/skipCompressionBelow via SNSOptions intersection; strip them with Omit (not at the top level, to preserve the SQSConsumerOptions fifoQueue discriminated union). Update SnsSqsPermissionConsumer fixture and codec spec to use codecs[] instead of the now-removed codec option. P1 — Stale README examples updated: consumer codec option → codecs array, compressed-size threshold description → codec envelope wire size. P2 — Fix base64 size estimate: Math.ceil(N/3)*4 (exact) instead of Math.ceil(N*4/3) (underestimates by up to 2 bytes near protocol limit). P2 — Move dedup check before prepareOutgoingPayload in both SQS and SNS publishers so duplicates skip compression and S3 upload entirely. P2 — Bump @message-queue-toolkit/core peer-dep floor to >=25.5.0 in codec, sqs, and sns packages (codec types were added in this version). P3 — Export BASE64_RE from core and import it in the codec package to eliminate the duplicate regex definition. P3 — Migrate AMQP devDep to workspace:* so options.codec resolves via the local TypeScript types and the (options as {codec?:unknown}) cast can be removed. P3 — Remove dead !handler guard in AbstractSqsConsumer inline-envelope path (isCodecEnvelope already guarantees the key is in codecRegistry). P3 — Add JSDoc to decompressMessageBody noting it only handles built-in codecs; custom-codec decoding goes through the consumer's registry. Co-Authored-By: Claude Sonnet 4.6 --- packages/amqp/lib/AbstractAmqpPublisher.ts | 4 +--- packages/amqp/package.json | 2 +- packages/codec/README.md | 6 ++--- packages/codec/lib/codec/codecHandler.ts | 22 ++++++++++--------- packages/codec/package.json | 2 +- packages/core/README.md | 10 ++++----- packages/core/lib/codec/messageCodec.ts | 3 ++- packages/core/lib/index.ts | 1 + .../core/lib/queues/AbstractQueueService.ts | 2 +- packages/sns/lib/sns/AbstractSnsPublisher.ts | 17 +++++++------- .../sns/lib/sns/AbstractSnsSqsConsumer.ts | 4 +++- packages/sns/package.json | 2 +- .../SnsSqsPermissionConsumer.codec.spec.ts | 5 ++--- .../consumers/SnsSqsPermissionConsumer.ts | 4 ++-- packages/sqs/README.md | 15 +++++-------- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 4 ++-- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 17 +++++++------- packages/sqs/package.json | 2 +- pnpm-lock.yaml | 4 ++-- 19 files changed, 63 insertions(+), 63 deletions(-) diff --git a/packages/amqp/lib/AbstractAmqpPublisher.ts b/packages/amqp/lib/AbstractAmqpPublisher.ts index 11aaa196..2f1a478b 100644 --- a/packages/amqp/lib/AbstractAmqpPublisher.ts +++ b/packages/amqp/lib/AbstractAmqpPublisher.ts @@ -51,9 +51,7 @@ export abstract class AbstractAmqpPublisher< dependencies: AMQPDependencies, options: AMQPPublisherOptions, ) { - // `codec` lives on CommonQueueOptions (added after this package's core peer dep was frozen), - // so it is not present in the local type. The cast guards JS callers and future versions. - if ((options as { codec?: unknown }).codec) { + if (options.codec) { throw new Error( 'codec is not supported by AbstractAmqpPublisher. Remove the codec option or use an SQS/SNS publisher.', ) diff --git a/packages/amqp/package.json b/packages/amqp/package.json index 9bd0d2d2..d55f7fde 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -43,7 +43,7 @@ "@biomejs/biome": "^2.3.8", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/core": "*", + "@message-queue-toolkit/core": "workspace:*", "@types/amqplib": "0.10.8", "@types/node": "^25.5.0", "@vitest/coverage-v8": "^4.0.18", diff --git a/packages/codec/README.md b/packages/codec/README.md index 9ab0e6af..c7b25a6b 100644 --- a/packages/codec/README.md +++ b/packages/codec/README.md @@ -88,11 +88,11 @@ const codec = { name: 'lz4', handler: new MyLz4Handler() } // Publisher — wraps each outgoing message in { __mqtCodec: 'lz4', __mqtData: '' } new MyPublisher(deps, { codec }) -// Consumer — only auto-detects envelopes whose __mqtCodec matches 'lz4' -new MyConsumer(deps, { codec }) +// Consumer — register the custom codec so envelopes with __mqtCodec 'lz4' are decompressed +new MyConsumer(deps, { codecs: [codec] }) ``` -**Consumer-side scoping.** A consumer configured with `{ name: 'lz4', handler }` will only decompress envelopes that carry `__mqtCodec: 'lz4'`. A consumer configured with the built-in `MessageCodecEnum.ZSTD` will ignore `lz4` envelopes entirely — they reach schema validation as raw objects and are rejected. This prevents accidental cross-codec decompression. +**Consumer-side scoping.** A consumer registered with `{ name: 'lz4', handler }` via `codecs` will only decompress envelopes that carry `__mqtCodec: 'lz4'`. Built-in codecs (e.g. zstd) are always auto-registered — no `codecs` entry needed for them. This prevents accidental cross-codec decompression. ## Codec envelope format diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/codec/lib/codec/codecHandler.ts index 978b2cf8..65799cf2 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/codec/lib/codec/codecHandler.ts @@ -6,16 +6,7 @@ import type { MessageCodecHandler, MessageCodecRegistration, } from '@message-queue-toolkit/core' -import { MessageCodecEnum } from '@message-queue-toolkit/core' - -/** - * Validates that a string is properly-padded base64 before passing it to Buffer.from. - * Buffer.from(str, 'base64') silently ignores non-base64 characters, so without this - * check a malformed __mqtData field produces garbage bytes and a confusing codec error - * instead of a clear "invalid envelope" message. - */ -const BASE64_RE = - /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})?$/ +import { BASE64_RE, MessageCodecEnum } from '@message-queue-toolkit/core' const ZSTD_UNSUPPORTED_MSG = 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + @@ -102,6 +93,17 @@ export function buildCodecEnvelope(compressed: Buffer, codecName: string): strin return '{"__mqtCodec":"' + codecName + '","__mqtData":"' + compressed.toString('base64') + '"}' } +/** + * Decompresses a codec envelope produced by {@link compressMessageBody} or + * {@link buildCodecEnvelope} and returns the original parsed JSON value. + * + * **Built-in codecs only.** This utility resolves the handler via + * {@link resolveCodecHandler}, which only recognises built-in codec names + * (e.g. `MessageCodecEnum.ZSTD`). Calling it with a custom-codec envelope + * (where `__mqtCodec` is a user-chosen name) will throw "Unsupported codec". + * Consumer-side decoding of custom codecs is handled automatically via the + * consumer's codec registry; this function is intended for one-off, built-in use cases. + */ export async function decompressMessageBody(envelope: CodecEnvelope): Promise { if (!BASE64_RE.test(envelope.__mqtData)) { throw new Error(`Codec envelope __mqtData is not valid base64 (codec: ${envelope.__mqtCodec})`) diff --git a/packages/codec/package.json b/packages/codec/package.json index 9ab98761..45d4ccad 100644 --- a/packages/codec/package.json +++ b/packages/codec/package.json @@ -27,7 +27,7 @@ "node": ">=22.15.0" }, "peerDependencies": { - "@message-queue-toolkit/core": ">=25.0.0" + "@message-queue-toolkit/core": ">=25.5.0" }, "devDependencies": { "@biomejs/biome": "^2.3.8", diff --git a/packages/core/README.md b/packages/core/README.md index 6fc1de03..dcac7667 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -671,21 +671,21 @@ class MyLz4Handler implements MessageCodecHandler { const codec = { name: 'lz4', handler: new MyLz4Handler() } new MyPublisher(deps, { codec }) -new MyConsumer(deps, { codec }) // same registration required on the consumer +new MyConsumer(deps, { codecs: [codec] }) // register custom codec on the consumer ``` See the [`@message-queue-toolkit/codec` README](../codec/README.md) for a full custom codec example. -Compressed messages are wrapped in a self-describing envelope `{ __mqtCodec: '', __mqtData: '' }`. Consumers configured with a matching `codec` registration decompress transparently. Consumers without a matching registration ignore the envelope — `codec` does not need to be set when using the built-in zstd and you are happy with auto-detection. +Compressed messages are wrapped in a self-describing envelope `{ __mqtCodec: '', __mqtData: '' }`. Built-in codecs (e.g. zstd) are auto-detected on every consumer — no consumer option needed. For custom codecs, pass `codecs: [{ name, handler }]` to register them on the consumer. #### Interaction with codec (compression) When both `codec` and `payloadStoreConfig` are set on a publisher, compression and offloading work together with a single compression pass: 1. The message is compressed **once** at publish time. -2. The **compressed** size is compared against `messageSizeThreshold`. -3. If the compressed size exceeds the threshold, the raw compressed bytes are stored in the payload store. The codec name is written to `payloadRef.codec` so the consumer knows how to decompress after retrieval. -4. If the compressed size fits within the threshold, the message is sent inline as a self-describing codec envelope — S3 is never touched. +2. The **codec envelope wire size** (base64-encoded compressed bytes + JSON framing) is compared against `messageSizeThreshold`. +3. If the envelope size exceeds the threshold, the raw compressed bytes are stored in the payload store. The codec name is written to `payloadRef.codec` so the consumer knows how to decompress after retrieval. +4. If the envelope size fits within the threshold, the message is sent inline as a self-describing codec envelope — S3 is never touched. This means compression can prevent offloading entirely for messages that are large before compression but small after. diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 98bcb237..016d5da4 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -69,8 +69,9 @@ export type MessageCodecRegistration = MessageCodec | { name: string; handler: M /** * Base64 pattern: groups of 4 chars from the alphabet, with at most 2 trailing `=` pads. * An empty string (compressed payload of 0 bytes) is also valid. + * Exported so codec implementations can reuse it without duplicating the regex. */ -const BASE64_RE = +export const BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})?$/ /** Built once at module load — avoids a fresh array allocation on every hot-path call. */ diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index b2868728..6426848b 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,4 +1,5 @@ export { + BASE64_RE, type CodecEnvelope, isCodecEnvelope, KNOWN_CODECS, diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 7949e984..face2cc7 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -802,7 +802,7 @@ export abstract class AbstractQueueService< // Compare the envelope wire size (not raw compressed bytes) against the threshold. // buildCodecEnvelope produces {"__mqtCodec":"","__mqtData":""}. // Base64 expands by ⌈N/3⌉×4; the fixed JSON framing adds 32 chars + codec name length. - const envelopeSize = Math.ceil((compressedSize * 4) / 3) + 32 + codecName.length + const envelopeSize = Math.ceil(compressedSize / 3) * 4 + 32 + codecName.length if (envelopeSize > this.payloadStoreConfig.messageSizeThreshold) { const { store, storeName } = this.resolveOutgoingStore() diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index ac1a4bd5..591706bf 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -126,14 +126,7 @@ export abstract class AbstractSnsPublisher const topicName = this.locatorConfig?.topicName ?? this.creationConfig?.topic?.Name ?? 'unknown' - const updatedMessage = this.updateInternalProperties(message) - - // Resolve FIFO options from original message BEFORE offloading - // (offloaded payload won't have user fields needed for messageGroupIdField) - const resolvedOptions = this.resolveFifoOptions(updatedMessage, options) - - const { payload, preBuiltBody } = await this.prepareOutgoingPayload(updatedMessage) - + // Dedup check before compression/offload: skip expensive work for duplicates. if ( this.isDeduplicationEnabledForMessage(parsedMessage) && (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) @@ -148,6 +141,14 @@ export abstract class AbstractSnsPublisher return } + const updatedMessage = this.updateInternalProperties(message) + + // Resolve FIFO options from original message BEFORE offloading + // (offloaded payload won't have user fields needed for messageGroupIdField) + const resolvedOptions = this.resolveFifoOptions(updatedMessage, options) + + const { payload, preBuiltBody } = await this.prepareOutgoingPayload(updatedMessage) + await this.sendMessage(payload, resolvedOptions, preBuiltBody) this.handleMessageProcessed({ diff --git a/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts index 37f2d857..2033c957 100644 --- a/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts +++ b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts @@ -41,7 +41,9 @@ export type SNSSQSConsumerOptions< SNSSQSCreationConfig, SNSSQSQueueLocatorType > & - SNSOptions & { + // Omit here instead of at the top level so the SQSConsumerOptions discriminated + // union (fifoQueue: true | false) is preserved and Extract<…, {fifoQueue:true}> works. + Omit & { subscriptionConfig?: SNSSubscriptionOptions } diff --git a/packages/sns/package.json b/packages/sns/package.json index 39684782..5c6d232e 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -39,7 +39,7 @@ "@aws-sdk/client-sqs": "^3.1034.0", "@aws-sdk/client-sts": "^3.632.0", "@message-queue-toolkit/codec": ">=1.0.0", - "@message-queue-toolkit/core": ">=24.0.0", + "@message-queue-toolkit/core": ">=25.5.0", "@message-queue-toolkit/schemas": ">=7.0.0", "@message-queue-toolkit/sqs": ">=23.0.0", "zod": ">=3.25.76 <5.0.0" diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts index bdf311df..3a026457 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts @@ -27,9 +27,8 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { await testAdmin.deleteQueues(SnsSqsPermissionConsumer.CONSUMED_QUEUE_NAME) await testAdmin.deleteTopics(SnsSqsPermissionConsumer.SUBSCRIBED_TOPIC_NAME) - consumer = new SnsSqsPermissionConsumer(diContainer.cradle, { - codec: 'zstd', - }) + // No codec option needed — zstd is auto-registered on every consumer. + consumer = new SnsSqsPermissionConsumer(diContainer.cradle) publisher = new SnsPermissionPublisher(diContainer.cradle, { codec: 'zstd', }) diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts index 5d3548a5..b4f967ae 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.ts @@ -41,7 +41,7 @@ type SnsSqsPermissionConsumerOptions = Pick< | 'maxRetryDuration' | 'payloadStoreConfig' | 'concurrentConsumersAmount' - | 'codec' + | 'codecs' > & { addPreHandlerBarrier?: ( message: SupportedMessages, @@ -149,7 +149,7 @@ export class SnsSqsPermissionConsumer extends AbstractSnsSqsConsumer< deleteIfExists: false, }, payloadStoreConfig: options.payloadStoreConfig, - codec: options.codec, + codecs: options.codecs, consumerOverrides: options.consumerOverrides ?? { terminateVisibilityTimeout: true, // this allows to retry failed messages immediately }, diff --git a/packages/sqs/README.md b/packages/sqs/README.md index f6b3961e..db235279 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -544,10 +544,8 @@ When using `locatorConfig`, you connect to an existing queue without creating it payloadStore: s3Store, }, - // Optional - Compression (Node.js >=22.15.0 required) - // Auto-detection is always active: consumers decompress codec envelopes - // even without this option set. - codec: MessageCodecEnum.ZSTD, + // Note: consumers have no `codec` option — auto-detection handles built-in zstd. + // Use `codecs: [{ name: 'lz4', handler: new LZ4Handler() }]` only for custom codecs. // Optional - Other logMessages: false, @@ -847,14 +845,11 @@ class MyPublisher extends AbstractSqsPublisher { #### Consumer ```typescript -import { MessageCodecEnum } from '@message-queue-toolkit/core' - class MyConsumer extends AbstractSqsConsumer { constructor(deps: SQSConsumerDependencies) { super(deps, { - // Optional: explicitly declare that messages are compressed. - // Without this, consumers still auto-detect and decompress codec envelopes. - codec: MessageCodecEnum.ZSTD, + // No codec option needed for built-in zstd — auto-detection handles it. + // For a custom codec: codecs: [{ name: 'lz4', handler: new LZ4Handler() }] creationConfig: { queue: { QueueName: 'my-queue' } }, handlers: new MessageHandlerConfigBuilder() .addConfig(MySchema, myHandler) @@ -867,7 +862,7 @@ class MyConsumer extends AbstractSqsConsumer const messageProcessingStartTimestamp = Date.now() const parsedMessage = messageSchemaResult.result.parse(message) - message = this.updateInternalProperties(message) - - // Resolve FIFO options from original message BEFORE offloading - // (offloaded payload won't have user fields needed for messageGroupIdField) - const resolvedOptions = this.resolveFifoOptions(message, options) - - const { payload, preBuiltBody } = await this.prepareOutgoingPayload(message) - + // Dedup check before compression/offload: skip expensive work for duplicates. if ( this.isDeduplicationEnabledForMessage(parsedMessage) && (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) @@ -140,6 +133,14 @@ export abstract class AbstractSqsPublisher return } + message = this.updateInternalProperties(message) + + // Resolve FIFO options from original message BEFORE offloading + // (offloaded payload won't have user fields needed for messageGroupIdField) + const resolvedOptions = this.resolveFifoOptions(message, options) + + const { payload, preBuiltBody } = await this.prepareOutgoingPayload(message) + await this.sendMessage(payload, resolvedOptions, preBuiltBody) this.handleMessageProcessed({ message: parsedMessage, diff --git a/packages/sqs/package.json b/packages/sqs/package.json index 9a50b306..d6b8e2fa 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -38,7 +38,7 @@ "peerDependencies": { "@aws-sdk/client-sqs": "^3.1034.0", "@message-queue-toolkit/codec": ">=1.0.0", - "@message-queue-toolkit/core": ">=25.0.0", + "@message-queue-toolkit/core": ">=25.5.0", "zod": ">=3.25.76 <5.0.0" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e60ba8a8..7a80aa37 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,8 +31,8 @@ importers: specifier: ^3.0.0 version: 3.1.0 '@message-queue-toolkit/core': - specifier: '*' - version: 25.5.0(zod@4.4.3) + specifier: workspace:* + version: link:../core '@types/amqplib': specifier: 0.10.8 version: 0.10.8 From d26536511bbedb1a015fd57db5bcfca746b0a943 Mon Sep 17 00:00:00 2001 From: Irfan Hodzic Date: Fri, 22 May 2026 12:20:56 +0200 Subject: [PATCH 19/23] refactor: hoist prepareOutgoingPayload to AbstractQueueService and add codec/offload improvements - Move duplicate prepareOutgoingPayload from AbstractSqsPublisher and AbstractSnsPublisher to AbstractQueueService; publishers pre-resolve codec handler/name in constructors to avoid circular dependency between core and codec package - Add buildInlineCodecEnvelope and calculateOutgoingMessageSize to AbstractQueueService; publisher subclasses override calculateOutgoingMessageSize with their transport utility - Add in-memory fast path in compressAndOffloadPayload: small string payloads are compressed directly into a Buffer, skipping the temp-file pipeline entirely - Fix tmpPath construction to use path.join instead of string concatenation - Add JSDoc caveat to skipCompressionBelow noting it is ignored when payloadStoreConfig is set - Add test: consumer with disableCodecAutoDetection:true does not decompress real zstd envelope Co-Authored-By: Claude Sonnet 4.6 --- .../core/lib/queues/AbstractQueueService.ts | 177 ++++++++++++++---- packages/core/lib/types/queueOptionsTypes.ts | 6 + packages/sns/lib/sns/AbstractSnsPublisher.ts | 61 +----- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 61 +----- .../SqsPermissionConsumer.codec.spec.ts | 36 ++++ 5 files changed, 205 insertions(+), 136 deletions(-) diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index face2cc7..439b1119 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto' import * as fs from 'node:fs' import * as os from 'node:os' +import * as path from 'node:path' import { Readable } from 'node:stream' import { pipeline } from 'node:stream/promises' import { types } from 'node:util' @@ -144,6 +145,17 @@ export abstract class AbstractQueueService< protected readonly codec?: MessageCodecRegistration protected readonly skipCompressionBelow: number protected readonly disableCodecAutoDetection: boolean + /** + * Pre-resolved codec handler set by publisher subclasses in their constructors. + * Avoids importing `@message-queue-toolkit/codec` in the base class (which would + * create a circular dependency — codec peer-depends on core). + */ + protected resolvedCodecHandler?: MessageCodecHandler + /** + * Pre-resolved codec name matching `resolvedCodecHandler`. + * Set alongside `resolvedCodecHandler` in publisher subclass constructors. + */ + protected resolvedCodecName?: string protected isInitted: boolean @@ -753,13 +765,95 @@ export abstract class AbstractQueueService< } /** - * Streaming compress-and-offload path for large payloads (used when both `codec` and - * `payloadStoreConfig` are set). + * Compresses (when codec is configured) or offloads (when a payload store is configured) + * the outgoing message. Shared by all publisher subclasses — they pre-resolve the codec + * handler and name in their constructors and store them in `resolvedCodecHandler` / + * `resolvedCodecName` so that this base-class method never imports from + * `@message-queue-toolkit/codec` directly. + * + * Returns: + * - `{ payload, preBuiltBody }` — `preBuiltBody` is a ready-to-send codec envelope string + * when compression is applied inline; `sendMessage` must use it as-is. + * - `{ payload }` — plain JSON path (no compression or offload needed). + */ + protected async prepareOutgoingPayload(message: MessagePayloadSchemas): Promise<{ + payload: MessagePayloadSchemas | OffloadedPayloadPointerPayload + preBuiltBody?: string + }> { + const handler = this.resolvedCodecHandler + const codecName = this.resolvedCodecName + + if (handler && codecName) { + if (this.payloadStoreConfig) { + // Streaming path: avoids 3× buffer materialisation for large payloads. + // JSON → compress → temp file → threshold check → offload or inline envelope. + const result = await this.compressAndOffloadPayload(message, handler, codecName) + if (result.pointer) { + return { payload: result.pointer } + } + return { + payload: message, + preBuiltBody: this.buildInlineCodecEnvelope(result.compressedBuffer, codecName), + } + } + + // No offload store — bounded by the transport limit (SQS/SNS 256 KB), safe to buffer. + // Serialize once so we can check the raw size before deciding whether to compress. + const jsonBuffer = Buffer.from(JSON.stringify(message), 'utf8') + + // Skip compression for messages below the configured floor — small payloads + // often grow when compressed, so we send them as plain JSON instead. + if (jsonBuffer.byteLength < this.skipCompressionBelow) { + return { payload: message } + } + + const compressed = await handler.compress(jsonBuffer) + return { + payload: message, + preBuiltBody: this.buildInlineCodecEnvelope(compressed, codecName), + } + } + + return { + payload: + (await this.offloadPayload(message, () => this.calculateOutgoingMessageSize(message))) ?? + message, + } + } + + /** + * Wraps an already-compressed buffer in a self-describing codec envelope string. + * + * Replicates the logic of `buildCodecEnvelope` from `@message-queue-toolkit/codec` + * without importing it — keeping the base class free of a circular dependency on the + * codec package (which peer-depends on core). + */ + protected buildInlineCodecEnvelope(compressed: Buffer, codecName: string): string { + return `{"__mqtCodec":"${codecName}","__mqtData":"${compressed.toString('base64')}"}` + } + + /** + * Returns the wire size of the outgoing message in bytes, used by `offloadPayload` to decide + * whether the payload exceeds `messageSizeThreshold`. + * + * Overridden by publisher subclasses (SQS, SNS) to call their transport-specific utility. + * Not called on the consumer path; consumers do not override this method. + */ + protected calculateOutgoingMessageSize(_message: MessagePayloadSchemas): number { + /* c8 ignore next */ + throw new Error('calculateOutgoingMessageSize must be implemented by the publisher subclass') + } + + /** + * Compress-and-offload path used when both `codec` and `payloadStoreConfig` are set. + * + * **In-memory fast path (string payloads):** when the serializer produces a string and its + * byte length is below `messageSizeThreshold`, the payload is compressed directly into a + * Buffer in memory — no temp file is created, no disk I/O occurs. * - * Avoids the 3× memory materialisation that occurs when doing - * JSON.stringify → Buffer → compress(Buffer) before deciding whether to offload. - * Instead serializes the payload once via the configured serializer, pipes the stream - * through a zstd transform into a temp file, then decides based on the compressed size: + * **Streaming path (stream payloads or large strings):** serializes the payload once, + * pipes it through the codec Transform into a temp file, then decides based on the + * compressed size: * - Uploads the temp file as a stream when compressed size exceeds `messageSizeThreshold` * - Returns the small compressed buffer for the caller to wrap in an inline codec envelope * when compressed size fits within the threshold @@ -785,44 +879,59 @@ export abstract class AbstractQueueService< throw new Error('Payload store is not configured') } - const tmpPath = `${os.tmpdir()}/${randomUUID()}` const serialized = await this.payloadStoreConfig.serializer.serialize(message) try { + // In-memory fast path: avoid disk I/O entirely for small string payloads. + // The string byte size is checked against messageSizeThreshold — even before + // compression, if the raw bytes fit in the threshold, the compressed result + // will too (compression only shrinks). This skips the temp-file pipeline. + if ( + typeof serialized.value === 'string' && + Buffer.byteLength(serialized.value, 'utf8') < this.payloadStoreConfig.messageSizeThreshold + ) { + const compressed = await handler.compress(Buffer.from(serialized.value, 'utf8')) + return { compressedBuffer: compressed } + } + // Streaming pipeline: serializer output → codec transform → temp file. // No full-payload buffer is materialised; each codec supplies its own Transform. - await pipeline( - typeof serialized.value === 'string' ? Readable.from(serialized.value) : serialized.value, - handler.createCompressStream(), - fs.createWriteStream(tmpPath), - ) - - const compressedSize = (await fs.promises.stat(tmpPath)).size + const tmpPath = path.join(os.tmpdir(), randomUUID()) + try { + await pipeline( + typeof serialized.value === 'string' ? Readable.from(serialized.value) : serialized.value, + handler.createCompressStream(), + fs.createWriteStream(tmpPath), + ) - // Compare the envelope wire size (not raw compressed bytes) against the threshold. - // buildCodecEnvelope produces {"__mqtCodec":"","__mqtData":""}. - // Base64 expands by ⌈N/3⌉×4; the fixed JSON framing adds 32 chars + codec name length. - const envelopeSize = Math.ceil(compressedSize / 3) * 4 + 32 + codecName.length + const compressedSize = (await fs.promises.stat(tmpPath)).size + + // Compare the envelope wire size (not raw compressed bytes) against the threshold. + // buildCodecEnvelope produces {"__mqtCodec":"","__mqtData":""}. + // Base64 expands by ⌈N/3⌉×4; the fixed JSON framing adds 32 chars + codec name length. + const envelopeSize = Math.ceil(compressedSize / 3) * 4 + 32 + codecName.length + + if (envelopeSize > this.payloadStoreConfig.messageSizeThreshold) { + const { store, storeName } = this.resolveOutgoingStore() + const payloadId = await store.storePayload({ + value: fs.createReadStream(tmpPath), + size: compressedSize, + }) + return { + pointer: this.buildPointer(message, payloadId, storeName, compressedSize, codecName), + } + } - if (envelopeSize > this.payloadStoreConfig.messageSizeThreshold) { - const { store, storeName } = this.resolveOutgoingStore() - const payloadId = await store.storePayload({ - value: fs.createReadStream(tmpPath), - size: compressedSize, - }) - return { - pointer: this.buildPointer(message, payloadId, storeName, compressedSize, codecName), + // Compressed payload fits inline — return the buffer; caller wraps it in a codec envelope. + return { compressedBuffer: await fs.promises.readFile(tmpPath) } + } finally { + try { + await fs.promises.unlink(tmpPath) + } catch { + // ignore cleanup errors } } - - // Compressed payload fits inline — return the buffer; caller wraps it in a codec envelope. - return { compressedBuffer: await fs.promises.readFile(tmpPath) } } finally { - try { - await fs.promises.unlink(tmpPath) - } catch { - // ignore cleanup errors - } if (isDestroyable(serialized)) { await serialized.destroy() } diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index b299caac..b614966e 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -172,6 +172,12 @@ export type CommonQueueOptions = { * * Set to `0` to compress every message regardless of size. * + * **Ignored when `payloadStoreConfig` is also set.** When both options are configured, + * the compress-and-offload pipeline always runs regardless of message size — the + * threshold is instead `messageSizeThreshold` (whether to upload to the store or + * inline the envelope). Use `skipCompressionBelow: 0` in that combination to make the + * intent explicit. + * * @example * // Compress only messages ≥ 1 KB * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD, skipCompressionBelow: 1024 }) diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 591706bf..370f301c 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -2,7 +2,7 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sns' import { PublishCommand } from '@aws-sdk/client-sns' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { buildCodecEnvelope, getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' +import { getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -90,6 +90,13 @@ export abstract class AbstractSnsPublisher this.isFifoTopic = options.fifoTopic ?? false this.messageGroupIdField = options.messageGroupIdField this.defaultMessageGroupId = options.defaultMessageGroupId + + // Pre-resolve codec handler and name so the base-class prepareOutgoingPayload + // does not need to import @message-queue-toolkit/codec (circular dep risk). + if (options.codec) { + this.resolvedCodecHandler = resolveCodecHandler(options.codec) + this.resolvedCodecName = getCodecName(options.codec) + } } override async init(): Promise { @@ -205,56 +212,8 @@ export abstract class AbstractSnsPublisher return this.isDeduplicationEnabled && super.isDeduplicationEnabledForMessage(message) } - /** - * Compresses (when codec is set) or offloads (when store is configured) the message. - * Returns the payload to send and an optional pre-built body string. - * When preBuiltBody is set, it is a ready-to-send codec envelope — sendMessage must use it as-is. - * - * When both codec and payloadStoreConfig are set, uses a streaming pipeline - * (JSON → zstd → temp file → store) to avoid materialising the full payload in memory. - */ - private async prepareOutgoingPayload(message: MessagePayloadType): Promise<{ - payload: MessagePayloadType | OffloadedPayloadPointerPayload - preBuiltBody?: string - }> { - const codec = this.codec - - if (codec) { - const handler = resolveCodecHandler(codec) - const codecName = getCodecName(codec) - - if (this.payloadStoreConfig) { - // Streaming path: avoids 3× buffer materialisation for large payloads. - // JSON → compress → temp file → threshold check → offload or inline envelope. - const result = await this.compressAndOffloadPayload(message, handler, codecName) - if (result.pointer) { - return { payload: result.pointer } - } - return { - payload: message, - preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codecName), - } - } - - // No offload store — bounded by SNS 256 KB limit, safe to buffer. - // Serialize once so we can check the raw size before deciding whether to compress. - const jsonBuffer = Buffer.from(JSON.stringify(message), 'utf8') - - // Skip compression for messages below the configured floor — small payloads - // often grow when compressed, so we send them as plain JSON instead. - if (jsonBuffer.byteLength < this.skipCompressionBelow) { - return { payload: message } - } - - const compressed = await handler.compress(jsonBuffer) - return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codecName) } - } - - return { - payload: - (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? - message, - } + protected override calculateOutgoingMessageSize(message: MessagePayloadType): number { + return calculateOutgoingMessageSize(message) } protected async sendMessage( diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index 42acd33a..399385ad 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -2,7 +2,7 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sqs' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { buildCodecEnvelope, getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' +import { getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -96,6 +96,13 @@ export abstract class AbstractSqsPublisher this.isDeduplicationEnabled = !!options.enablePublisherDeduplication this.messageGroupIdField = options.messageGroupIdField this.defaultMessageGroupId = options.defaultMessageGroupId + + // Pre-resolve codec handler and name so the base-class prepareOutgoingPayload + // does not need to import @message-queue-toolkit/codec (circular dep risk). + if (options.codec) { + this.resolvedCodecHandler = resolveCodecHandler(options.codec) + this.resolvedCodecName = getCodecName(options.codec) + } } async publish(message: MessagePayloadType, options: SQSMessageOptions = {}): Promise { @@ -199,56 +206,8 @@ export abstract class AbstractSqsPublisher return this.messageSchemaContainer.resolveSchema(message) } - /** - * Compresses (when codec is set) or offloads (when store is configured) the message. - * Returns the payload to send and an optional pre-built body string. - * When preBuiltBody is set, it is a ready-to-send codec envelope — sendMessage must use it as-is. - * - * When both codec and payloadStoreConfig are set, uses a streaming pipeline - * (JSON → zstd → temp file → store) to avoid materialising the full payload in memory. - */ - private async prepareOutgoingPayload(message: MessagePayloadType): Promise<{ - payload: MessagePayloadType | OffloadedPayloadPointerPayload - preBuiltBody?: string - }> { - const codec = this.codec - - if (codec) { - const handler = resolveCodecHandler(codec) - const codecName = getCodecName(codec) - - if (this.payloadStoreConfig) { - // Streaming path: avoids 3× buffer materialisation for large payloads. - // JSON → compress → temp file → threshold check → offload or inline envelope. - const result = await this.compressAndOffloadPayload(message, handler, codecName) - if (result.pointer) { - return { payload: result.pointer } - } - return { - payload: message, - preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codecName), - } - } - - // No offload store — bounded by SQS 256 KB limit, safe to buffer. - // Serialize once so we can check the raw size before deciding whether to compress. - const jsonBuffer = Buffer.from(JSON.stringify(message), 'utf8') - - // Skip compression for messages below the configured floor — small payloads - // often grow when compressed, so we send them as plain JSON instead. - if (jsonBuffer.byteLength < this.skipCompressionBelow) { - return { payload: message } - } - - const compressed = await handler.compress(jsonBuffer) - return { payload: message, preBuiltBody: buildCodecEnvelope(compressed, codecName) } - } - - return { - payload: - (await this.offloadPayload(message, () => calculateOutgoingMessageSize(message))) ?? - message, - } + protected override calculateOutgoingMessageSize(message: MessagePayloadType): number { + return calculateOutgoingMessageSize(message) } protected async sendMessage( diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index c026c374..d7ce7004 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -1,6 +1,7 @@ import type { Transform } from 'node:stream' import { PassThrough } from 'node:stream' import { ReceiveMessageCommand, SendMessageCommand } from '@aws-sdk/client-sqs' +import { waitAndRetry } from '@lokalise/node-core' import { compressMessageBody } from '@message-queue-toolkit/codec' import type { MessageCodecHandler } from '@message-queue-toolkit/core' import { MessageCodecEnum } from '@message-queue-toolkit/core' @@ -469,6 +470,41 @@ describe('SqsPermissionConsumer - skipCompressionBelow', () => { await wirePublisher.close() await noAutoConsumer.close(true) }) + + it('consumer with disableCodecAutoDetection:true does not decompress a real zstd envelope', async () => { + // When disableCodecAutoDetection is true, a real codec envelope reaching the consumer + // must NOT be decompressed — the raw envelope object is passed to schema validation, + // which fails because { __mqtCodec, __mqtData } has no `messageType` field. + const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-disable-real-envelope` + await testAdmin.deleteQueues(queueName) + + const noAutoConsumer = new SqsPermissionConsumer(diContainer.cradle, { + creationConfig: { queue: { QueueName: queueName } }, + deletionConfig: { deleteIfExists: false }, + disableCodecAutoDetection: true, + }) + await noAutoConsumer.start() + + // Build a real zstd envelope and inject it directly into the queue + const message: PERMISSIONS_ADD_MESSAGE_TYPE = { + id: 'disable-real-envelope-1', + messageType: 'add', + } + const compressedBody = await compressMessageBody(JSON.stringify(message), MessageCodecEnum.ZSTD) + await diContainer.cradle.sqsClient.send( + new SendMessageCommand({ + QueueUrl: noAutoConsumer.queueProps.url, + MessageBody: compressedBody, + }), + ) + + // The envelope fails schema validation — consumer records it as an error, not consumed. + await waitAndRetry(() => noAutoConsumer.handlerSpy.counts.error > 0, 100, 20) + expect(noAutoConsumer.handlerSpy.counts.error).toBeGreaterThan(0) + expect(noAutoConsumer.addCounter).toBe(0) + + await noAutoConsumer.close(true) + }, 15000) }) // --------------------------------------------------------------------------- From 18ae98270518af537cee7ffd7e4d344721e21d3c Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 22 May 2026 14:36:36 +0300 Subject: [PATCH 20/23] =?UTF-8?q?fix(codec):=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20merge=20codec=20into=20core,=20fix=20offload=20thre?= =?UTF-8?q?shold=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses code-review findings on the zstd message compression feature. P0 — In-memory fast path in compressAndOffloadPayload could emit an inline codec envelope larger than messageSizeThreshold (and the 256 KB protocol limit): it gated on raw size and assumed "compression only shrinks", which is false for incompressible data. It now compares the base64 envelope wire size and offloads when it exceeds the threshold, matching the streaming path. P0 — The codec implementation was effectively a mandatory peer dependency of sqs/sns (static import) despite being documented as opt-in, breaking every existing consumer on upgrade. The codec uses only the Node.js built-in zlib (no native deps), so the standalone @message-queue-toolkit/codec package is merged into @message-queue-toolkit/core and removed. Nothing extra to install. P1 — buildCodecEnvelope is no longer duplicated; AbstractQueueService uses the shared core implementation. P1 — codec/skipCompressionBelow moved off the shared CommonQueueOptions onto QueuePublisherOptions; codecs/disableCodecAutoDetection onto the SQS consumer options. AMQP/Pub-Sub consumers no longer silently accept a codec option. P1 — Documented the publisher dedup-before-offload ordering trade-off. P2 — skipCompressionBelow is now honored even when payloadStoreConfig is set: small messages are offloaded/sent as plain JSON instead of always compressed. P2 — Removed the misleading "compression only shrinks" comment; corrected the codecMicro benchmark header (it is a manual benchmark, not a CI gate). P2 — prepareOutgoingPayload serializes the message once instead of twice. P3 — An unregistered codec on an offloaded pointer is now a retriable error (consumer misconfiguration) instead of being routed to the DLQ as poison. P3 — Documented the os.tmpdir() requirement of the streaming offload path and widened the codec test padding margin. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 1 - packages/codec/README.md | 112 ------------ packages/codec/lib/index.ts | 8 - packages/codec/package.json | 58 ------ packages/codec/tsconfig.build.json | 4 - packages/codec/tsconfig.json | 4 - packages/core/README.md | 8 +- .../{codec => core}/lib/codec/codecHandler.ts | 11 +- packages/core/lib/codec/messageCodec.ts | 6 +- packages/core/lib/index.ts | 8 + .../core/lib/queues/AbstractQueueService.ts | 173 ++++++++++-------- packages/core/lib/types/queueOptionsTypes.ts | 102 +++++------ packages/sns/README.md | 1 - packages/sns/lib/sns/AbstractSnsPublisher.ts | 15 +- .../sns/lib/sns/AbstractSnsSqsConsumer.ts | 6 +- packages/sns/package.json | 2 - .../SnsSqsPermissionConsumer.codec.spec.ts | 3 + .../test/publishers/SnsPermissionPublisher.ts | 2 + packages/sqs/README.md | 7 +- packages/sqs/bench/codecMicro.bench.ts | 19 +- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 50 +++-- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 15 +- packages/sqs/package.json | 2 - packages/sqs/test/codec/codecHandler.spec.ts | 3 +- .../SqsPermissionConsumer.codec.spec.ts | 11 +- ...rmissionConsumer.payloadOffloading.spec.ts | 3 + pnpm-lock.yaml | 43 +---- 27 files changed, 244 insertions(+), 433 deletions(-) delete mode 100644 packages/codec/README.md delete mode 100644 packages/codec/lib/index.ts delete mode 100644 packages/codec/package.json delete mode 100644 packages/codec/tsconfig.build.json delete mode 100644 packages/codec/tsconfig.json rename packages/{codec => core}/lib/codec/codecHandler.ts (90%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c6cc4bf..1796fc95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,6 @@ jobs: run: | declare -A PATH_TO_NAME=( ["packages/amqp"]="@message-queue-toolkit/amqp" - ["packages/codec"]="@message-queue-toolkit/codec" ["packages/core"]="@message-queue-toolkit/core" ["packages/gcp-pubsub"]="@message-queue-toolkit/gcp-pubsub" ["packages/gcs-payload-store"]="@message-queue-toolkit/gcs-payload-store" diff --git a/packages/codec/README.md b/packages/codec/README.md deleted file mode 100644 index c7b25a6b..00000000 --- a/packages/codec/README.md +++ /dev/null @@ -1,112 +0,0 @@ -# @message-queue-toolkit/codec - -Message compression codec implementations for [message-queue-toolkit](https://github.com/kibertoad/message-queue-toolkit). - -This package provides the concrete codec implementations (e.g. zstd) used by the SQS and SNS adapters. The codec interfaces and types (`MessageCodecEnum`, `MessageCodecHandler`, `CodecEnvelope`) live in `@message-queue-toolkit/core`. - -## Installation - -```sh -npm install @message-queue-toolkit/codec @message-queue-toolkit/core -``` - -> **Requirements:** Node.js >=22.15.0 (uses the built-in `zlib` zstd support). - -## Usage - -Codec options are typically set on the publisher/consumer constructor in the SQS or SNS adapter packages. You do not need to interact with this package directly unless you are building a custom adapter. - -### How compression works during publish - -When `codec` is set on a publisher, compression happens **exactly once** at the start of `publish()`, before any other processing: - -1. The message JSON is compressed to a raw `Buffer`. -2. If a payload store is configured **and** the compressed size exceeds `messageSizeThreshold`, the compressed bytes are stored in S3 and only a lightweight pointer is sent. The codec name is recorded in `payloadRef.codec` so the consumer can decompress after retrieval. -3. If the compressed size fits within the threshold (or no store is configured), the message is sent inline as a self-describing codec envelope. - -The payload is never compressed twice. The same compressed `Buffer` from step 1 is either uploaded to S3 or wrapped in the envelope — whichever path is taken. - -### Compress / decompress a message body - -```typescript -import { compressMessageBody, decompressMessageBody } from '@message-queue-toolkit/codec' -import { MessageCodecEnum } from '@message-queue-toolkit/core' - -// Compress (returns a JSON string containing the codec envelope) -const compressed = await compressMessageBody(JSON.stringify(payload), MessageCodecEnum.ZSTD) - -// Decompress (parses the envelope and returns the original object) -const original = await decompressMessageBody(JSON.parse(compressed)) -``` - -### Build a codec envelope from already-compressed bytes - -When you have pre-compressed bytes (e.g., from `resolveCodecHandler(codec).compress(...)`) and want to produce the envelope string without compressing again: - -```typescript -import { buildCodecEnvelope, resolveCodecHandler } from '@message-queue-toolkit/codec' -import { MessageCodecEnum } from '@message-queue-toolkit/core' - -const handler = resolveCodecHandler(MessageCodecEnum.ZSTD) -const compressed: Buffer = await handler.compress(Buffer.from(JSON.stringify(payload), 'utf8')) - -// Build envelope without a second compression pass -const envelopeString = buildCodecEnvelope(compressed, MessageCodecEnum.ZSTD) -// → '{"__mqtCodec":"zstd","__mqtData":""}' -``` - -### Custom codec handler - -Implement `MessageCodecHandler` and register it via the `{ name, handler }` form of the `codec` option. The same registration must be provided on both the publisher and the consumer. - -```typescript -import type { Transform } from 'node:stream' -import type { MessageCodecHandler } from '@message-queue-toolkit/core' - -class MyLz4Handler implements MessageCodecHandler { - async compress(data: Buffer): Promise { - return lz4.encode(data) // your compression library - } - - async decompress(data: Buffer): Promise { - return lz4.decode(data) - } - - // Required for the streaming offload path (codec + payloadStoreConfig). - // Return a Transform stream that compresses its input chunk-by-chunk. - createCompressStream(): Transform { - return lz4.createEncoderStream() - } -} -``` - -Register the handler on the publisher and consumer using the `{ name, handler }` object form: - -```typescript -const codec = { name: 'lz4', handler: new MyLz4Handler() } - -// Publisher — wraps each outgoing message in { __mqtCodec: 'lz4', __mqtData: '' } -new MyPublisher(deps, { codec }) - -// Consumer — register the custom codec so envelopes with __mqtCodec 'lz4' are decompressed -new MyConsumer(deps, { codecs: [codec] }) -``` - -**Consumer-side scoping.** A consumer registered with `{ name: 'lz4', handler }` via `codecs` will only decompress envelopes that carry `__mqtCodec: 'lz4'`. Built-in codecs (e.g. zstd) are always auto-registered — no `codecs` entry needed for them. This prevents accidental cross-codec decompression. - -## Codec envelope format - -Compressed messages are wrapped in a self-describing JSON envelope: - -```json -{ - "__mqtCodec": "zstd", - "__mqtData": "" -} -``` - -Consumers auto-detect this envelope and decompress transparently, even if the `codec` option is not set on the consumer. - -## License - -MIT diff --git a/packages/codec/lib/index.ts b/packages/codec/lib/index.ts deleted file mode 100644 index f723f350..00000000 --- a/packages/codec/lib/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export { - buildCodecEnvelope, - compressMessageBody, - decompressMessageBody, - getCodecName, - resolveCodecHandler, - ZstdCodecHandler, -} from './codec/codecHandler.ts' diff --git a/packages/codec/package.json b/packages/codec/package.json deleted file mode 100644 index 45d4ccad..00000000 --- a/packages/codec/package.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "name": "@message-queue-toolkit/codec", - "version": "1.0.0", - "private": false, - "license": "MIT", - "description": "Message compression codec implementations for message-queue-toolkit", - "maintainers": [ - { - "name": "Igor Savin", - "email": "kibertoad@gmail.com" - } - ], - "type": "module", - "main": "./dist/index.js", - "exports": { - ".": "./dist/index.js", - "./package.json": "./package.json" - }, - "scripts": { - "build": "pnpm run clean && tsc --project tsconfig.build.json", - "clean": "rimraf dist", - "lint": "biome check . && tsc", - "lint:fix": "biome check --write .", - "prepublishOnly": "pnpm run lint && pnpm run build" - }, - "engines": { - "node": ">=22.15.0" - }, - "peerDependencies": { - "@message-queue-toolkit/core": ">=25.5.0" - }, - "devDependencies": { - "@biomejs/biome": "^2.3.8", - "@lokalise/biome-config": "^3.1.0", - "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/core": "workspace:*", - "@types/node": "^22.0.0", - "rimraf": "^6.0.1", - "typescript": "^5.9.3" - }, - "homepage": "https://github.com/kibertoad/message-queue-toolkit", - "repository": { - "type": "git", - "url": "git://github.com/kibertoad/message-queue-toolkit.git" - }, - "keywords": [ - "message", - "queue", - "codec", - "compression", - "zstd" - ], - "files": [ - "README.md", - "LICENSE", - "dist/*" - ] -} diff --git a/packages/codec/tsconfig.build.json b/packages/codec/tsconfig.build.json deleted file mode 100644 index 198dcfd5..00000000 --- a/packages/codec/tsconfig.build.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": ["./tsconfig.json", "@lokalise/tsconfig/build-public-lib"], - "include": ["lib/**/*"] -} diff --git a/packages/codec/tsconfig.json b/packages/codec/tsconfig.json deleted file mode 100644 index 8dca583f..00000000 --- a/packages/codec/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "@lokalise/tsconfig/tsc", - "include": ["lib/**/*"] -} diff --git a/packages/core/README.md b/packages/core/README.md index dcac7667..a6915a2f 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -643,7 +643,7 @@ class MyPayloadStore implements PayloadStore { #### Message compression (codec) -Publishers can compress outgoing messages by setting `codec` in their options. Requires **Node.js >=22.15.0** and the [`@message-queue-toolkit/codec`](../codec/README.md) package. +Publishers can compress outgoing messages by setting `codec` in their options. The codec implementation (zstd via the Node.js built-in `zlib` module) ships inside `@message-queue-toolkit/core` — there is no extra package to install. Requires **Node.js >=22.15.0**. Only the SQS and SNS adapters support compression. **Built-in zstd:** @@ -674,8 +674,6 @@ new MyPublisher(deps, { codec }) new MyConsumer(deps, { codecs: [codec] }) // register custom codec on the consumer ``` -See the [`@message-queue-toolkit/codec` README](../codec/README.md) for a full custom codec example. - Compressed messages are wrapped in a self-describing envelope `{ __mqtCodec: '', __mqtData: '' }`. Built-in codecs (e.g. zstd) are auto-detected on every consumer — no consumer option needed. For custom codecs, pass `codecs: [{ name, handler }]` to register them on the consumer. #### Interaction with codec (compression) @@ -687,6 +685,10 @@ When both `codec` and `payloadStoreConfig` are set on a publisher, compression a 3. If the envelope size exceeds the threshold, the raw compressed bytes are stored in the payload store. The codec name is written to `payloadRef.codec` so the consumer knows how to decompress after retrieval. 4. If the envelope size fits within the threshold, the message is sent inline as a self-describing codec envelope — S3 is never touched. +`skipCompressionBelow` is honored here too: a message whose serialized JSON is below the threshold skips compression entirely and is offloaded (or sent inline) as plain JSON. + +> **Note:** for large payloads the compress-and-offload path streams the message through a temporary file under `os.tmpdir()` to avoid buffering the whole payload in memory. The temp file is always removed in a `finally` block. Environments with a read-only or unavailable temp directory (rare; AWS Lambda's `/tmp` is writable) cannot use the codec + payload-store combination. + This means compression can prevent offloading entirely for messages that are large before compression but small after. ## API Reference diff --git a/packages/codec/lib/codec/codecHandler.ts b/packages/core/lib/codec/codecHandler.ts similarity index 90% rename from packages/codec/lib/codec/codecHandler.ts rename to packages/core/lib/codec/codecHandler.ts index 65799cf2..9de442f1 100644 --- a/packages/codec/lib/codec/codecHandler.ts +++ b/packages/core/lib/codec/codecHandler.ts @@ -5,14 +5,16 @@ import type { CodecEnvelope, MessageCodecHandler, MessageCodecRegistration, -} from '@message-queue-toolkit/core' -import { BASE64_RE, MessageCodecEnum } from '@message-queue-toolkit/core' +} from './messageCodec.ts' +import { BASE64_RE, MessageCodecEnum } from './messageCodec.ts' const ZSTD_UNSUPPORTED_MSG = 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + - '@message-queue-toolkit/codec requires Node.js >=22.15.0 or >=23.8.0.' + 'Message compression requires Node.js >=22.15.0 or >=23.8.0.' // Resolved lazily — undefined on Node versions that lack zstd support. +// Keeping these lazy means importing core never throws on older Node; only an +// actual compress/decompress call does, and only when zstd is genuinely used. const zstdCompress = typeof zlib.zstdCompress === 'function' ? promisify(zlib.zstdCompress) : undefined const zstdDecompress = @@ -88,6 +90,9 @@ export async function compressMessageBody( * Uses string concatenation instead of JSON.stringify to avoid allocating an * intermediate object — the base64 string and the envelope string are the only * two allocations on the inline path. + * + * `codecName` must already be a JSON-safe identifier (see {@link getCodecName}, + * which is enforced for every registration before it reaches this function). */ export function buildCodecEnvelope(compressed: Buffer, codecName: string): string { return '{"__mqtCodec":"' + codecName + '","__mqtData":"' + compressed.toString('base64') + '"}' diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 016d5da4..d98643ca 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -29,8 +29,8 @@ export type CodecEnvelope = { * Low-level interface for a compression codec. * * Implement this interface to plug in a custom compression algorithm. - * The built-in implementation (`ZstdCodecHandler` in `@message-queue-toolkit/codec`) - * uses Node.js built-in `zlib` zstd support. + * The built-in implementation (`ZstdCodecHandler`, exported from + * `@message-queue-toolkit/core`) uses Node.js built-in `zlib` zstd support. * * All three methods are required: * - `compress` / `decompress` are used for the inline (non-offloaded) publish path. @@ -62,7 +62,7 @@ export interface MessageCodecHandler { * import { LZ4Handler } from './lz4Handler.ts' * const codec = { name: 'lz4', handler: new LZ4Handler() } * new MyPublisher(deps, { codec }) - * new MyConsumer(deps, { codec }) // same registration on the consumer side + * new MyConsumer(deps, { codecs: [codec] }) // register the same codec on the consumer */ export type MessageCodecRegistration = MessageCodec | { name: string; handler: MessageCodecHandler } diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index 6426848b..48451c50 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,3 +1,11 @@ +export { + buildCodecEnvelope, + compressMessageBody, + decompressMessageBody, + getCodecName, + resolveCodecHandler, + ZstdCodecHandler, +} from './codec/codecHandler.ts' export { BASE64_RE, type CodecEnvelope, diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 439b1119..88f3ab95 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -20,6 +20,7 @@ import { } from '@message-queue-toolkit/schemas' import { getProperty, setProperty } from 'dot-prop' import type { ZodSchema, ZodType } from 'zod/v4' +import { buildCodecEnvelope, getCodecName, resolveCodecHandler } from '../codec/codecHandler.ts' import type { MessageCodecHandler, MessageCodecRegistration } from '../codec/messageCodec.ts' import type { MessageInvalidFormatError, MessageValidationError } from '../errors/Errors.ts' import { @@ -142,20 +143,15 @@ export abstract class AbstractQueueService< protected readonly messageDeduplicationConfig?: MessageDeduplicationConfig protected readonly messageMetricsManager?: MessageMetricsManager protected readonly _handlerSpy?: HandlerSpy - protected readonly codec?: MessageCodecRegistration protected readonly skipCompressionBelow: number protected readonly disableCodecAutoDetection: boolean /** - * Pre-resolved codec handler set by publisher subclasses in their constructors. - * Avoids importing `@message-queue-toolkit/codec` in the base class (which would - * create a circular dependency — codec peer-depends on core). + * Codec handler resolved from the `codec` option, used by `prepareOutgoingPayload`. + * Undefined when no codec is configured (the common case). */ - protected resolvedCodecHandler?: MessageCodecHandler - /** - * Pre-resolved codec name matching `resolvedCodecHandler`. - * Set alongside `resolvedCodecHandler` in publisher subclass constructors. - */ - protected resolvedCodecName?: string + protected readonly resolvedCodecHandler?: MessageCodecHandler + /** Codec name matching `resolvedCodecHandler`, written into every codec envelope. */ + protected readonly resolvedCodecName?: string protected isInitted: boolean @@ -193,9 +189,22 @@ export abstract class AbstractQueueService< } : undefined this.messageDeduplicationConfig = options.messageDeduplicationConfig - this.codec = options.codec - this.skipCompressionBelow = options.skipCompressionBelow ?? 512 - this.disableCodecAutoDetection = options.disableCodecAutoDetection ?? false + + // Codec options live on role-specific option types (`codec`/`skipCompressionBelow` + // on publishers, `disableCodecAutoDetection` on consumers), so they are not part of + // the shared `QueueOptions` constraint. The base class handles both roles, hence the + // localized widening cast. + const codecOptions = options as Partial<{ + codec: MessageCodecRegistration + skipCompressionBelow: number + disableCodecAutoDetection: boolean + }> + this.skipCompressionBelow = codecOptions.skipCompressionBelow ?? 512 + this.disableCodecAutoDetection = codecOptions.disableCodecAutoDetection ?? false + if (codecOptions.codec) { + this.resolvedCodecHandler = resolveCodecHandler(codecOptions.codec) + this.resolvedCodecName = getCodecName(codecOptions.codec) + } this.logMessages = options.logMessages ?? false this._handlerSpy = resolveHandlerSpy(options) @@ -766,15 +775,15 @@ export abstract class AbstractQueueService< /** * Compresses (when codec is configured) or offloads (when a payload store is configured) - * the outgoing message. Shared by all publisher subclasses — they pre-resolve the codec - * handler and name in their constructors and store them in `resolvedCodecHandler` / - * `resolvedCodecName` so that this base-class method never imports from - * `@message-queue-toolkit/codec` directly. + * the outgoing message. Shared by all publisher subclasses via the `resolvedCodecHandler` + * / `resolvedCodecName` fields resolved from the `codec` option in the base constructor. * * Returns: - * - `{ payload, preBuiltBody }` — `preBuiltBody` is a ready-to-send codec envelope string - * when compression is applied inline; `sendMessage` must use it as-is. - * - `{ payload }` — plain JSON path (no compression or offload needed). + * - `{ payload, preBuiltBody }` — `preBuiltBody` is a ready-to-send wire body string + * (codec envelope, or plain JSON when compression was skipped); `sendMessage` must + * use it as-is. + * - `{ payload }` — the payload is sent through the normal `JSON.stringify` path + * (no codec configured), optionally replaced by an offloaded-payload pointer. */ protected async prepareOutgoingPayload(message: MessagePayloadSchemas): Promise<{ payload: MessagePayloadSchemas | OffloadedPayloadPointerPayload @@ -784,33 +793,42 @@ export abstract class AbstractQueueService< const codecName = this.resolvedCodecName if (handler && codecName) { + // Serialize once up-front. The result is reused to apply `skipCompressionBelow` + // and, on the inline path, as the codec input / plain-JSON wire body — so the + // message is never stringified twice. + const json = JSON.stringify(message) + + // Skip compression for messages below the configured floor — small payloads + // often grow rather than shrink when compressed. Honored whether or not a + // payload store is configured. + if (Buffer.byteLength(json, 'utf8') < this.skipCompressionBelow) { + if (this.payloadStoreConfig) { + const pointer = await this.offloadPayload(message, () => + this.calculateOutgoingMessageSize(message), + ) + return { payload: pointer ?? message } + } + // Reuse the already-serialized JSON as the wire body — no second stringify. + return { payload: message, preBuiltBody: json } + } + if (this.payloadStoreConfig) { - // Streaming path: avoids 3× buffer materialisation for large payloads. - // JSON → compress → temp file → threshold check → offload or inline envelope. + // Compress once, then offload or inline based on the codec envelope wire size. const result = await this.compressAndOffloadPayload(message, handler, codecName) if (result.pointer) { return { payload: result.pointer } } return { payload: message, - preBuiltBody: this.buildInlineCodecEnvelope(result.compressedBuffer, codecName), + preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codecName), } } // No offload store — bounded by the transport limit (SQS/SNS 256 KB), safe to buffer. - // Serialize once so we can check the raw size before deciding whether to compress. - const jsonBuffer = Buffer.from(JSON.stringify(message), 'utf8') - - // Skip compression for messages below the configured floor — small payloads - // often grow when compressed, so we send them as plain JSON instead. - if (jsonBuffer.byteLength < this.skipCompressionBelow) { - return { payload: message } - } - - const compressed = await handler.compress(jsonBuffer) + const compressed = await handler.compress(Buffer.from(json, 'utf8')) return { payload: message, - preBuiltBody: this.buildInlineCodecEnvelope(compressed, codecName), + preBuiltBody: buildCodecEnvelope(compressed, codecName), } } @@ -822,14 +840,13 @@ export abstract class AbstractQueueService< } /** - * Wraps an already-compressed buffer in a self-describing codec envelope string. - * - * Replicates the logic of `buildCodecEnvelope` from `@message-queue-toolkit/codec` - * without importing it — keeping the base class free of a circular dependency on the - * codec package (which peer-depends on core). + * Estimates the wire size in bytes of the codec envelope wrapping `compressedSize` + * compressed bytes. The envelope is `{"__mqtCodec":"","__mqtData":""}`: + * base64 expands the payload to `⌈N/3⌉×4`, and the fixed JSON framing adds 32 chars + * plus the codec name length. */ - protected buildInlineCodecEnvelope(compressed: Buffer, codecName: string): string { - return `{"__mqtCodec":"${codecName}","__mqtData":"${compressed.toString('base64')}"}` + protected estimateCodecEnvelopeSize(compressedSize: number, codecName: string): number { + return Math.ceil(compressedSize / 3) * 4 + 32 + codecName.length } /** @@ -846,22 +863,20 @@ export abstract class AbstractQueueService< /** * Compress-and-offload path used when both `codec` and `payloadStoreConfig` are set. + * The caller (`prepareOutgoingPayload`) has already applied `skipCompressionBelow`, so + * this method always compresses. * * **In-memory fast path (string payloads):** when the serializer produces a string and its * byte length is below `messageSizeThreshold`, the payload is compressed directly into a * Buffer in memory — no temp file is created, no disk I/O occurs. * * **Streaming path (stream payloads or large strings):** serializes the payload once, - * pipes it through the codec Transform into a temp file, then decides based on the - * compressed size: - * - Uploads the temp file as a stream when compressed size exceeds `messageSizeThreshold` - * - Returns the small compressed buffer for the caller to wrap in an inline codec envelope - * when compressed size fits within the threshold + * pipes it through the codec Transform into a temp file. * - * `skipCompressionBelow` is intentionally NOT checked here: when both `codec` and - * `payloadStoreConfig` are configured the user explicitly opted into compression, and even - * small messages may need to be offloaded (e.g. a very low `messageSizeThreshold`). - * The caller is responsible for applying `skipCompressionBelow` on the inline-only path. + * Either way, the offload decision is made against the **codec envelope wire size** + * (base64-encoded compressed bytes + JSON framing), not the raw compressed byte count: + * compression does not always shrink data, so the base64 envelope can exceed + * `messageSizeThreshold` even when the raw payload did not. * * @returns * - `{ pointer }` — compressed payload was offloaded; use pointer as the message payload @@ -878,19 +893,30 @@ export abstract class AbstractQueueService< if (!this.payloadStoreConfig) { throw new Error('Payload store is not configured') } + const threshold = this.payloadStoreConfig.messageSizeThreshold const serialized = await this.payloadStoreConfig.serializer.serialize(message) try { // In-memory fast path: avoid disk I/O entirely for small string payloads. - // The string byte size is checked against messageSizeThreshold — even before - // compression, if the raw bytes fit in the threshold, the compressed result - // will too (compression only shrinks). This skips the temp-file pipeline. if ( typeof serialized.value === 'string' && - Buffer.byteLength(serialized.value, 'utf8') < this.payloadStoreConfig.messageSizeThreshold + Buffer.byteLength(serialized.value, 'utf8') < threshold ) { const compressed = await handler.compress(Buffer.from(serialized.value, 'utf8')) + // The wire body is a base64 codec envelope, which can exceed the threshold even + // when the raw payload did not (compression does not always shrink). Re-check the + // envelope size and offload the compressed bytes if it no longer fits inline. + if (this.estimateCodecEnvelopeSize(compressed.length, codecName) > threshold) { + const { store, storeName } = this.resolveOutgoingStore() + const payloadId = await store.storePayload({ + value: Readable.from(compressed), + size: compressed.length, + }) + return { + pointer: this.buildPointer(message, payloadId, storeName, compressed.length, codecName), + } + } return { compressedBuffer: compressed } } @@ -907,11 +933,7 @@ export abstract class AbstractQueueService< const compressedSize = (await fs.promises.stat(tmpPath)).size // Compare the envelope wire size (not raw compressed bytes) against the threshold. - // buildCodecEnvelope produces {"__mqtCodec":"","__mqtData":""}. - // Base64 expands by ⌈N/3⌉×4; the fixed JSON framing adds 32 chars + codec name length. - const envelopeSize = Math.ceil(compressedSize / 3) * 4 + 32 + codecName.length - - if (envelopeSize > this.payloadStoreConfig.messageSizeThreshold) { + if (this.estimateCodecEnvelopeSize(compressedSize, codecName) > threshold) { const { store, storeName } = this.resolveOutgoingStore() const payloadId = await store.storePayload({ value: fs.createReadStream(tmpPath), @@ -944,12 +966,16 @@ export abstract class AbstractQueueService< * * Supports both new multi-store format (payloadRef) and legacy format (offloadedPayloadPointer). * - * When `decompress` is provided and the pointer's `payloadRef.codec` matches, the fetched bytes - * are treated as raw compressed binary and decompressed before JSON parsing. + * When `resolveDecompressor` is provided and the pointer's `payloadRef.codec` is set, the + * fetched bytes are treated as raw compressed binary and decompressed before JSON parsing. + * `resolveDecompressor` is invoked *outside* the catch block: if it throws (e.g. the codec + * named in the pointer is not registered on this consumer — a deployment misconfiguration, + * not a bad message), the throw propagates as a retriable error so the message stays on the + * queue instead of being silently routed to the DLQ. */ protected async retrieveOffloadedMessagePayload( maybeOffloadedPayloadPointerPayload: unknown, - decompress?: (codec: string, data: Buffer) => Promise, + resolveDecompressor?: (codec: string) => (data: Buffer) => Promise, ): Promise> { if (!this.payloadStoreConfig) { return { @@ -1000,22 +1026,23 @@ export abstract class AbstractQueueService< } const codec = parsedPayload.payloadRef?.codec - if (codec && decompress) { - // Stream read is kept outside the try/catch so transient retrieval errors (truncated - // S3 stream, network blip) propagate as thrown exceptions rather than being caught and - // returned as { error }. The caller (consumer) lets unhandled throws bubble up to - // sqs-consumer, which does NOT delete the message — it becomes visible again after the - // visibility timeout and is retried. Only deterministic failures (wrong codec, corrupt - // compressed bytes, invalid JSON after decompression) are caught and returned as - // { error }, which the consumer treats as a poison message and routes to the DLQ. - // This mirrors the non-codec path below, where streamWithKnownSizeToString is also - // outside its try/catch for the same reason. + if (codec && resolveDecompressor) { + // Resolve the decompressor OUTSIDE the try/catch. An unknown/unregistered codec is a + // consumer misconfiguration, not a poison message — letting it throw here makes it a + // retriable error (the message stays on the queue and becomes visible again after the + // visibility timeout) instead of being silently routed to the DLQ. + const decompress = resolveDecompressor(codec) + // The stream read is likewise kept outside the try/catch so transient retrieval errors + // (truncated S3 stream, network blip) propagate as thrown exceptions and are retried. + // Only deterministic failures (corrupt compressed bytes, invalid JSON after + // decompression) are caught and returned as { error }, which the consumer treats as a + // poison message and routes to the DLQ. This mirrors the non-codec path below. const compressedBuffer = await streamWithKnownSizeToBuffer( serializedOffloadedPayloadReadable, payloadSize, ) try { - const decompressed = await decompress(codec, compressedBuffer) + const decompressed = await decompress(compressedBuffer) return { result: JSON.parse(decompressed.toString('utf8')) } } catch (e) { return { diff --git a/packages/core/lib/types/queueOptionsTypes.ts b/packages/core/lib/types/queueOptionsTypes.ts index b614966e..d06ce9e1 100644 --- a/packages/core/lib/types/queueOptionsTypes.ts +++ b/packages/core/lib/types/queueOptionsTypes.ts @@ -140,66 +140,6 @@ export type CommonQueueOptions = { deletionConfig?: DeletionConfig payloadStoreConfig?: PayloadStoreConfig messageDeduplicationConfig?: MessageDeduplicationConfig - /** - * Compression codec applied to message bodies. - * - * - **Publisher**: every outgoing message body is compressed and wrapped in a - * self-describing envelope `{ __mqtCodec: 'zstd', __mqtData: '' }`. - * - **Consumer**: when set, the consumer expects compressed messages. - * Even without this option, consumers auto-detect and decompress any message - * that carries a codec envelope, so mixed queues work transparently. - * - * Uses Node.js built-in `zlib` zstd support — **requires Node.js >=22.15.0** (or >=23.8.0). - * - * @example - * import { MessageCodecEnum } from '@message-queue-toolkit/core' - * - * // Publisher - * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) - * - * // Consumer (optional — auto-detection handles it even without this) - * new MyConsumer(deps, { codec: MessageCodecEnum.ZSTD }) - */ - codec?: MessageCodecRegistration - /** - * Minimum serialized size in bytes a message must reach before compression is applied. - * Only meaningful when `codec` is set. Defaults to `512`. - * - * Small messages often expand rather than shrink when compressed due to algorithm - * framing overhead. When the UTF-8 JSON representation of a message is strictly - * smaller than this value, the message is sent as plain JSON instead of a codec - * envelope, avoiding the compression cost with no loss of correctness. - * - * Set to `0` to compress every message regardless of size. - * - * **Ignored when `payloadStoreConfig` is also set.** When both options are configured, - * the compress-and-offload pipeline always runs regardless of message size — the - * threshold is instead `messageSizeThreshold` (whether to upload to the store or - * inline the envelope). Use `skipCompressionBelow: 0` in that combination to make the - * intent explicit. - * - * @example - * // Compress only messages ≥ 1 KB - * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD, skipCompressionBelow: 1024 }) - * - * // Always compress (disable the floor) - * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD, skipCompressionBelow: 0 }) - */ - skipCompressionBelow?: number - /** - * Disables automatic codec-envelope detection on the consumer. - * - * By default, consumers inspect every incoming message body with `isCodecEnvelope`. - * If the body matches the envelope shape (`__mqtCodec` + `__mqtData` as the only two - * fields), it is treated as compressed and decompressed before schema validation. - * - * Set this to `true` if your message schema legitimately contains fields named - * `__mqtCodec` and `__mqtData` with exactly those two keys, and you do not want - * auto-detection to intercept them. Publisher behaviour is unaffected. - * - * @default false - */ - disableCodecAutoDetection?: boolean } export type CommonCreationConfigType = { @@ -353,6 +293,48 @@ export type QueuePublisherOptions< > = QueueOptions & { messageSchemas: readonly ZodSchema[] enablePublisherDeduplication?: boolean + /** + * Compression codec applied to outgoing message bodies. + * + * Every outgoing message body is compressed and wrapped in a self-describing + * envelope `{ __mqtCodec: 'zstd', __mqtData: '' }`. Consumers auto-detect + * the envelope and decompress transparently — no consumer-side option is needed + * for built-in codecs, so mixed (compressed/uncompressed) queues work seamlessly. + * + * Only the SQS and SNS adapters support compression; AMQP and Pub/Sub publishers + * throw at construction time if a codec is supplied. + * + * Uses Node.js built-in `zlib` zstd support — **requires Node.js >=22.15.0** (or >=23.8.0). + * + * @example + * import { MessageCodecEnum } from '@message-queue-toolkit/core' + * + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD }) + */ + codec?: MessageCodecRegistration + /** + * Minimum serialized size in bytes a message must reach before compression is applied. + * Only meaningful when `codec` is set. Defaults to `512`. + * + * Small messages often expand rather than shrink when compressed due to algorithm + * framing overhead. When the UTF-8 JSON representation of a message is strictly + * smaller than this value, the message is sent as plain JSON instead of a codec + * envelope, avoiding the compression cost with no loss of correctness. + * + * This floor is honored regardless of whether `payloadStoreConfig` is also set: + * a message below the threshold is sent (or offloaded) as plain JSON without + * compression. + * + * Set to `0` to compress every message regardless of size. + * + * @example + * // Compress only messages ≥ 1 KB + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD, skipCompressionBelow: 1024 }) + * + * // Always compress (disable the floor) + * new MyPublisher(deps, { codec: MessageCodecEnum.ZSTD, skipCompressionBelow: 0 }) + */ + skipCompressionBelow?: number } export type DeadLetterQueueOptions< diff --git a/packages/sns/README.md b/packages/sns/README.md index a9f5c4bb..44872985 100644 --- a/packages/sns/README.md +++ b/packages/sns/README.md @@ -43,7 +43,6 @@ npm install @message-queue-toolkit/sns @message-queue-toolkit/sqs @message-queue - `@aws-sdk/client-sqs` - AWS SDK for SQS (required for consumers) - `@aws-sdk/client-sts` - AWS SDK for STS (for ARN resolution) - `zod` - Schema validation -- `@message-queue-toolkit/codec` - Required when using message compression ## Features diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index 370f301c..c040e09c 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -2,7 +2,6 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sns' import { PublishCommand } from '@aws-sdk/client-sns' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -90,13 +89,6 @@ export abstract class AbstractSnsPublisher this.isFifoTopic = options.fifoTopic ?? false this.messageGroupIdField = options.messageGroupIdField this.defaultMessageGroupId = options.defaultMessageGroupId - - // Pre-resolve codec handler and name so the base-class prepareOutgoingPayload - // does not need to import @message-queue-toolkit/codec (circular dep risk). - if (options.codec) { - this.resolvedCodecHandler = resolveCodecHandler(options.codec) - this.resolvedCodecName = getCodecName(options.codec) - } } override async init(): Promise { @@ -133,7 +125,12 @@ export abstract class AbstractSnsPublisher const topicName = this.locatorConfig?.topicName ?? this.creationConfig?.topic?.Name ?? 'unknown' - // Dedup check before compression/offload: skip expensive work for duplicates. + // Dedup check runs before compression/offload so duplicates skip that expensive work + // entirely and never leave an orphaned object in the payload store. Trade-off: the + // publisher dedup key is persisted before the message is actually sent, so a crash + // between here and `sendMessage` drops the message on a subsequent retry. This is the + // same window that already existed around the bare `sendMessage` call; it now also + // spans compression/offload. if ( this.isDeduplicationEnabledForMessage(parsedMessage) && (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) diff --git a/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts index 2033c957..80eeb0f3 100644 --- a/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts +++ b/packages/sns/lib/sns/AbstractSnsSqsConsumer.ts @@ -41,9 +41,11 @@ export type SNSSQSConsumerOptions< SNSSQSCreationConfig, SNSSQSQueueLocatorType > & - // Omit here instead of at the top level so the SQSConsumerOptions discriminated + // Intersected after SQSConsumerOptions (rather than wrapping it) so its discriminated // union (fifoQueue: true | false) is preserved and Extract<…, {fifoQueue:true}> works. - Omit & { + // Codec consumer options (`codecs`, `disableCodecAutoDetection`) come from + // SQSConsumerOptions; SNSOptions carries no codec fields. + SNSOptions & { subscriptionConfig?: SNSSubscriptionOptions } diff --git a/packages/sns/package.json b/packages/sns/package.json index 5c6d232e..2736e230 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -38,7 +38,6 @@ "@aws-sdk/client-sns": "^3.632.0", "@aws-sdk/client-sqs": "^3.1034.0", "@aws-sdk/client-sts": "^3.632.0", - "@message-queue-toolkit/codec": ">=1.0.0", "@message-queue-toolkit/core": ">=25.5.0", "@message-queue-toolkit/schemas": ">=7.0.0", "@message-queue-toolkit/sqs": ">=23.0.0", @@ -51,7 +50,6 @@ "@biomejs/biome": "^2.3.6", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/codec": "workspace:*", "@message-queue-toolkit/core": "workspace:*", "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", "@message-queue-toolkit/s3-payload-store": "workspace:*", diff --git a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts index 3a026457..6b8c84c7 100644 --- a/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts +++ b/packages/sns/test/consumers/SnsSqsPermissionConsumer.codec.spec.ts @@ -29,8 +29,11 @@ describe('SnsSqsPermissionConsumer - zstd codec', () => { // No codec option needed — zstd is auto-registered on every consumer. consumer = new SnsSqsPermissionConsumer(diContainer.cradle) + // skipCompressionBelow: 0 forces compression for these small test messages so the + // suite genuinely exercises the codec path (they are well under the 512 B default). publisher = new SnsPermissionPublisher(diContainer.cradle, { codec: 'zstd', + skipCompressionBelow: 0, }) await consumer.start() diff --git a/packages/sns/test/publishers/SnsPermissionPublisher.ts b/packages/sns/test/publishers/SnsPermissionPublisher.ts index 659f8d87..da9dbe0c 100644 --- a/packages/sns/test/publishers/SnsPermissionPublisher.ts +++ b/packages/sns/test/publishers/SnsPermissionPublisher.ts @@ -27,6 +27,7 @@ export class SnsPermissionPublisher extends AbstractSnsPublisher | 'messageDeduplicationConfig' | 'enablePublisherDeduplication' | 'codec' + | 'skipCompressionBelow' >, ) { super(dependencies, { @@ -42,6 +43,7 @@ export class SnsPermissionPublisher extends AbstractSnsPublisher }, payloadStoreConfig: options?.payloadStoreConfig, codec: options?.codec, + skipCompressionBelow: options?.skipCompressionBelow, messageSchemas: [PERMISSIONS_ADD_MESSAGE_SCHEMA, PERMISSIONS_REMOVE_MESSAGE_SCHEMA], handlerSpy: true, messageTypeResolver: { messageTypePath: 'messageType' }, diff --git a/packages/sqs/README.md b/packages/sqs/README.md index db235279..28a0bd8b 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -49,7 +49,6 @@ npm install @message-queue-toolkit/sqs @message-queue-toolkit/core **Peer Dependencies:** - `@aws-sdk/client-sqs` - AWS SDK for SQS - `zod` - Schema validation -- `@message-queue-toolkit/codec` - Required when using message compression ## Features @@ -818,11 +817,7 @@ The codec embedded in `payloadRef.codec` tells the consumer which algorithm to u Compress message bodies with zstd using the Node.js built-in `zlib` module. Requires **Node.js >=22.15.0**. -The codec implementation lives in the separate [`@message-queue-toolkit/codec`](../codec/README.md) package, which must be installed alongside this package when using compression. - -```bash -npm install @message-queue-toolkit/codec -``` +The codec implementation ships inside `@message-queue-toolkit/core` — no extra package to install. Compression is opt-in: it is only active when you set the `codec` option on a publisher. Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __mqtCodec: 'zstd', __mqtData: '' }`), so a consumer without `codec` set will still decompress automatically via envelope detection. This allows a gradual rollout — enable compression on the publisher first, consumers adapt without configuration changes. diff --git a/packages/sqs/bench/codecMicro.bench.ts b/packages/sqs/bench/codecMicro.bench.ts index e41f45ce..f089469c 100644 --- a/packages/sqs/bench/codecMicro.bench.ts +++ b/packages/sqs/bench/codecMicro.bench.ts @@ -1,21 +1,24 @@ /** * Codec micro-benchmarks — compress/decompress latency in isolation (no network). * - * Run: pnpm --filter @message-queue-toolkit/sqs bench + * Run manually: pnpm --filter @message-queue-toolkit/sqs bench * - * These tests measure the CPU cost of the codec only, free from LocalStack - * network noise. Each case runs ITERATIONS compress+decompress round-trips and - * asserts the total time is below a very conservative ceiling, making them safe - * to run in CI as regression guards. + * These cases measure the CPU cost of the codec only, free from LocalStack network + * noise. Each runs ITERATIONS compress+decompress round-trips and asserts the total + * time is below a very conservative ceiling. * - * Expected times on typical developer/CI hardware: + * This is a manual/local benchmark — it is NOT wired into CI (the `bench` script is + * separate from `test`). Wall-clock assertions are inherently flaky on shared CI + * runners, so the ceilings here are a sanity check for local runs, not a CI gate. + * + * Expected times on typical developer hardware: * small payload (~100 B) → ~20–100 ms for 100 iterations * large payload (~6 KB) → ~50–300 ms for 100 iterations * * The ceilings below are set at ~10× the expected worst case so that only a - * genuine algorithmic regression (or a severely starved CI runner) will fail. + * genuine algorithmic regression (or a severely starved runner) will fail. */ -import { ZstdCodecHandler } from '@message-queue-toolkit/codec' +import { ZstdCodecHandler } from '@message-queue-toolkit/core' import { describe, expect, it } from 'vitest' const handler = new ZstdCodecHandler() diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index a3aa16bf..27739ee1 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -5,12 +5,12 @@ import { SetQueueAttributesCommand, } from '@aws-sdk/client-sqs' import type { Either, ErrorResolver } from '@lokalise/node-core' -import { getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' import type { ProcessedMessageMetadata } from '@message-queue-toolkit/core' import { type BarrierResult, type DeadLetterQueueOptions, DeduplicationRequesterEnum, + getCodecName, HandlerContainer, isCodecEnvelope, isMessageError, @@ -26,6 +26,7 @@ import { type QueueConsumer, type QueueConsumerDependencies, type QueueConsumerOptions, + resolveCodecHandler, type TransactionObservabilityManager, } from '@message-queue-toolkit/core' import type { ConsumerOptions } from 'sqs-consumer' @@ -80,18 +81,15 @@ type SQSConsumerCommonOptions< PrehandlerOutput, CreationConfigType extends SQSCreationConfig = SQSCreationConfig, QueueLocatorType extends object = SQSQueueLocatorType, -> = Omit< - QueueConsumerOptions< - CreationConfigType, - QueueLocatorType, - SQSDeadLetterQueueOptions, - MessagePayloadSchemas, - ExecutionContext, - PrehandlerOutput, - SQSCreationConfig, - SQSQueueLocatorType - >, - 'codec' | 'skipCompressionBelow' +> = QueueConsumerOptions< + CreationConfigType, + QueueLocatorType, + SQSDeadLetterQueueOptions, + MessagePayloadSchemas, + ExecutionContext, + PrehandlerOutput, + SQSCreationConfig, + SQSQueueLocatorType > & { /** * Additional codecs to register on this consumer. @@ -104,6 +102,19 @@ type SQSConsumerCommonOptions< * new MyConsumer(deps, { codecs: [codec] }) */ codecs?: MessageCodecRegistration[] + /** + * Disables automatic codec-envelope detection on this consumer. + * + * By default, consumers inspect every incoming message body with `isCodecEnvelope`. + * If the body matches the envelope shape (`__mqtCodec` + `__mqtData` as the only two + * fields), it is treated as compressed and decompressed before schema validation. + * + * Set this to `true` if your message schema legitimately contains exactly those two + * fields and you do not want auto-detection to intercept them. + * + * @default false + */ + disableCodecAutoDetection?: boolean /** * Wait time in seconds the consumer passes to SQS ReceiveMessage (long-polling). * AWS allows integer values 0–20; anything else throws a RangeError at @@ -955,10 +966,17 @@ export abstract class AbstractSqsConsumer< if (hasOffloadedPayload(resolveMessageResult.result)) { const retrieveOffloadedMessagePayloadResult = await this.retrieveOffloadedMessagePayload( resolveMessageResult.result.body, - (codecName, data) => { + (codecName) => { const handler = this.codecRegistry.get(codecName) - if (!handler) throw new Error(`Unknown codec: ${codecName}`) - return handler.decompress(data) + if (!handler) { + // Misconfiguration, not a poison message: the pointer names a codec this + // consumer has not registered. Throwing here (outside retrieveOffloadedMessagePayload's + // catch) surfaces it as a retriable error so the message is not lost to the DLQ. + throw new Error( + `No codec handler registered for "${codecName}". Register it via the consumer's \`codecs\` option.`, + ) + } + return (data) => handler.decompress(data) }, ) if (retrieveOffloadedMessagePayloadResult.error) { diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index 399385ad..ebc00365 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -2,7 +2,6 @@ import type { MessageAttributeValue } from '@aws-sdk/client-sqs' import { SendMessageCommand } from '@aws-sdk/client-sqs' import type { Either } from '@lokalise/node-core' import { InternalError } from '@lokalise/node-core' -import { getCodecName, resolveCodecHandler } from '@message-queue-toolkit/codec' import { type AsyncPublisher, type BarrierResult, @@ -96,13 +95,6 @@ export abstract class AbstractSqsPublisher this.isDeduplicationEnabled = !!options.enablePublisherDeduplication this.messageGroupIdField = options.messageGroupIdField this.defaultMessageGroupId = options.defaultMessageGroupId - - // Pre-resolve codec handler and name so the base-class prepareOutgoingPayload - // does not need to import @message-queue-toolkit/codec (circular dep risk). - if (options.codec) { - this.resolvedCodecHandler = resolveCodecHandler(options.codec) - this.resolvedCodecName = getCodecName(options.codec) - } } async publish(message: MessagePayloadType, options: SQSMessageOptions = {}): Promise { @@ -125,7 +117,12 @@ export abstract class AbstractSqsPublisher const messageProcessingStartTimestamp = Date.now() const parsedMessage = messageSchemaResult.result.parse(message) - // Dedup check before compression/offload: skip expensive work for duplicates. + // Dedup check runs before compression/offload so duplicates skip that expensive work + // entirely and never leave an orphaned object in the payload store. Trade-off: the + // publisher dedup key is persisted before the message is actually sent, so a crash + // between here and `sendMessage` drops the message on a subsequent retry. This is the + // same window that already existed around the bare `sendMessage` call; it now also + // spans compression/offload. if ( this.isDeduplicationEnabledForMessage(parsedMessage) && (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) diff --git a/packages/sqs/package.json b/packages/sqs/package.json index d6b8e2fa..a93374a7 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -37,7 +37,6 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.1034.0", - "@message-queue-toolkit/codec": ">=1.0.0", "@message-queue-toolkit/core": ">=25.5.0", "zod": ">=3.25.76 <5.0.0" }, @@ -47,7 +46,6 @@ "@biomejs/biome": "^2.3.8", "@lokalise/biome-config": "^3.1.0", "@lokalise/tsconfig": "^3.0.0", - "@message-queue-toolkit/codec": "workspace:*", "@message-queue-toolkit/core": "workspace:*", "@message-queue-toolkit/redis-message-deduplication-store": "workspace:*", "@message-queue-toolkit/s3-payload-store": "workspace:*", diff --git a/packages/sqs/test/codec/codecHandler.spec.ts b/packages/sqs/test/codec/codecHandler.spec.ts index 8addbc8d..18fd6b35 100644 --- a/packages/sqs/test/codec/codecHandler.spec.ts +++ b/packages/sqs/test/codec/codecHandler.spec.ts @@ -1,5 +1,4 @@ -import { decompressMessageBody, getCodecName } from '@message-queue-toolkit/codec' -import { MessageCodecEnum } from '@message-queue-toolkit/core' +import { decompressMessageBody, getCodecName, MessageCodecEnum } from '@message-queue-toolkit/core' import { describe, expect, it } from 'vitest' describe('getCodecName', () => { diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index d7ce7004..84edea10 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -2,9 +2,8 @@ import type { Transform } from 'node:stream' import { PassThrough } from 'node:stream' import { ReceiveMessageCommand, SendMessageCommand } from '@aws-sdk/client-sqs' import { waitAndRetry } from '@lokalise/node-core' -import { compressMessageBody } from '@message-queue-toolkit/codec' import type { MessageCodecHandler } from '@message-queue-toolkit/core' -import { MessageCodecEnum } from '@message-queue-toolkit/core' +import { compressMessageBody, MessageCodecEnum } from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' @@ -16,9 +15,11 @@ import { registerDependencies } from '../utils/testContext.ts' import { SqsPermissionConsumer } from './SqsPermissionConsumer.ts' import type { PERMISSIONS_ADD_MESSAGE_TYPE } from './userConsumerSchemas.ts' -// Padding that pushes any test message's JSON representation above the default -// skipCompressionBelow threshold (512 bytes), ensuring compression is actually applied. -const LARGE_PADDING = 'x'.repeat(450) +// Padding that pushes any test message's JSON representation comfortably above the +// default skipCompressionBelow threshold (512 bytes), ensuring compression is actually +// applied. Sized with generous margin so unrelated schema tweaks cannot drop a test +// message back under the threshold. +const LARGE_PADDING = 'x'.repeat(800) describe('SqsPermissionConsumer - zstd codec', () => { let diContainer: AwilixContainer diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts index 22e369e6..15467c92 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.payloadOffloading.spec.ts @@ -448,6 +448,7 @@ describe('SqsPermissionConsumer - codec + payload offloading', () => { const wirePublisher = new SqsPermissionPublisher(diContainer.cradle, { codec: MessageCodecEnum.ZSTD, + skipCompressionBelow: 0, // always compress: this suite exercises the codec + offload path payloadStoreConfig, creationConfig: { queue: { QueueName: wireQueueName } }, }) @@ -498,6 +499,7 @@ describe('SqsPermissionConsumer - codec + payload offloading', () => { const publisher = new SqsPermissionPublisher(diContainer.cradle, { codec: MessageCodecEnum.ZSTD, + skipCompressionBelow: 0, // always compress: this suite exercises the codec + offload path payloadStoreConfig, creationConfig: { queue: { QueueName: queueName } }, }) @@ -536,6 +538,7 @@ describe('SqsPermissionConsumer - codec + payload offloading', () => { const publisher = new SqsPermissionPublisher(diContainer.cradle, { codec: MessageCodecEnum.ZSTD, + skipCompressionBelow: 0, // always compress: this suite exercises the codec + offload path payloadStoreConfig, creationConfig: { queue: { QueueName: queueName } }, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7a80aa37..7b3ecc48 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -64,30 +64,6 @@ importers: specifier: ^4.1.13 version: 4.4.3 - packages/codec: - devDependencies: - '@biomejs/biome': - specifier: ^2.3.8 - version: 2.4.15 - '@lokalise/biome-config': - specifier: ^3.1.0 - version: 3.1.1 - '@lokalise/tsconfig': - specifier: ^3.0.0 - version: 3.1.0 - '@message-queue-toolkit/core': - specifier: workspace:* - version: link:../core - '@types/node': - specifier: ^22.0.0 - version: 22.19.19 - rimraf: - specifier: ^6.0.1 - version: 6.1.3 - typescript: - specifier: ^5.9.3 - version: 5.9.3 - packages/core: dependencies: '@lokalise/node-core': @@ -521,9 +497,6 @@ importers: '@lokalise/tsconfig': specifier: ^3.0.0 version: 3.1.0 - '@message-queue-toolkit/codec': - specifier: workspace:* - version: link:../codec '@message-queue-toolkit/core': specifier: workspace:* version: link:../core @@ -591,9 +564,6 @@ importers: '@lokalise/tsconfig': specifier: ^3.0.0 version: 3.1.0 - '@message-queue-toolkit/codec': - specifier: workspace:* - version: link:../codec '@message-queue-toolkit/core': specifier: workspace:* version: link:../core @@ -681,6 +651,7 @@ packages: '@aws-sdk/core@3.974.11': resolution: {integrity: sha512-QpnINq5FZH6EOaDEkmHdT7eUunbvD27pDNQypaWjFyYz7Zl1q3UCMQErBZxpmfGfI7MvI2TlK8KTkgNpv8b1ug==} engines: {node: '>=20.0.0'} + deprecated: Deprecated due to an error deserialization bug in JSON 1.0 protocol services, see https://github.com/aws/aws-sdk-js-v3/pull/8031. Newer version available. '@aws-sdk/crc64-nvme@3.972.8': resolution: {integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==} @@ -1434,9 +1405,6 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} - '@types/node@22.19.19': - resolution: {integrity: sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==} - '@types/node@25.8.0': resolution: {integrity: sha512-TCFSk8IZh+iLX1xtksoBVtdmgL+1IX0fC9BeU4QqFSuNdN/K+HUlhqOzEmSYYpZUVsLYcPqc9KX+60iDuninSQ==} @@ -2613,9 +2581,6 @@ packages: resolution: {integrity: sha512-xY7c8LPyzvhvew0Fn+Ek3wBC9STZAuDI/Y5andCKi9AX6/jvfaX45PhsDX8oxgPL0YFp0Jhr8qWMbS/p9375Xg==} engines: {node: '>=16'} - undici-types@6.21.0: - resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} - undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} @@ -3713,10 +3678,6 @@ snapshots: '@types/estree@1.0.9': {} - '@types/node@22.19.19': - dependencies: - undici-types: 6.21.0 - '@types/node@25.8.0': dependencies: undici-types: 7.24.6 @@ -4946,8 +4907,6 @@ snapshots: dependencies: layerr: 3.0.0 - undici-types@6.21.0: {} - undici-types@7.24.6: {} util-deprecate@1.0.2: {} From 084666a317ca638212a1c711fb25cefb5ab8a18b Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 22 May 2026 16:57:40 +0300 Subject: [PATCH 21/23] Extra refinement --- .github/workflows/ensure-labels.yml | 2 +- .github/workflows/publish.yml | 30 +- packages/amqp/package.json | 2 +- packages/core/README.md | 4 +- packages/core/lib/codec/codecHandler.ts | 55 ++- packages/core/lib/codec/messageCodec.ts | 50 ++- packages/core/lib/index.ts | 2 + .../offloadedPayloadMessageSchemas.ts | 15 +- .../core/lib/queues/AbstractQueueService.ts | 372 +++++++++++------- packages/core/package.json | 2 +- packages/core/test/codec/codecHandler.spec.ts | 59 +++ packages/core/test/codec/messageCodec.spec.ts | 56 ++- packages/gcp-pubsub/package.json | 4 +- packages/sns/lib/sns/AbstractSnsPublisher.ts | 34 +- packages/sns/package.json | 4 +- packages/sqs/README.md | 6 +- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 25 +- packages/sqs/lib/sqs/AbstractSqsPublisher.ts | 34 +- packages/sqs/package.json | 4 +- .../SqsPermissionConsumer.codec.spec.ts | 19 +- 20 files changed, 562 insertions(+), 217 deletions(-) create mode 100644 packages/core/test/codec/codecHandler.spec.ts diff --git a/.github/workflows/ensure-labels.yml b/.github/workflows/ensure-labels.yml index 2a506924..81d2ed74 100644 --- a/.github/workflows/ensure-labels.yml +++ b/.github/workflows/ensure-labels.yml @@ -24,5 +24,5 @@ jobs: - name: Check one of required labels are set uses: docker://agilepathway/pull-request-label-checker:v1.6.65 with: - one_of: major,minor,patch,skip-release + one_of: major,minor,patch,skip-release,release-same-version repo_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4da12fc7..f1e5c9da 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,6 +23,7 @@ jobs: has_changes: ${{ steps.finalize.outputs.has_changes }} bump: ${{ steps.finalize.outputs.bump }} should_publish: ${{ steps.finalize.outputs.should_publish }} + same_version: ${{ steps.finalize.outputs.same_version }} steps: - name: Set default outputs id: defaults @@ -31,6 +32,7 @@ jobs: echo "has_changes=false" >> $GITHUB_OUTPUT echo "bump=patch" >> $GITHUB_OUTPUT echo "should_publish=false" >> $GITHUB_OUTPUT + echo "same_version=false" >> $GITHUB_OUTPUT - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -79,11 +81,11 @@ jobs: echo "PR labels: $LABELS" echo "labels=$LABELS" >> $GITHUB_OUTPUT - # Check if PR has version bump labels - if echo "$LABELS" | grep -qE '\b(patch|minor|major)\b'; then + # Check if PR has a release label (a version-bump label or release-same-version) + if echo "$LABELS" | grep -qE '\b(patch|minor|major|release-same-version)\b'; then echo "should_publish=true" >> $GITHUB_OUTPUT else - echo "No version bump label found (patch/minor/major)" + echo "No release label found (patch/minor/major/release-same-version)" echo "should_publish=false" >> $GITHUB_OUTPUT fi @@ -93,13 +95,23 @@ jobs: env: LABELS: ${{ steps.pr-info.outputs.labels }} run: | - if echo "$LABELS" | grep -qE '\bmajor\b'; then + # release-same-version takes precedence: publish the versions already in + # package.json as-is, with no bump. Use it when versions were set explicitly + # in the PR (e.g. a deliberate, hand-committed semver-major bump). + if echo "$LABELS" | grep -qE '\brelease-same-version\b'; then + echo "same_version=true" >> $GITHUB_OUTPUT + echo "bump=none" >> $GITHUB_OUTPUT + echo "Release mode: publish current package.json versions without bumping" + elif echo "$LABELS" | grep -qE '\bmajor\b'; then + echo "same_version=false" >> $GITHUB_OUTPUT echo "bump=major" >> $GITHUB_OUTPUT echo "Version bump: major" elif echo "$LABELS" | grep -qE '\bminor\b'; then + echo "same_version=false" >> $GITHUB_OUTPUT echo "bump=minor" >> $GITHUB_OUTPUT echo "Version bump: minor" else + echo "same_version=false" >> $GITHUB_OUTPUT echo "bump=patch" >> $GITHUB_OUTPUT echo "Version bump: patch" fi @@ -240,6 +252,8 @@ jobs: DEFAULT_BUMP: ${{ steps.defaults.outputs.bump }} PR_SHOULD_PUBLISH: ${{ steps.pr-info.outputs.should_publish }} DEFAULT_SHOULD_PUBLISH: ${{ steps.defaults.outputs.should_publish }} + VERSION_SAME: ${{ steps.version.outputs.same_version }} + DEFAULT_SAME_VERSION: ${{ steps.defaults.outputs.same_version }} run: | # Use build-matrix outputs if available, otherwise defaults if [ -n "$BUILD_MATRIX" ]; then @@ -266,6 +280,12 @@ jobs: echo "should_publish=$DEFAULT_SHOULD_PUBLISH" >> $GITHUB_OUTPUT fi + if [ -n "$VERSION_SAME" ]; then + echo "same_version=$VERSION_SAME" >> $GITHUB_OUTPUT + else + echo "same_version=$DEFAULT_SAME_VERSION" >> $GITHUB_OUTPUT + fi + # Single job that bumps, publishes, and pushes tags/commits at the end # This avoids the "checkout wrong commit" problem of multi-job workflows release: @@ -305,8 +325,10 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts + # Skipped for release-same-version: the versions in package.json are published as-is. - name: Bump versions for changed packages id: bump + if: needs.detect-changes.outputs.same_version != 'true' env: MATRIX: ${{ needs.detect-changes.outputs.matrix }} BUMP: ${{ needs.detect-changes.outputs.bump }} diff --git a/packages/amqp/package.json b/packages/amqp/package.json index d55f7fde..6030de1e 100644 --- a/packages/amqp/package.json +++ b/packages/amqp/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/amqp", - "version": "24.0.0", + "version": "24.1.0", "engines": { "node": ">=18" }, diff --git a/packages/core/README.md b/packages/core/README.md index a6915a2f..b83760cd 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -674,7 +674,9 @@ new MyPublisher(deps, { codec }) new MyConsumer(deps, { codecs: [codec] }) // register custom codec on the consumer ``` -Compressed messages are wrapped in a self-describing envelope `{ __mqtCodec: '', __mqtData: '' }`. Built-in codecs (e.g. zstd) are auto-detected on every consumer — no consumer option needed. For custom codecs, pass `codecs: [{ name, handler }]` to register them on the consumer. +Compressed messages are wrapped in a self-describing envelope `{ __mqtCodec: '', __mqtData: '', ...preserved fields }`. The message's identity/routing fields (`id`, `timestamp`, `type`, deduplication fields) are copied alongside `__mqtData` as plaintext — the same fields an offloaded-payload pointer preserves — so broker-side filtering (e.g. SNS body-scoped `FilterPolicy`) keeps working on them. Built-in codecs (e.g. zstd) are auto-detected on every consumer — no consumer option needed. For custom codecs, pass `codecs: [{ name, handler }]` to register them on the consumer. + +> **Roll out consumers before publishers:** auto-detection only works on a consumer running a library version that supports the codec (and, for custom codecs, with that codec registered). Upgrade all consumers of a queue first, then enable `codec` on publishers. #### Interaction with codec (compression) diff --git a/packages/core/lib/codec/codecHandler.ts b/packages/core/lib/codec/codecHandler.ts index 9de442f1..cbc74f4d 100644 --- a/packages/core/lib/codec/codecHandler.ts +++ b/packages/core/lib/codec/codecHandler.ts @@ -12,6 +12,18 @@ const ZSTD_UNSUPPORTED_MSG = 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + 'Message compression requires Node.js >=22.15.0 or >=23.8.0.' +/** + * Default upper bound on the decompressed size of a single message, in bytes (100 MiB). + * + * Protects consumers from decompression-bomb inputs: a tiny compressed envelope can + * otherwise expand to gigabytes of highly-repetitive data and exhaust process memory. + * 100 MiB is far above any realistic queue message (SQS/SNS cap bodies at 256 KiB, and + * even offloaded payloads are typically single-digit MiB) while still bounding the blast + * radius of a malicious or corrupt frame. Override via the {@link ZstdCodecHandler} + * constructor if you legitimately handle larger messages. + */ +export const DEFAULT_MAX_DECOMPRESSED_BYTES = 100 * 1024 * 1024 + // Resolved lazily — undefined on Node versions that lack zstd support. // Keeping these lazy means importing core never throws on older Node; only an // actual compress/decompress call does, and only when zstd is genuinely used. @@ -21,6 +33,17 @@ const zstdDecompress = typeof zlib.zstdDecompress === 'function' ? promisify(zlib.zstdDecompress) : undefined export class ZstdCodecHandler implements MessageCodecHandler { + private readonly maxDecompressedBytes: number + + /** + * @param maxDecompressedBytes upper bound on a single decompressed message, in bytes. + * Defaults to {@link DEFAULT_MAX_DECOMPRESSED_BYTES} (100 MiB). Decompression of an + * input that would exceed this limit is rejected before the full payload is buffered. + */ + constructor(maxDecompressedBytes: number = DEFAULT_MAX_DECOMPRESSED_BYTES) { + this.maxDecompressedBytes = maxDecompressedBytes + } + compress(data: Buffer): Promise { if (!zstdCompress) throw new Error(ZSTD_UNSUPPORTED_MSG) return zstdCompress(data) @@ -28,7 +51,9 @@ export class ZstdCodecHandler implements MessageCodecHandler { decompress(data: Buffer): Promise { if (!zstdDecompress) throw new Error(ZSTD_UNSUPPORTED_MSG) - return zstdDecompress(data) + // maxOutputLength caps the decompressed size: zstdDecompress rejects with a + // RangeError once the limit is exceeded, guarding against decompression bombs. + return zstdDecompress(data, { maxOutputLength: this.maxDecompressedBytes }) } createCompressStream(): Transform { @@ -87,15 +112,33 @@ export async function compressMessageBody( * Wraps an already-compressed buffer in a codec envelope string. * Use this when you have pre-compressed bytes and want to avoid compressing twice. * - * Uses string concatenation instead of JSON.stringify to avoid allocating an - * intermediate object — the base64 string and the envelope string are the only - * two allocations on the inline path. + * `preservedFields`, when provided, are emitted as plaintext siblings of the codec + * fields (`{ ...preserved, __mqtCodec, __mqtData }`). Publishers use this to keep + * identity/routing fields (`id`, `type`, …) visible on the wire so broker-side + * filtering (e.g. SNS body-scoped FilterPolicy) still works on compressed messages — + * the same fields an offloaded-payload pointer carries. The codec fields are written + * last, so a colliding preserved key can never corrupt the envelope; consumers ignore + * the preserved siblings and decode `__mqtData` only. + * + * Without `preservedFields` the fast path uses string concatenation instead of + * JSON.stringify, avoiding an intermediate object — the base64 string and the + * envelope string are the only two allocations on the inline path. * * `codecName` must already be a JSON-safe identifier (see {@link getCodecName}, * which is enforced for every registration before it reaches this function). */ -export function buildCodecEnvelope(compressed: Buffer, codecName: string): string { - return '{"__mqtCodec":"' + codecName + '","__mqtData":"' + compressed.toString('base64') + '"}' +export function buildCodecEnvelope( + compressed: Buffer, + codecName: string, + preservedFields?: Record, +): string { + const data = compressed.toString('base64') + if (!preservedFields || Object.keys(preservedFields).length === 0) { + return `{"__mqtCodec":"${codecName}","__mqtData":"${data}"}` + } + // Preserved fields present: a single JSON.stringify handles all value escaping. + // Codec fields are listed last so they always win over any colliding preserved key. + return JSON.stringify({ ...preservedFields, __mqtCodec: codecName, __mqtData: data }) } /** diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index d98643ca..389c2803 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -78,27 +78,47 @@ export const BASE64_RE = export const KNOWN_CODECS: ReadonlySet = new Set(Object.values(MessageCodecEnum)) /** - * Returns true when `value` is a codec envelope that the consumer should decompress. + * Structural check: returns true when `value` has the shape of a codec envelope — + * a non-empty string `__mqtCodec` and a base64 `__mqtData` — **regardless of whether + * the named codec is one this consumer can decode**. * - * Pass `knownCodecs` to restrict auto-detection to the codecs your consumer is - * configured to handle (built from the `codec` option). Defaults to the built-in - * codec set — backwards-compatible for consumers that don't configure a codec. + * Detection is **presence-based**: extra sibling fields are allowed, because publishers + * copy identity/routing fields (`id`, `type`, …) alongside the codec fields so + * broker-side filtering (e.g. SNS body-scoped FilterPolicy) keeps working on compressed + * messages. This mirrors how an offloaded-payload pointer is recognised by its + * `payloadRef` shape, not by an exact object shape. + * + * Consumers use this (rather than {@link isCodecEnvelope}) so an envelope for an + * unregistered codec can be told apart from an ordinary message and surfaced as a + * misconfiguration instead of being validated as an incomplete skeleton. The cheap + * `in` checks run first, so a non-envelope value returns without allocating anything. */ -export function isCodecEnvelope( - value: unknown, - knownCodecs: ReadonlySet = KNOWN_CODECS, -): value is CodecEnvelope { +export function hasCodecEnvelopeShape(value: unknown): value is CodecEnvelope { + if (typeof value !== 'object' || value === null) return false const record = value as Record return ( - typeof value === 'object' && - value !== null && - // Exact two-key shape — extra fields mean this is a real message, not an envelope. - Object.keys(value).length === 2 && - CODEC_FIELD in value && - DATA_FIELD in value && - knownCodecs.has(record[CODEC_FIELD] as string) && + CODEC_FIELD in record && + DATA_FIELD in record && + typeof record[CODEC_FIELD] === 'string' && + (record[CODEC_FIELD] as string).length > 0 && typeof record[DATA_FIELD] === 'string' && // Validate __mqtData is a properly-padded base64 string before handing it to the codec. BASE64_RE.test(record[DATA_FIELD] as string) ) } + +/** + * Returns true when `value` is a codec envelope **for a codec in `knownCodecs`** — i.e. + * one this consumer can actually decode. Combines the structural + * {@link hasCodecEnvelopeShape} check with a codec-name lookup. + * + * Pass `knownCodecs` to restrict the match to the codecs your consumer is configured to + * handle. Defaults to the built-in codec set — backwards-compatible for consumers that + * don't configure a codec. + */ +export function isCodecEnvelope( + value: unknown, + knownCodecs: ReadonlySet = KNOWN_CODECS, +): value is CodecEnvelope { + return hasCodecEnvelopeShape(value) && knownCodecs.has((value as CodecEnvelope).__mqtCodec) +} diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index 48451c50..b6b11bfd 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,6 +1,7 @@ export { buildCodecEnvelope, compressMessageBody, + DEFAULT_MAX_DECOMPRESSED_BYTES, decompressMessageBody, getCodecName, resolveCodecHandler, @@ -9,6 +10,7 @@ export { export { BASE64_RE, type CodecEnvelope, + hasCodecEnvelopeShape, isCodecEnvelope, KNOWN_CODECS, type MessageCodec, diff --git a/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts b/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts index e6c1dee1..807fc84e 100644 --- a/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts +++ b/packages/core/lib/payload-store/offloadedPayloadMessageSchemas.ts @@ -1,5 +1,16 @@ import { z } from 'zod/v4' +/** + * Upper bound for the declared size of an offloaded payload, in bytes (256 MiB). + * + * The size is read from an incoming (potentially untrusted) pointer and used to + * pre-allocate a buffer when streaming the payload back from the store. Capping it + * here means a malformed or hostile pointer claiming a multi-gigabyte size fails + * schema validation (and is routed to the DLQ) instead of triggering a huge + * allocation. 256 MiB is far above any realistic queue payload. + */ +export const MAX_OFFLOADED_PAYLOAD_SIZE = 256 * 1024 * 1024 + /** * Multi-store payload reference schema. * Contains information about where and how the payload was stored. @@ -10,7 +21,7 @@ export const PAYLOAD_REF_SCHEMA = z.object({ /** Name/identifier of the store where the payload is stored */ store: z.string().min(1), /** Size of the payload in bytes */ - size: z.number().int().positive(), + size: z.number().int().positive().max(MAX_OFFLOADED_PAYLOAD_SIZE), /** * Codec used to compress the stored payload. * When set, the stored bytes are raw compressed binary (not base64 JSON). @@ -42,7 +53,7 @@ export const OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA = z payloadRef: PAYLOAD_REF_SCHEMA.optional(), // Legacy payload reference, preserved for backward compatibility. offloadedPayloadPointer: z.string().min(1).optional(), - offloadedPayloadSize: z.number().int().positive().optional(), + offloadedPayloadSize: z.number().int().positive().max(MAX_OFFLOADED_PAYLOAD_SIZE).optional(), }) // Pass-through allows to pass message ID, type, timestamp and message-deduplication-related fields that are using dynamic keys. .passthrough() diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 88f3ab95..2b98de6f 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -44,6 +44,7 @@ import { import type { MultiPayloadStoreConfig, PayloadStore, + SerializedPayload, SinglePayloadStoreConfig, } from '../payload-store/payloadStoreTypes.ts' import { isDestroyable, isMultiPayloadStoreConfig } from '../payload-store/payloadStoreTypes.ts' @@ -199,7 +200,9 @@ export abstract class AbstractQueueService< skipCompressionBelow: number disableCodecAutoDetection: boolean }> - this.skipCompressionBelow = codecOptions.skipCompressionBelow ?? 512 + // Clamp to a non-negative integer: a negative or fractional floor is meaningless + // (a negative value would behave like 0 — "always compress" — anyway). + this.skipCompressionBelow = Math.max(0, Math.trunc(codecOptions.skipCompressionBelow ?? 512)) this.disableCodecAutoDetection = codecOptions.disableCodecAutoDetection ?? false if (codecOptions.codec) { this.resolvedCodecHandler = resolveCodecHandler(codecOptions.codec) @@ -690,13 +693,45 @@ export abstract class AbstractQueueService< } /** - * Builds an OffloadedPayloadPointerPayload from the given message and storage metadata. - * Copies identity fields and preserves the message type field through offloading. + * Collects the identity/routing fields that must remain visible in plaintext on the + * wire even when the message body is compressed or offloaded: the configured id, + * timestamp and deduplication fields, plus the message `type` (resolved via + * `messageTypeResolver`, defaulting to the conventional top-level `type` path). * - * We default to the conventional top-level `type` path so that routing/identity fields are - * handled consistently with `messageIdField`/`messageTimestampField`/etc. Without this - * fallback, `messageTypeResolver` modes that don't specify a body path silently strip `type` - * from the offloaded body, breaking downstream SNS subscription FilterPolicy filters. + * Keeping these as plaintext siblings of an offload pointer / codec envelope is what + * lets routing and downstream filtering keep working. Without the `type` fallback, + * `messageTypeResolver` modes that don't specify a body path silently strip `type`, + * breaking downstream SNS subscription FilterPolicy filters (`FilterPolicyScope: + * 'MessageBody'`). Shared by `buildPointer` (offload) and the codec-envelope path + * (`prepareOutgoingPayload`) so both behave identically. + */ + protected collectPreservedFields(message: MessagePayloadSchemas): Record { + const fields: Record = { + // @ts-expect-error dynamic field access + [this.messageIdField]: message[this.messageIdField], + // @ts-expect-error dynamic field access + [this.messageTimestampField]: message[this.messageTimestampField], + // @ts-expect-error dynamic field access + [this.messageDeduplicationIdField]: message[this.messageDeduplicationIdField], + // @ts-expect-error dynamic field access + [this.messageDeduplicationOptionsField]: message[this.messageDeduplicationOptionsField], + } + + const typePath = + this.messageTypeResolver && isMessageTypePathConfig(this.messageTypeResolver) + ? this.messageTypeResolver.messageTypePath + : 'type' + const typeValue = getProperty(message, typePath) + if (typeValue !== undefined) { + setProperty(fields, typePath, typeValue) + } + + return fields + } + + /** + * Builds an OffloadedPayloadPointerPayload from the given message and storage metadata. + * Identity/routing fields are preserved through offloading via {@link collectPreservedFields}. */ private buildPointer( message: MessagePayloadSchemas, @@ -705,7 +740,7 @@ export abstract class AbstractQueueService< size: number, codecName?: string, ): OffloadedPayloadPointerPayload { - const result: OffloadedPayloadPointerPayload = { + return { payloadRef: { id: payloadId, store: storeName, @@ -714,26 +749,8 @@ export abstract class AbstractQueueService< }, offloadedPayloadPointer: payloadId, offloadedPayloadSize: size, - // @ts-expect-error - [this.messageIdField]: message[this.messageIdField], - // @ts-expect-error - [this.messageTimestampField]: message[this.messageTimestampField], - // @ts-expect-error - [this.messageDeduplicationIdField]: message[this.messageDeduplicationIdField], - // @ts-expect-error - [this.messageDeduplicationOptionsField]: message[this.messageDeduplicationOptionsField], - } - - const typePath = - this.messageTypeResolver && isMessageTypePathConfig(this.messageTypeResolver) - ? this.messageTypeResolver.messageTypePath - : 'type' - const typeValue = getProperty(message, typePath) - if (typeValue !== undefined) { - setProperty(result, typePath, typeValue) - } - - return result + ...this.collectPreservedFields(message), + } as OffloadedPayloadPointerPayload } /** @@ -793,43 +810,7 @@ export abstract class AbstractQueueService< const codecName = this.resolvedCodecName if (handler && codecName) { - // Serialize once up-front. The result is reused to apply `skipCompressionBelow` - // and, on the inline path, as the codec input / plain-JSON wire body — so the - // message is never stringified twice. - const json = JSON.stringify(message) - - // Skip compression for messages below the configured floor — small payloads - // often grow rather than shrink when compressed. Honored whether or not a - // payload store is configured. - if (Buffer.byteLength(json, 'utf8') < this.skipCompressionBelow) { - if (this.payloadStoreConfig) { - const pointer = await this.offloadPayload(message, () => - this.calculateOutgoingMessageSize(message), - ) - return { payload: pointer ?? message } - } - // Reuse the already-serialized JSON as the wire body — no second stringify. - return { payload: message, preBuiltBody: json } - } - - if (this.payloadStoreConfig) { - // Compress once, then offload or inline based on the codec envelope wire size. - const result = await this.compressAndOffloadPayload(message, handler, codecName) - if (result.pointer) { - return { payload: result.pointer } - } - return { - payload: message, - preBuiltBody: buildCodecEnvelope(result.compressedBuffer, codecName), - } - } - - // No offload store — bounded by the transport limit (SQS/SNS 256 KB), safe to buffer. - const compressed = await handler.compress(Buffer.from(json, 'utf8')) - return { - payload: message, - preBuiltBody: buildCodecEnvelope(compressed, codecName), - } + return this.prepareCompressedOutgoingPayload(message, handler, codecName) } return { @@ -839,14 +820,74 @@ export abstract class AbstractQueueService< } } + /** + * Codec branch of {@link prepareOutgoingPayload}, extracted so each method stays within + * the cognitive-complexity budget. + * + * With a payload store, serialization is delegated to {@link compressAndOffloadPayload}, + * which uses the store serializer's reported size to honor `skipCompressionBelow` — the + * message is never separately `JSON.stringify`'d, so a streaming serializer is not forced + * to fully materialize a large payload. Without a payload store, the wire body is bounded + * by the SQS/SNS 256 KB transport limit (and is a string parameter that cannot be + * streamed), so a single in-memory `JSON.stringify` is both necessary and safe. + */ + private async prepareCompressedOutgoingPayload( + message: MessagePayloadSchemas, + handler: MessageCodecHandler, + codecName: string, + ): Promise<{ + payload: MessagePayloadSchemas | OffloadedPayloadPointerPayload + preBuiltBody?: string + }> { + if (this.payloadStoreConfig) { + return this.compressAndOffloadPayload(message, handler, codecName) + } + + // No offload store: serialize exactly once. The result is the codec input and, when + // compression is skipped, the plain-JSON wire body. + const json = JSON.stringify(message) + + // Skip compression for messages below the configured floor — small payloads often + // grow rather than shrink when compressed. + if (Buffer.byteLength(json, 'utf8') < this.skipCompressionBelow) { + return { payload: message, preBuiltBody: json } + } + + // Identity/routing fields are copied as plaintext envelope siblings so broker-side + // filtering (e.g. SNS body-scoped FilterPolicy) keeps working on compressed messages — + // mirroring how offloaded-payload pointers preserve these fields. + const compressed = await handler.compress(Buffer.from(json, 'utf8')) + return { + payload: message, + preBuiltBody: buildCodecEnvelope(compressed, codecName, this.collectPreservedFields(message)), + } + } + /** * Estimates the wire size in bytes of the codec envelope wrapping `compressedSize` - * compressed bytes. The envelope is `{"__mqtCodec":"","__mqtData":""}`: - * base64 expands the payload to `⌈N/3⌉×4`, and the fixed JSON framing adds 32 chars - * plus the codec name length. + * compressed bytes. The envelope is `{...preservedFields,"__mqtCodec":"", + * "__mqtData":""}`: base64 expands the payload to `⌈N/3⌉×4`, the fixed JSON + * framing adds 32 chars plus the codec name length, and any preserved sibling fields + * add their serialized length. + * + * Note: this measures the envelope body only — transport-specific message attributes + * (small, and identical with or without codec) are not included. */ - protected estimateCodecEnvelopeSize(compressedSize: number, codecName: string): number { - return Math.ceil(compressedSize / 3) * 4 + 32 + codecName.length + protected estimateCodecEnvelopeSize( + compressedSize: number, + codecName: string, + preservedFields?: Record, + ): number { + let size = Math.ceil(compressedSize / 3) * 4 + 32 + codecName.length + if (preservedFields) { + const serialized = JSON.stringify(preservedFields) + // Merged into the envelope, the preserved fields cost their serialized content + // minus the two outer braces, plus one joining comma. + if (serialized.length > 2) { + size += Buffer.byteLength(serialized, 'utf8') - 1 + } + } + return size } /** @@ -862,100 +903,157 @@ export abstract class AbstractQueueService< } /** - * Compress-and-offload path used when both `codec` and `payloadStoreConfig` are set. - * The caller (`prepareOutgoingPayload`) has already applied `skipCompressionBelow`, so - * this method always compresses. + * Store-path handler for codec publishers (both `codec` and `payloadStoreConfig` set). * - * **In-memory fast path (string payloads):** when the serializer produces a string and its - * byte length is below `messageSizeThreshold`, the payload is compressed directly into a - * Buffer in memory — no temp file is created, no disk I/O occurs. + * Serializes the message **once**, through the payload store's serializer (which may + * stream large payloads). `skipCompressionBelow` is evaluated against the size the + * serializer reports — there is no separate `JSON.stringify`, so a streaming serializer + * is never forced to fully materialize the payload just to evaluate the floor. * - * **Streaming path (stream payloads or large strings):** serializes the payload once, - * pipes it through the codec Transform into a temp file. - * - * Either way, the offload decision is made against the **codec envelope wire size** - * (base64-encoded compressed bytes + JSON framing), not the raw compressed byte count: - * compression does not always shrink data, so the base64 envelope can exceed - * `messageSizeThreshold` even when the raw payload did not. + * - **Below `skipCompressionBelow`:** compression is skipped — the payload is sent inline + * as plain JSON, or offloaded uncompressed if it still exceeds `messageSizeThreshold` + * (possible only when the threshold is configured below the floor). + * - **Otherwise:** the payload is compressed and either inlined as a codec envelope or + * offloaded as raw compressed bytes (see {@link compressSerializedPayload}). * - * @returns - * - `{ pointer }` — compressed payload was offloaded; use pointer as the message payload - * - `{ compressedBuffer }` — compressed payload fits inline; caller builds the codec envelope + * Returns the same `{ payload, preBuiltBody? }` shape as {@link prepareOutgoingPayload}. */ protected async compressAndOffloadPayload( message: MessagePayloadSchemas, handler: MessageCodecHandler, codecName: string, - ): Promise< - | { pointer: OffloadedPayloadPointerPayload; compressedBuffer?: never } - | { compressedBuffer: Buffer; pointer?: never } - > { + ): Promise<{ + payload: MessagePayloadSchemas | OffloadedPayloadPointerPayload + preBuiltBody?: string + }> { if (!this.payloadStoreConfig) { throw new Error('Payload store is not configured') } const threshold = this.payloadStoreConfig.messageSizeThreshold + // Serialize once. `serialized.size` is the authoritative wire size — used to honor + // `skipCompressionBelow` without a redundant `JSON.stringify` of the whole message. const serialized = await this.payloadStoreConfig.serializer.serialize(message) try { - // In-memory fast path: avoid disk I/O entirely for small string payloads. - if ( - typeof serialized.value === 'string' && - Buffer.byteLength(serialized.value, 'utf8') < threshold - ) { - const compressed = await handler.compress(Buffer.from(serialized.value, 'utf8')) - // The wire body is a base64 codec envelope, which can exceed the threshold even - // when the raw payload did not (compression does not always shrink). Re-check the - // envelope size and offload the compressed bytes if it no longer fits inline. - if (this.estimateCodecEnvelopeSize(compressed.length, codecName) > threshold) { + // Below the compression floor: skip compression entirely. The payload is offloaded + // uncompressed if it still exceeds the threshold (only possible when the threshold + // is set below the floor), otherwise sent inline as plain JSON. + if (serialized.size < this.skipCompressionBelow) { + if (serialized.size > threshold) { const { store, storeName } = this.resolveOutgoingStore() - const payloadId = await store.storePayload({ - value: Readable.from(compressed), - size: compressed.length, - }) - return { - pointer: this.buildPointer(message, payloadId, storeName, compressed.length, codecName), - } + const payloadId = await store.storePayload(serialized) + return { payload: this.buildPointer(message, payloadId, storeName, serialized.size) } + } + return { + payload: message, + preBuiltBody: + typeof serialized.value === 'string' + ? serialized.value + : await streamWithKnownSizeToString(serialized.value, serialized.size), } - return { compressedBuffer: compressed } } - // Streaming pipeline: serializer output → codec transform → temp file. - // No full-payload buffer is materialised; each codec supplies its own Transform. - const tmpPath = path.join(os.tmpdir(), randomUUID()) - try { - await pipeline( - typeof serialized.value === 'string' ? Readable.from(serialized.value) : serialized.value, - handler.createCompressStream(), - fs.createWriteStream(tmpPath), - ) + return await this.compressSerializedPayload( + message, + serialized, + handler, + codecName, + threshold, + ) + } finally { + if (isDestroyable(serialized)) { + await serialized.destroy() + } + } + } - const compressedSize = (await fs.promises.stat(tmpPath)).size + /** + * Compresses an already-serialized payload and decides — against the **codec envelope + * wire size** (base64-encoded compressed bytes + JSON framing), not the raw compressed + * byte count — whether to send it inline as a codec envelope or offload the compressed + * bytes. Compression does not always shrink data, so the base64 envelope can exceed + * `messageSizeThreshold` even when the raw payload did not. + * + * - **In-memory fast path:** a string payload below `messageSizeThreshold` is compressed + * directly into a Buffer — no temp file is created, no disk I/O occurs. + * - **Streaming path:** a stream payload (or large string) is piped through the codec + * Transform into a temp file, so no full-payload buffer is materialized. + * + * Split out of {@link compressAndOffloadPayload} for the cognitive-complexity budget. + */ + private async compressSerializedPayload( + message: MessagePayloadSchemas, + serialized: SerializedPayload, + handler: MessageCodecHandler, + codecName: string, + threshold: number, + ): Promise<{ + payload: MessagePayloadSchemas | OffloadedPayloadPointerPayload + preBuiltBody?: string + }> { + // Identity/routing fields are copied as plaintext envelope siblings so broker-side + // filtering (e.g. SNS body-scoped FilterPolicy) keeps working on compressed messages. + const preservedFields = this.collectPreservedFields(message) - // Compare the envelope wire size (not raw compressed bytes) against the threshold. - if (this.estimateCodecEnvelopeSize(compressedSize, codecName) > threshold) { - const { store, storeName } = this.resolveOutgoingStore() - const payloadId = await store.storePayload({ - value: fs.createReadStream(tmpPath), - size: compressedSize, - }) - return { - pointer: this.buildPointer(message, payloadId, storeName, compressedSize, codecName), - } + // In-memory fast path: avoid disk I/O entirely for small string payloads. + if (typeof serialized.value === 'string' && serialized.size < threshold) { + const compressed = await handler.compress(Buffer.from(serialized.value, 'utf8')) + if ( + this.estimateCodecEnvelopeSize(compressed.length, codecName, preservedFields) > threshold + ) { + const { store, storeName } = this.resolveOutgoingStore() + const payloadId = await store.storePayload({ + value: Readable.from(compressed), + size: compressed.length, + }) + return { + payload: this.buildPointer(message, payloadId, storeName, compressed.length, codecName), } + } + return { + payload: message, + preBuiltBody: buildCodecEnvelope(compressed, codecName, preservedFields), + } + } + + // Streaming pipeline: serializer output → codec transform → temp file. + // No full-payload buffer is materialised; each codec supplies its own Transform. + const tmpPath = path.join(os.tmpdir(), randomUUID()) + try { + await pipeline( + typeof serialized.value === 'string' ? Readable.from(serialized.value) : serialized.value, + handler.createCompressStream(), + fs.createWriteStream(tmpPath), + ) + + const compressedSize = (await fs.promises.stat(tmpPath)).size - // Compressed payload fits inline — return the buffer; caller wraps it in a codec envelope. - return { compressedBuffer: await fs.promises.readFile(tmpPath) } - } finally { - try { - await fs.promises.unlink(tmpPath) - } catch { - // ignore cleanup errors + if (this.estimateCodecEnvelopeSize(compressedSize, codecName, preservedFields) > threshold) { + const { store, storeName } = this.resolveOutgoingStore() + const payloadId = await store.storePayload({ + value: fs.createReadStream(tmpPath), + size: compressedSize, + }) + return { + payload: this.buildPointer(message, payloadId, storeName, compressedSize, codecName), } } + + // Compressed payload fits inline — wrap the buffer in a codec envelope. + return { + payload: message, + preBuiltBody: buildCodecEnvelope( + await fs.promises.readFile(tmpPath), + codecName, + preservedFields, + ), + } } finally { - if (isDestroyable(serialized)) { - await serialized.destroy() + try { + await fs.promises.unlink(tmpPath) + } catch { + // ignore cleanup errors } } } diff --git a/packages/core/package.json b/packages/core/package.json index 041db97b..bd8aec0d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/core", - "version": "25.5.0", + "version": "26.0.0", "license": "MIT", "description": "Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently", "maintainers": [ diff --git a/packages/core/test/codec/codecHandler.spec.ts b/packages/core/test/codec/codecHandler.spec.ts new file mode 100644 index 00000000..ad09b136 --- /dev/null +++ b/packages/core/test/codec/codecHandler.spec.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from 'vitest' + +import { buildCodecEnvelope } from '../../lib/codec/codecHandler.ts' +import { hasCodecEnvelopeShape, isCodecEnvelope } from '../../lib/codec/messageCodec.ts' + +const COMPRESSED = Buffer.from('compressed-bytes') +const COMPRESSED_B64 = COMPRESSED.toString('base64') + +describe('buildCodecEnvelope', () => { + it('emits a two-field envelope when no preserved fields are given', () => { + expect(JSON.parse(buildCodecEnvelope(COMPRESSED, 'zstd'))).toEqual({ + __mqtCodec: 'zstd', + __mqtData: COMPRESSED_B64, + }) + }) + + it('emits a two-field envelope when preserved fields are an empty object', () => { + expect(JSON.parse(buildCodecEnvelope(COMPRESSED, 'zstd', {}))).toEqual({ + __mqtCodec: 'zstd', + __mqtData: COMPRESSED_B64, + }) + }) + + it('emits preserved fields as plaintext siblings of the codec fields', () => { + const envelope = JSON.parse( + buildCodecEnvelope(COMPRESSED, 'zstd', { id: 'm-1', type: 'permissions.add' }), + ) + expect(envelope).toEqual({ + id: 'm-1', + type: 'permissions.add', + __mqtCodec: 'zstd', + __mqtData: COMPRESSED_B64, + }) + }) + + it('escapes preserved field values that contain JSON metacharacters', () => { + const envelope = JSON.parse( + buildCodecEnvelope(COMPRESSED, 'zstd', { id: 'quote " and \\ backslash' }), + ) + expect(envelope.id).toBe('quote " and \\ backslash') + expect(envelope.__mqtData).toBe(COMPRESSED_B64) + }) + + it('codec fields always win over a colliding preserved field name', () => { + const envelope = JSON.parse( + buildCodecEnvelope(COMPRESSED, 'zstd', { __mqtData: 'tampered', __mqtCodec: 'tampered' }), + ) + expect(envelope.__mqtData).toBe(COMPRESSED_B64) + expect(envelope.__mqtCodec).toBe('zstd') + }) + + it('produces an envelope still recognised by isCodecEnvelope and hasCodecEnvelopeShape', () => { + const envelope = JSON.parse( + buildCodecEnvelope(COMPRESSED, 'zstd', { id: 'm-1', type: 'permissions.add' }), + ) + expect(isCodecEnvelope(envelope)).toBe(true) + expect(hasCodecEnvelopeShape(envelope)).toBe(true) + }) +}) diff --git a/packages/core/test/codec/messageCodec.spec.ts b/packages/core/test/codec/messageCodec.spec.ts index c383ffba..958bec0c 100644 --- a/packages/core/test/codec/messageCodec.spec.ts +++ b/packages/core/test/codec/messageCodec.spec.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from 'vitest' -import { isCodecEnvelope, MessageCodecEnum } from '../../lib/codec/messageCodec.ts' +import { + hasCodecEnvelopeShape, + isCodecEnvelope, + MessageCodecEnum, +} from '../../lib/codec/messageCodec.ts' const VALID_BASE64 = Buffer.from('hello compressed world').toString('base64') @@ -43,15 +47,21 @@ describe('isCodecEnvelope', () => { }) }) - describe('extra fields — real messages must not be misclassified', () => { - it('returns false when envelope has an extra field alongside __mqtCodec and __mqtData', () => { + describe('preserved sibling fields — detection is presence-based', () => { + it('returns true when the envelope carries preserved sibling fields (id, type, …)', () => { + // Publishers copy identity/routing fields alongside the codec fields so broker-side + // filtering (e.g. SNS body-scoped FilterPolicy) keeps working on compressed + // messages. Detection is presence-based and must accept these extra siblings, + // mirroring how offloaded-payload pointers are detected by marker-field presence. expect( isCodecEnvelope({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: VALID_BASE64, id: 'real-message', + type: 'permissions.add', + timestamp: '2026-05-22T00:00:00.000Z', }), - ).toBe(false) + ).toBe(true) }) it('returns false when only __mqtCodec is present (no __mqtData)', () => { @@ -108,3 +118,41 @@ describe('isCodecEnvelope', () => { }) }) }) + +describe('hasCodecEnvelopeShape', () => { + it('returns true for an envelope naming a codec unknown to this consumer', () => { + // Structural check — unlike isCodecEnvelope it does not consult a knownCodecs set, + // so an envelope for an unregistered codec is still recognised (and can be surfaced + // as a misconfiguration rather than processed as an incomplete message). + expect(hasCodecEnvelopeShape({ __mqtCodec: 'lz4', __mqtData: VALID_BASE64 })).toBe(true) + }) + + it('returns true for an envelope carrying preserved sibling fields', () => { + expect( + hasCodecEnvelopeShape({ + __mqtCodec: MessageCodecEnum.ZSTD, + __mqtData: VALID_BASE64, + id: 'm-1', + type: 'permissions.add', + }), + ).toBe(true) + }) + + it('returns false when __mqtCodec is an empty string', () => { + expect(hasCodecEnvelopeShape({ __mqtCodec: '', __mqtData: VALID_BASE64 })).toBe(false) + }) + + it('returns false when a marker field is missing', () => { + expect(hasCodecEnvelopeShape({ __mqtCodec: 'lz4' })).toBe(false) + expect(hasCodecEnvelopeShape({ __mqtData: VALID_BASE64 })).toBe(false) + }) + + it('returns false when __mqtData is not valid base64', () => { + expect(hasCodecEnvelopeShape({ __mqtCodec: 'lz4', __mqtData: 'not base64!!!' })).toBe(false) + }) + + it('returns false for non-object inputs', () => { + expect(hasCodecEnvelopeShape(null)).toBe(false) + expect(hasCodecEnvelopeShape('a string')).toBe(false) + }) +}) diff --git a/packages/gcp-pubsub/package.json b/packages/gcp-pubsub/package.json index b43d284d..6478656e 100644 --- a/packages/gcp-pubsub/package.json +++ b/packages/gcp-pubsub/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/gcp-pubsub", - "version": "2.2.0", + "version": "3.0.0", "license": "MIT", "description": "Google Cloud Pub/Sub adapter for message-queue-toolkit", "maintainers": [ @@ -32,7 +32,7 @@ }, "peerDependencies": { "@google-cloud/pubsub": "^5.2.0", - "@message-queue-toolkit/core": ">=25.0.0", + "@message-queue-toolkit/core": ">=26.0.0", "zod": ">=3.25.76 <5.0.0" }, "devDependencies": { diff --git a/packages/sns/lib/sns/AbstractSnsPublisher.ts b/packages/sns/lib/sns/AbstractSnsPublisher.ts index c040e09c..910954fd 100644 --- a/packages/sns/lib/sns/AbstractSnsPublisher.ts +++ b/packages/sns/lib/sns/AbstractSnsPublisher.ts @@ -125,17 +125,10 @@ export abstract class AbstractSnsPublisher const topicName = this.locatorConfig?.topicName ?? this.creationConfig?.topic?.Name ?? 'unknown' - // Dedup check runs before compression/offload so duplicates skip that expensive work - // entirely and never leave an orphaned object in the payload store. Trade-off: the - // publisher dedup key is persisted before the message is actually sent, so a crash - // between here and `sendMessage` drops the message on a subsequent retry. This is the - // same window that already existed around the bare `sendMessage` call; it now also - // spans compression/offload. - if ( - this.isDeduplicationEnabledForMessage(parsedMessage) && - (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) - .isDuplicated - ) { + // Fast read-only pre-check: skip compression/offload for messages already known to + // be duplicates. This does NOT persist a dedup key, so a transient failure in the + // expensive work below leaves no key behind and the publish stays safely retriable. + if (await this.isMessageDuplicated(parsedMessage, DeduplicationRequesterEnum.Publisher)) { this.handleMessageProcessed({ message: parsedMessage, processingResult: { status: 'published', skippedAsDuplicate: true }, @@ -153,6 +146,25 @@ export abstract class AbstractSnsPublisher const { payload, preBuiltBody } = await this.prepareOutgoingPayload(updatedMessage) + // Persist the dedup key only now — immediately before send — so the window in which + // a stored key plus a failed send could drop the message on retry stays as small as + // possible and no longer spans compression/offload (a transient failure there leaves + // no key behind). The pre-check above already skipped the expensive work for the + // common duplicate case; this check additionally closes the race where a concurrent + // publish stored the key in the meantime. + if ( + (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) + .isDuplicated + ) { + this.handleMessageProcessed({ + message: parsedMessage, + processingResult: { status: 'published', skippedAsDuplicate: true }, + messageProcessingStartTimestamp, + queueName: topicName, + }) + return + } + await this.sendMessage(payload, resolvedOptions, preBuiltBody) this.handleMessageProcessed({ diff --git a/packages/sns/package.json b/packages/sns/package.json index 2736e230..37224df1 100644 --- a/packages/sns/package.json +++ b/packages/sns/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sns", - "version": "24.7.0", + "version": "25.0.0", "private": false, "license": "MIT", "description": "SNS adapter for message-queue-toolkit", @@ -38,7 +38,7 @@ "@aws-sdk/client-sns": "^3.632.0", "@aws-sdk/client-sqs": "^3.1034.0", "@aws-sdk/client-sts": "^3.632.0", - "@message-queue-toolkit/core": ">=25.5.0", + "@message-queue-toolkit/core": ">=26.0.0", "@message-queue-toolkit/schemas": ">=7.0.0", "@message-queue-toolkit/sqs": ">=23.0.0", "zod": ">=3.25.76 <5.0.0" diff --git a/packages/sqs/README.md b/packages/sqs/README.md index 28a0bd8b..726cb2a2 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -819,7 +819,9 @@ Compress message bodies with zstd using the Node.js built-in `zlib` module. Requ The codec implementation ships inside `@message-queue-toolkit/core` — no extra package to install. Compression is opt-in: it is only active when you set the `codec` option on a publisher. -Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __mqtCodec: 'zstd', __mqtData: '' }`), so a consumer without `codec` set will still decompress automatically via envelope detection. This allows a gradual rollout — enable compression on the publisher first, consumers adapt without configuration changes. +Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __mqtCodec: 'zstd', __mqtData: '', ...preserved fields }`), so a consumer without `codec` set will still decompress automatically via envelope detection. + +> **Roll out consumers before publishers.** Auto-detection only works on a consumer running a library version that supports the codec. Upgrade and deploy all consumers of a queue **first** (they keep handling plain messages unchanged), and only then enable `codec` on publishers. A publisher emitting compressed messages to a consumer on an older library version — or to a consumer missing a required custom codec — will cause those messages to fail processing. #### Publisher @@ -862,6 +864,8 @@ class MyConsumer extends AbstractSqsConsumer - /** Precomputed set of codec names in the registry, passed to isCodecEnvelope. */ - private readonly codecKnownNames: ReadonlySet protected deadLetterQueueUrl?: string protected readonly errorResolver: ErrorResolver @@ -319,7 +317,6 @@ export abstract class AbstractSqsConsumer< registry.set(getCodecName(registration), resolveCodecHandler(registration)) } this.codecRegistry = registry - this.codecKnownNames = new Set(registry.keys()) } override async init(): Promise { @@ -986,12 +983,24 @@ export abstract class AbstractSqsConsumer< resolveMessageResult.result.body = retrieveOffloadedMessagePayloadResult.result } else if ( !this.disableCodecAutoDetection && - isCodecEnvelope(resolveMessageResult.result.body, this.codecKnownNames) + hasCodecEnvelopeShape(resolveMessageResult.result.body) ) { + const envelope = resolveMessageResult.result.body + const handler = this.codecRegistry.get(envelope.__mqtCodec) + if (!handler) { + // Envelope-shaped body naming a codec this consumer has not registered. This is a + // deployment misconfiguration, not a poison message — surface it as an error + // rather than letting the envelope's preserved sibling fields (id, type, …) + // satisfy a lenient schema and be processed as an incomplete message. Register + // the codec via the `codecs` option. + this.handleError( + new Error( + `Received a message compressed with codec "${envelope.__mqtCodec}", which is not registered on this consumer. Register it via the \`codecs\` option.`, + ), + ) + return ABORT_EARLY_EITHER + } try { - const envelope = resolveMessageResult.result.body - // handler is guaranteed non-null: isCodecEnvelope already verified __mqtCodec ∈ codecKnownNames === codecRegistry.keys() - const handler = this.codecRegistry.get(envelope.__mqtCodec)! const compressed = Buffer.from(envelope.__mqtData, 'base64') resolveMessageResult.result.body = JSON.parse( (await handler.decompress(compressed)).toString('utf8'), diff --git a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts index ebc00365..34ea0ed9 100644 --- a/packages/sqs/lib/sqs/AbstractSqsPublisher.ts +++ b/packages/sqs/lib/sqs/AbstractSqsPublisher.ts @@ -117,17 +117,10 @@ export abstract class AbstractSqsPublisher const messageProcessingStartTimestamp = Date.now() const parsedMessage = messageSchemaResult.result.parse(message) - // Dedup check runs before compression/offload so duplicates skip that expensive work - // entirely and never leave an orphaned object in the payload store. Trade-off: the - // publisher dedup key is persisted before the message is actually sent, so a crash - // between here and `sendMessage` drops the message on a subsequent retry. This is the - // same window that already existed around the bare `sendMessage` call; it now also - // spans compression/offload. - if ( - this.isDeduplicationEnabledForMessage(parsedMessage) && - (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) - .isDuplicated - ) { + // Fast read-only pre-check: skip compression/offload for messages already known to + // be duplicates. This does NOT persist a dedup key, so a transient failure in the + // expensive work below leaves no key behind and the publish stays safely retriable. + if (await this.isMessageDuplicated(parsedMessage, DeduplicationRequesterEnum.Publisher)) { this.handleMessageProcessed({ message: parsedMessage, processingResult: { status: 'published', skippedAsDuplicate: true }, @@ -145,6 +138,25 @@ export abstract class AbstractSqsPublisher const { payload, preBuiltBody } = await this.prepareOutgoingPayload(message) + // Persist the dedup key only now — immediately before send — so the window in which + // a stored key plus a failed send could drop the message on retry stays as small as + // possible and no longer spans compression/offload (a transient failure there leaves + // no key behind). The pre-check above already skipped the expensive work for the + // common duplicate case; this check additionally closes the race where a concurrent + // publish stored the key in the meantime. + if ( + (await this.deduplicateMessage(parsedMessage, DeduplicationRequesterEnum.Publisher)) + .isDuplicated + ) { + this.handleMessageProcessed({ + message: parsedMessage, + processingResult: { status: 'published', skippedAsDuplicate: true }, + messageProcessingStartTimestamp, + queueName: this.queueName, + }) + return + } + await this.sendMessage(payload, resolvedOptions, preBuiltBody) this.handleMessageProcessed({ message: parsedMessage, diff --git a/packages/sqs/package.json b/packages/sqs/package.json index a93374a7..e8cb8200 100644 --- a/packages/sqs/package.json +++ b/packages/sqs/package.json @@ -1,6 +1,6 @@ { "name": "@message-queue-toolkit/sqs", - "version": "24.3.0", + "version": "25.0.0", "private": false, "license": "MIT", "description": "SQS adapter for message-queue-toolkit", @@ -37,7 +37,7 @@ }, "peerDependencies": { "@aws-sdk/client-sqs": "^3.1034.0", - "@message-queue-toolkit/core": ">=25.5.0", + "@message-queue-toolkit/core": ">=26.0.0", "zod": ">=3.25.76 <5.0.0" }, "devDependencies": { diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index 84edea10..df419c5e 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -621,11 +621,13 @@ describe('SqsPermissionConsumer - custom codec registration', () => { await consumer.close(true) }) - it('consumer without the custom codec does not auto-detect envelopes with that codec name', async () => { - // A consumer without extra codecs has codecKnownNames = Set(['zstd']) (built-ins only). - // isCodecEnvelope(body, Set(['zstd'])) returns false for a 'noop' envelope, so the - // raw envelope object reaches schema validation and fails (no messageType field). - // The message is never successfully handled — addCounter stays at 0. + it('consumer without the custom codec rejects envelopes with that codec name', async () => { + // A consumer without extra codecs registers only the built-in codecs (e.g. zstd), not + // 'noop'. An envelope-shaped body naming an unregistered codec is a misconfiguration: + // the consumer surfaces it as an error rather than letting the envelope's preserved + // sibling fields (id, messageType) satisfy the schema and be processed as an + // incomplete message. So an error is recorded and the message is never handled — + // addCounter stays at 0. const queueName = `${SqsPermissionConsumer.QUEUE_NAME}-custom-codec-no-autodetect` await testAdmin.deleteQueues(queueName) @@ -651,9 +653,10 @@ describe('SqsPermissionConsumer - custom codec registration', () => { } await publisher.publish(message) - // Give the consumer time to attempt processing, then verify no message was consumed. - // The spy can't track by ID because the raw envelope has no top-level `id` field. - await new Promise((resolve) => setTimeout(resolve, 2000)) + // The unregistered-codec envelope is rejected as an error; wait for that error to be + // recorded, then confirm the handler never ran (no incomplete message was processed). + await waitAndRetry(() => zstdConsumer.handlerSpy.counts.error > 0, 100, 20) + expect(zstdConsumer.handlerSpy.counts.error).toBeGreaterThan(0) expect(zstdConsumer.addCounter).toBe(0) await publisher.close() From b3ae7c47b575b1ae95f6eec757835a20e208f186 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 22 May 2026 17:33:50 +0300 Subject: [PATCH 22/23] =?UTF-8?q?fix(codec):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20retriable=20missing-codec,=20drop=20redundant=20hel?= =?UTF-8?q?pers,=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 — An inline message whose `__mqtCodec` named an unregistered codec was aborted (poison treatment) and silently dropped, while the offloaded-payload path treated the same misconfiguration as retriable. The inline path now throws, so the message stays on the queue until the codec is registered — consistent with retrieveOffloadedMessagePayload. P2 — Documented the deliberate buffer-based decompression (no createDecompressStream: the consumer must JSON.parse the whole payload anyway) on MessageCodecHandler and in the SQS README. Corrected the stale `disableCodecAutoDetection` JSDoc (detection is presence-based via hasCodecEnvelopeShape, not an exact two-field match) and warned that, with it enabled, a compressed message can pass a lenient schema as an incomplete one. P3 — Removed compressMessageBody / decompressMessageBody: test-only helpers that leaked into the public API, duplicated the exported buildCodecEnvelope / resolveCodecHandler primitives, and were footguns (no preserved routing fields; built-in codecs only). Tests now build envelopes from the primitives. Clarified the 256 KB transport-limit wording, made estimateCodecEnvelopeSize measure UTF-8 bytes consistently, and removed a stray packages/sqs/NUL file. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/core/lib/codec/codecHandler.ts | 38 +---------------- packages/core/lib/codec/messageCodec.ts | 15 ++++++- packages/core/lib/index.ts | 2 - .../core/lib/queues/AbstractQueueService.ts | 9 ++-- packages/sqs/README.md | 5 ++- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 42 ++++++++++++------- packages/sqs/test/codec/codecHandler.spec.ts | 18 +------- .../SqsPermissionConsumer.codec.spec.ts | 21 ++++++++-- 8 files changed, 71 insertions(+), 79 deletions(-) diff --git a/packages/core/lib/codec/codecHandler.ts b/packages/core/lib/codec/codecHandler.ts index cbc74f4d..f1515267 100644 --- a/packages/core/lib/codec/codecHandler.ts +++ b/packages/core/lib/codec/codecHandler.ts @@ -1,12 +1,8 @@ import type { Transform } from 'node:stream' import { promisify } from 'node:util' import zlib from 'node:zlib' -import type { - CodecEnvelope, - MessageCodecHandler, - MessageCodecRegistration, -} from './messageCodec.ts' -import { BASE64_RE, MessageCodecEnum } from './messageCodec.ts' +import type { MessageCodecHandler, MessageCodecRegistration } from './messageCodec.ts' +import { MessageCodecEnum } from './messageCodec.ts' const ZSTD_UNSUPPORTED_MSG = 'zlib.zstdCompress and zlib.zstdDecompress are not available in this Node.js version. ' + @@ -99,15 +95,6 @@ export function resolveCodecHandler(codec: MessageCodecRegistration): MessageCod throw new Error(`Unsupported codec: ${codec}`) } -export async function compressMessageBody( - jsonBody: string, - codec: MessageCodecRegistration, -): Promise { - const handler = resolveCodecHandler(codec) - const compressed = await handler.compress(Buffer.from(jsonBody, 'utf8')) - return buildCodecEnvelope(compressed, getCodecName(codec)) -} - /** * Wraps an already-compressed buffer in a codec envelope string. * Use this when you have pre-compressed bytes and want to avoid compressing twice. @@ -140,24 +127,3 @@ export function buildCodecEnvelope( // Codec fields are listed last so they always win over any colliding preserved key. return JSON.stringify({ ...preservedFields, __mqtCodec: codecName, __mqtData: data }) } - -/** - * Decompresses a codec envelope produced by {@link compressMessageBody} or - * {@link buildCodecEnvelope} and returns the original parsed JSON value. - * - * **Built-in codecs only.** This utility resolves the handler via - * {@link resolveCodecHandler}, which only recognises built-in codec names - * (e.g. `MessageCodecEnum.ZSTD`). Calling it with a custom-codec envelope - * (where `__mqtCodec` is a user-chosen name) will throw "Unsupported codec". - * Consumer-side decoding of custom codecs is handled automatically via the - * consumer's codec registry; this function is intended for one-off, built-in use cases. - */ -export async function decompressMessageBody(envelope: CodecEnvelope): Promise { - if (!BASE64_RE.test(envelope.__mqtData)) { - throw new Error(`Codec envelope __mqtData is not valid base64 (codec: ${envelope.__mqtCodec})`) - } - const handler = resolveCodecHandler(envelope.__mqtCodec as MessageCodecRegistration) - const compressed = Buffer.from(envelope.__mqtData, 'base64') - const decompressed = await handler.decompress(compressed) - return JSON.parse(decompressed.toString('utf8')) -} diff --git a/packages/core/lib/codec/messageCodec.ts b/packages/core/lib/codec/messageCodec.ts index 389c2803..7d22c449 100644 --- a/packages/core/lib/codec/messageCodec.ts +++ b/packages/core/lib/codec/messageCodec.ts @@ -33,13 +33,26 @@ export type CodecEnvelope = { * `@message-queue-toolkit/core`) uses Node.js built-in `zlib` zstd support. * * All three methods are required: - * - `compress` / `decompress` are used for the inline (non-offloaded) publish path. + * - `compress` / `decompress` are used for the inline (non-offloaded) path on both sides. * - `createCompressStream` is used by the streaming offload path to pipe serialized * JSON directly through compression into the payload store without buffering the * full payload in memory. + * + * Note the deliberate asymmetry: there is no `createDecompressStream`. Decompression is + * always buffer-based (`decompress`) because the consumer must `JSON.parse` the whole + * payload anyway — a streaming decompressor could not avoid materializing it. For an + * offloaded compressed payload this means the compressed bytes and the decompressed + * payload are both briefly resident in memory; bound the worst case with the + * `maxDecompressedBytes` argument of {@link ZstdCodecHandler} (or the equivalent in a + * custom handler). */ export interface MessageCodecHandler { compress(data: Buffer): Promise + /** + * Decompresses a full compressed buffer. Buffer-based by design (see the interface + * note above); a custom implementation should cap the decompressed size to guard + * against decompression bombs. + */ decompress(data: Buffer): Promise /** Returns a Transform stream that compresses its input using this codec. */ createCompressStream(): import('node:stream').Transform diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts index b6b11bfd..d9d1b52b 100644 --- a/packages/core/lib/index.ts +++ b/packages/core/lib/index.ts @@ -1,8 +1,6 @@ export { buildCodecEnvelope, - compressMessageBody, DEFAULT_MAX_DECOMPRESSED_BYTES, - decompressMessageBody, getCodecName, resolveCodecHandler, ZstdCodecHandler, diff --git a/packages/core/lib/queues/AbstractQueueService.ts b/packages/core/lib/queues/AbstractQueueService.ts index 2b98de6f..b88c220f 100644 --- a/packages/core/lib/queues/AbstractQueueService.ts +++ b/packages/core/lib/queues/AbstractQueueService.ts @@ -880,11 +880,12 @@ export abstract class AbstractQueueService< ): number { let size = Math.ceil(compressedSize / 3) * 4 + 32 + codecName.length if (preservedFields) { - const serialized = JSON.stringify(preservedFields) // Merged into the envelope, the preserved fields cost their serialized content - // minus the two outer braces, plus one joining comma. - if (serialized.length > 2) { - size += Buffer.byteLength(serialized, 'utf8') - 1 + // minus the two outer braces ("{}"), plus one joining comma. An empty object + // ("{}", 2 bytes) contributes nothing. Measured in UTF-8 bytes throughout. + const serializedBytes = Buffer.byteLength(JSON.stringify(preservedFields), 'utf8') + if (serializedBytes > 2) { + size += serializedBytes - 1 } } return size diff --git a/packages/sqs/README.md b/packages/sqs/README.md index 726cb2a2..6cfb7d09 100644 --- a/packages/sqs/README.md +++ b/packages/sqs/README.md @@ -821,7 +821,7 @@ The codec implementation ships inside `@message-queue-toolkit/core` — no extra Compressed messages are **self-describing**: the codec is embedded in the message envelope (`{ __mqtCodec: 'zstd', __mqtData: '', ...preserved fields }`), so a consumer without `codec` set will still decompress automatically via envelope detection. -> **Roll out consumers before publishers.** Auto-detection only works on a consumer running a library version that supports the codec. Upgrade and deploy all consumers of a queue **first** (they keep handling plain messages unchanged), and only then enable `codec` on publishers. A publisher emitting compressed messages to a consumer on an older library version — or to a consumer missing a required custom codec — will cause those messages to fail processing. +> **Roll out consumers before publishers.** Auto-detection only works on a consumer running a library version that supports the codec. Upgrade and deploy all consumers of a queue **first** (they keep handling plain messages unchanged), and only then enable `codec` on publishers. A publisher emitting compressed messages to a consumer on an older library version — or to a consumer missing a required custom codec — will fail to process those messages. Such a missing-codec failure is treated as a **retriable** error (a misconfiguration, not a poison message): the message stays on the queue and is retried until the codec is registered, rather than being dropped or sent to the DLQ. This holds for both inline and offloaded compressed messages. #### Publisher @@ -861,7 +861,8 @@ class MyConsumer extends AbstractSqsConsumer { @@ -33,19 +33,3 @@ describe('getCodecName', () => { expect(() => getCodecName({ name: '', handler: {} as any })).toThrow('Invalid codec name ""') }) }) - -describe('decompressMessageBody', () => { - it('throws a descriptive error when __mqtData is not valid base64', async () => { - await expect( - decompressMessageBody({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: 'not-base64!!!' }), - ).rejects.toThrow('Codec envelope __mqtData is not valid base64 (codec: zstd)') - }) - - it('throws a descriptive error for base64 with incorrect padding', async () => { - // Valid base64 characters but wrong padding — Buffer.from would silently accept this - // and produce garbage bytes; the guard must catch it before the codec is invoked. - await expect( - decompressMessageBody({ __mqtCodec: MessageCodecEnum.ZSTD, __mqtData: 'abc' }), - ).rejects.toThrow('Codec envelope __mqtData is not valid base64 (codec: zstd)') - }) -}) diff --git a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts index df419c5e..56b428f6 100644 --- a/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts +++ b/packages/sqs/test/consumers/SqsPermissionConsumer.codec.spec.ts @@ -3,7 +3,11 @@ import { PassThrough } from 'node:stream' import { ReceiveMessageCommand, SendMessageCommand } from '@aws-sdk/client-sqs' import { waitAndRetry } from '@lokalise/node-core' import type { MessageCodecHandler } from '@message-queue-toolkit/core' -import { compressMessageBody, MessageCodecEnum } from '@message-queue-toolkit/core' +import { + buildCodecEnvelope, + MessageCodecEnum, + resolveCodecHandler, +} from '@message-queue-toolkit/core' import type { AwilixContainer } from 'awilix' import { asValue } from 'awilix' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' @@ -21,6 +25,17 @@ import type { PERMISSIONS_ADD_MESSAGE_TYPE } from './userConsumerSchemas.ts' // message back under the threshold. const LARGE_PADDING = 'x'.repeat(800) +/** + * Builds a raw zstd codec envelope, simulating an external (non-mqt) producer that + * compressed the message itself. Deliberately omits the preserved sibling fields the + * built-in publisher adds — these tests exercise the bare-envelope path. + */ +async function compressToZstdEnvelope(message: unknown): Promise { + const handler = resolveCodecHandler(MessageCodecEnum.ZSTD) + const compressed = await handler.compress(Buffer.from(JSON.stringify(message), 'utf8')) + return buildCodecEnvelope(compressed, MessageCodecEnum.ZSTD) +} + describe('SqsPermissionConsumer - zstd codec', () => { let diContainer: AwilixContainer let testAdmin: TestAwsResourceAdmin @@ -201,7 +216,7 @@ describe('SqsPermissionConsumer - zstd codec', () => { } // Simulate a publisher that compressed the message itself - const compressedBody = await compressMessageBody(JSON.stringify(message), MessageCodecEnum.ZSTD) + const compressedBody = await compressToZstdEnvelope(message) await diContainer.cradle.sqsClient.send( new SendMessageCommand({ QueueUrl: consumer.queueProps.url, @@ -491,7 +506,7 @@ describe('SqsPermissionConsumer - skipCompressionBelow', () => { id: 'disable-real-envelope-1', messageType: 'add', } - const compressedBody = await compressMessageBody(JSON.stringify(message), MessageCodecEnum.ZSTD) + const compressedBody = await compressToZstdEnvelope(message) await diContainer.cradle.sqsClient.send( new SendMessageCommand({ QueueUrl: noAutoConsumer.queueProps.url, From 29c5e365e2eaf2bf171cea0bdf90165cdc358547 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Fri, 22 May 2026 18:01:42 +0300 Subject: [PATCH 23/23] fix(codec): treat unregistered-codec messages as invalid, not retriable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A message envelope (inline body or offloaded-payload pointer) naming a codec the consumer has not registered is a deployment misconfiguration that will not be fixed within the SQS redelivery window. Retrying it just burns receive-count attempts before the message lands in the DLQ anyway — or, with no DLQ, spams the error reporter every visibility cycle until the message expires. deserializeMessage's missing-codec throw is now caught in handleMessage and funneled through the existing abort path: the error is logged/reported once and the message is routed to the DLQ (if configured) so it can be redriven after the codec is deployed. Inline and offloaded paths stay consistent — both terminal. This also fixes the CI failure in SqsPermissionConsumer.codec.spec.ts, whose assertion that an error is recorded could not hold while the throw bypassed handleMessageProcessed. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/sqs/lib/sqs/AbstractSqsConsumer.ts | 38 ++++++++++++++------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts index a4807584..ddf665ff 100644 --- a/packages/sqs/lib/sqs/AbstractSqsConsumer.ts +++ b/packages/sqs/lib/sqs/AbstractSqsConsumer.ts @@ -413,7 +413,21 @@ export abstract class AbstractSqsConsumer< const messageProcessingStartTimestamp = Date.now() - const deserializedMessage = await this.deserializeMessage(message) + let deserializedMessage: Either<'abort', ParseMessageResult> + try { + deserializedMessage = await this.deserializeMessage(message) + } catch (err) { + // A throw out of deserialization — e.g. a message envelope (inline body or an + // offloaded-payload pointer) naming a codec this consumer has not registered — + // is terminal for this consumer. Treat it as an invalid message: the abort + // branch below routes it to the DLQ if one is configured. Retrying would be + // pointless — a missing-codec deployment misconfiguration is not fixed within + // the redelivery window, so the message would only burn its receive-count + // attempts (or, with no DLQ, spam the error reporter for the whole retention + // period) before ending up in the DLQ anyway. + this.handleError(err) + deserializedMessage = ABORT_EARLY_EITHER + } if (deserializedMessage.error === 'abort') { await this.failProcessing(message) @@ -975,9 +989,10 @@ export abstract class AbstractSqsConsumer< (codecName) => { const handler = this.codecRegistry.get(codecName) if (!handler) { - // Misconfiguration, not a poison message: the pointer names a codec this - // consumer has not registered. Throwing here (outside retrieveOffloadedMessagePayload's - // catch) surfaces it as a retriable error so the message is not lost to the DLQ. + // The pointer names a codec this consumer has not registered. Throwing here + // (outside retrieveOffloadedMessagePayload's catch) propagates to handleMessage, + // which treats it as an invalid message and routes it to the DLQ (if one is + // configured) so it can be redriven once the codec is deployed. throw new Error( `No codec handler registered for "${codecName}". Register it via the consumer's \`codecs\` option.`, ) @@ -997,14 +1012,13 @@ export abstract class AbstractSqsConsumer< const envelope = resolveMessageResult.result.body const handler = this.codecRegistry.get(envelope.__mqtCodec) if (!handler) { - // Envelope-shaped body naming a codec this consumer has not registered. This is a - // deployment misconfiguration, not a poison message: throw (rather than handleError - // + abort) so it surfaces as a retriable error and the message stays on the queue - // until the codec is registered. This matches the offloaded-payload path above, - // where `resolveDecompressor` deliberately throws for the same reason. Aborting - // here would instead drop the message (or route it to the DLQ) and let the - // envelope's preserved sibling fields (id, type, …) satisfy a lenient schema and - // be processed as an incomplete message. Register the codec via the `codecs` option. + // Envelope-shaped body naming a codec this consumer has not registered. Throwing + // here propagates to handleMessage, which treats it as an invalid message and + // routes it to the DLQ (if one is configured) so it can be redriven once the codec + // is deployed. Throwing — rather than falling through — is essential: otherwise the + // envelope's preserved sibling fields (id, type, …) could satisfy a lenient schema + // and be processed as an incomplete message. Register the codec via the `codecs` + // option. throw new Error( `Received a message compressed with codec "${envelope.__mqtCodec}", which is not registered on this consumer. Register it via the \`codecs\` option.`, )