From 5e885959dc5bc2957a4b09caa59150b70419bf19 Mon Sep 17 00:00:00 2001 From: Andrew den Hertog Date: Thu, 8 Oct 2026 13:08:41 +1000 Subject: [PATCH 1/4] add a redis streams transport as @node-ts/bus-redis Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/redis-streams-transport.md | 7 + .circleci/config.yml | 10 + CLAUDE.md | 4 +- MIGRATING.md | 6 + README.md | 1 + docker-compose.yml | 22 + docs/.vitepress/config.mts | 1 + docs/guide/provisioning.md | 5 +- docs/guide/recoverability.md | 3 +- docs/package.json | 1 + docs/snippets/redis.ts | 24 + docs/transports.md | 1 + docs/transports/redis.md | 219 ++++ docs/typedoc.json | 1 + packages/bus-redis/CLAUDE.md | 20 + packages/bus-redis/README.md | 72 ++ packages/bus-redis/package.json | 76 ++ packages/bus-redis/src/error/index.ts | 3 + .../src/error/invalid-redis-key-name.ts | 23 + .../error/invalid-redis-transport-duration.ts | 27 + .../error/redis-transport-not-connected.ts | 14 + packages/bus-redis/src/index.mts | 5 + packages/bus-redis/src/index.ts | 4 + packages/bus-redis/src/redis-keys.spec.ts | 49 + packages/bus-redis/src/redis-keys.ts | 91 ++ packages/bus-redis/src/redis-scripts.ts | 232 ++++ .../src/redis-transport-configuration.ts | 58 + .../bus-redis/src/redis-transport-message.ts | 40 + .../src/redis-transport.integration.ts | 632 +++++++++ .../bus-redis/src/redis-transport.spec.ts | 266 ++++ packages/bus-redis/src/redis-transport.ts | 1130 +++++++++++++++++ packages/bus-redis/tsconfig.json | 9 + pnpm-lock.yaml | 102 ++ test.env | 1 + 34 files changed, 3154 insertions(+), 5 deletions(-) create mode 100644 .changeset/redis-streams-transport.md create mode 100644 docs/snippets/redis.ts create mode 100644 docs/transports/redis.md create mode 100644 packages/bus-redis/CLAUDE.md create mode 100644 packages/bus-redis/README.md create mode 100644 packages/bus-redis/package.json create mode 100644 packages/bus-redis/src/error/index.ts create mode 100644 packages/bus-redis/src/error/invalid-redis-key-name.ts create mode 100644 packages/bus-redis/src/error/invalid-redis-transport-duration.ts create mode 100644 packages/bus-redis/src/error/redis-transport-not-connected.ts create mode 100644 packages/bus-redis/src/index.mts create mode 100644 packages/bus-redis/src/index.ts create mode 100644 packages/bus-redis/src/redis-keys.spec.ts create mode 100644 packages/bus-redis/src/redis-keys.ts create mode 100644 packages/bus-redis/src/redis-scripts.ts create mode 100644 packages/bus-redis/src/redis-transport-configuration.ts create mode 100644 packages/bus-redis/src/redis-transport-message.ts create mode 100644 packages/bus-redis/src/redis-transport.integration.ts create mode 100644 packages/bus-redis/src/redis-transport.spec.ts create mode 100644 packages/bus-redis/src/redis-transport.ts create mode 100644 packages/bus-redis/tsconfig.json diff --git a/.changeset/redis-streams-transport.md b/.changeset/redis-streams-transport.md new file mode 100644 index 00000000..42d33eda --- /dev/null +++ b/.changeset/redis-streams-transport.md @@ -0,0 +1,7 @@ +--- +'@node-ts/bus-redis': minor +--- + +Add `RedisTransport`, a Redis Streams transport for Redis 7+ and Valkey 7.2+ (#347). Each service's queue is a stream read by a consumer group, with fan-out through subscription sets, delayed retries in a sorted set, messages taken over from stopped processes after `visibilityTimeoutMs`, a dead letter stream per queue with `deadLetterRetentionMs`, `bus provision` support with a `redis-acl` runtime permission plan, and replies. It needs `redis` (node-redis) 6 as a peer dependency. + +**Breaking:** this replaces `@node-ts/bus-redis` 0.x, the Redis lists transport for bus-core 1.x, which shared its name. It's a new transport with a new configuration and key layout, and messages in 0.x queues aren't moved: see "Migrating from @node-ts/bus-redis 0.x" at https://node-ts.github.io/bus/transports/redis. diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a63ece6..51f85781 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,7 @@ jobs: RABBITMQ_URL: amqp://guest:guest@127.0.0.1 POSTGRES_URL: postgres://postgres:password@127.0.0.1:5432/postgres MONGODB_URL: mongodb://127.0.0.1:27017/workflows?directConnection=true + REDIS_URL: redis://127.0.0.1:6379 - image: rabbitmq:3-management - image: localstack/localstack:3 environment: @@ -26,6 +27,10 @@ jobs: # A single-node replica set, since the outbox's transactions need one. "Wait for integration infra" initiates it. - image: mongo:7 command: ['--replSet', 'rs0'] + - image: redis:8 + # The Redis transport's tests run against Valkey too, on a port of its own since containers share the network + - image: valkey/valkey:8 + command: ['valkey-server', '--port', '6381'] resource_class: large steps: - checkout @@ -71,6 +76,8 @@ jobs: wait_for_port 5672 wait_for_port 5432 wait_for_port 27017 + wait_for_port 6379 + wait_for_port 6381 node .circleci/init-mongo-replica-set.mjs for i in $(seq 1 60); do curl -sf http://127.0.0.1:4566/_localstack/health | grep -q '"sqs": "\(available\|running\)"' && break @@ -80,6 +87,9 @@ jobs: - run: name: Integration Test command: pnpm run test:integration + - run: + name: Integration Test (Redis transport on Valkey) + command: REDIS_URL=redis://127.0.0.1:6381 pnpm exec dotenv -e test.env -- jest --runInBand --coverage=false packages/bus-redis/src/redis-transport.integration.ts # Save workspace for subsequent jobs (i.e. test) - persist_to_workspace: root: . diff --git a/CLAUDE.md b/CLAUDE.md index b402588d..2bdf0a1a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,7 @@ pnpm exec dotenv -e test.env -- jest packages/bus-sqs/src/sqs-transport.spec.ts - Jest is configured once at the root (`jest.config.ts`, ts-jest using `tsconfig.test.json`; a `moduleNameMapper` lets relative `.js` imports, as in generated message types, resolve to `.ts` source); `test/setup.ts` drops the default logger's `@node-ts/…` console warnings and errors (set `BUS_TEST_LOGS=true` to see them). Other console output still shows, so don't leave `console.log` in tests. - Tests: `*.spec.ts` = unit, `*.integration.ts` = integration. `packages/bus-test/src/in-memory.integration.ts` runs bus-test's own round trip suites over the in-memory queue and persistence. - **Build before testing across packages**: every package has `main: ./dist/index.js`, so e.g. bus-sqs tests import the built bus-core and bus-test, not their source. Changes in bus-core need `pnpm build` before they're visible to other packages. -- Integration tests for adapters need local infra; `docker compose up -d --wait` starts it all from the root `docker-compose.yml`, and waits until it's healthy (Mongo's healthcheck initiates its replica set). Endpoints default to the compose ports and can be overridden with env vars (listed in `test.env`): `LOCALSTACK_ENDPOINT` (default `http://localhost:4566`, dummy AWS creds in `test.env`), `RABBITMQ_URL` (`amqp://guest:guest@0.0.0.0`), `POSTGRES_URL` (`postgres://postgres:password@localhost:6432/postgres`), `MONGODB_URL` (`mongodb://localhost:27017/workflows?directConnection=true`, a single-node replica set, see `packages/bus-mongodb/CLAUDE.md`). CI runs all integration tests against CircleCI secondary containers (`.circleci/config.yml`). +- Integration tests for adapters need local infra; `docker compose up -d --wait` starts it all from the root `docker-compose.yml`, and waits until it's healthy (Mongo's healthcheck initiates its replica set). Endpoints default to the compose ports and can be overridden with env vars (listed in `test.env`): `LOCALSTACK_ENDPOINT` (default `http://localhost:4566`, dummy AWS creds in `test.env`), `RABBITMQ_URL` (`amqp://guest:guest@0.0.0.0`), `POSTGRES_URL` (`postgres://postgres:password@localhost:6432/postgres`), `MONGODB_URL` (`mongodb://localhost:27017/workflows?directConnection=true`, a single-node replica set, see `packages/bus-mongodb/CLAUDE.md`), `REDIS_URL` (`redis://localhost:6380`; compose also has Valkey on 6381). CI runs all integration tests against CircleCI secondary containers (`.circleci/config.yml`), and bus-redis' again against Valkey. - Formatting is automatic: a `.claude/settings.json` hook runs prettier on every file Claude edits, and husky + lint-staged format on commit. - Linting: `eslint.config.mjs` is `recommended` from ESLint and typescript-eslint, plus the type-aware `no-floating-promises`, `no-misused-promises` and `await-thenable`, with `eslint-config-prettier` so it never fights prettier. Type information comes from `tsconfig.eslint.json` (specs and `test/` fixtures included), and cross-package types come from each package's `dist`, so build first. lint-staged also runs ESLint on staged TS/JS files. Await or `.catch` every promise; use `void` only for deliberate fire-and-forget, with a comment saying why. Unused parameters are allowed when prefixed with `_`. - Dependency updates are grouped by Renovate (`renovate.json`); the maintainer installs the Renovate GitHub app. @@ -46,7 +46,7 @@ pnpm exec dotenv -e test.env -- jest packages/bus-sqs/src/sqs-transport.spec.ts - `bus-messages` — `Message`/`Command`/`Event` base types and `MessageAttributes`. Messages identify themselves by `$name` (usually a static `NAME`) and `$version`. - `bus-core` — the bus itself, the `Transport`/`Persistence`/`Serializer`/`Receiver` interfaces (`Transport.endpointName` is the service queue's name), and in-memory defaults (`InMemoryQueue`, `InMemoryPersistence`, `JsonSerializer`). -- Transports: `bus-sqs` (SNS topics fanned out to SQS queues, with policy generation), `bus-rabbitmq`, and `PostgresTransport` in `bus-postgres` (queues in shared Postgres tables, no broker; see `packages/bus-postgres/CLAUDE.md`). +- Transports: `bus-sqs` (SNS topics fanned out to SQS queues, with policy generation), `bus-rabbitmq`, `PostgresTransport` in `bus-postgres` (queues in shared Postgres tables, no broker; see `packages/bus-postgres/CLAUDE.md`), and `bus-redis` (a Redis Stream per queue, read by a consumer group, on Redis 7+/Valkey 7.2+; see `packages/bus-redis/CLAUDE.md`). - Persistence (workflow state and delayed messages): `bus-postgres`, `bus-mongodb`. - `bus-sqs-lambda` — a `Receiver` that feeds Lambda SQS events into the bus instead of the bus polling the transport. - `bus-opentelemetry` — `openTelemetry()` middleware for spans, W3C trace context (carried in message attributes) and messaging metrics, with only `@opentelemetry/api` as a peer. See `packages/bus-opentelemetry/CLAUDE.md`. diff --git a/MIGRATING.md b/MIGRATING.md index 06a771b4..9549d540 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -160,6 +160,12 @@ Two things behave differently: - **The AMQP `messageId` property is now the bus' `messageId`**, so it's the same for every message sent with the same id, rather than a new UUID per publish. `sentAt` is carried in a `sentAt` header. - `amqplib` is now version 2.2. It ships its own types, so remove `@types/amqplib`. `heartbeat=0` in a connection string now disables heartbeats. +## @node-ts/bus-redis + +- **It's a new transport on Redis Streams**, replacing the 0.x Redis lists transport from the `node-ts/bus-redis` repository. `BusRedisModule` and inversify are gone: pass `new RedisTransport({ queueName, connection: { url } })` to `withTransport()`. It needs Redis 7.0+ or Valkey 7.2+, and `redis` (node-redis) 6 as a peer dependency. +- **`maxRetries` is removed**: use the bus' recoverability policy, `defaultRecoverability({ maxAttempts })`. `visibilityTimeout` is now `visibilityTimeoutMs`, `subscriptionsKeyPrefix` is now `keyPrefix`, and `withScheduler` is removed. +- **Messages in 0.x queues aren't moved**, and the two versions can't send each other messages. Drain the old queues, then provision and deploy every service that exchanges messages together. See [Migrating from @node-ts/bus-redis 0.x](https://node-ts.github.io/bus/transports/redis#migrating-from-node-ts-bus-redis-0-x). + ## @node-ts/bus-sqs-lambda - **The receiver applies the bus' recoverability policy.** A record that's retried has its visibility timeout set to the policy's delay, rather than waiting out the queue's visibility timeout, and a record that's dead-lettered, by the policy or `failMessage()`, is moved to the dead letter queue with its failure metadata and reported to Lambda as handled. diff --git a/README.md b/README.md index 5f415ec3..58500ef0 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Requires Node.js 24 or later. - [@node-ts/bus-sqs](https://github.com/node-ts/bus/tree/master/packages/bus-sqs) - An Amazon SQS transport - [@node-ts/bus-sqs-lambda](https://github.com/node-ts/bus/tree/master/packages/bus-sqs-lambda) - A receiver that handles SQS messages in AWS Lambda - [@node-ts/bus-postgres](https://github.com/node-ts/bus/tree/master/packages/bus-postgres) - A Postgres persistence for workflow state, and a Postgres transport that needs no broker +- [@node-ts/bus-redis](https://github.com/node-ts/bus/tree/master/packages/bus-redis) - A Redis Streams transport, for Redis or Valkey - [@node-ts/bus-mongodb](https://github.com/node-ts/bus/tree/master/packages/bus-mongodb) - A MongoDB persistence for workflow state - [@node-ts/bus-opentelemetry](https://github.com/node-ts/bus/tree/master/packages/bus-opentelemetry) - OpenTelemetry tracing and metrics, as bus middleware - [@node-ts/bus-nestjs](https://github.com/node-ts/bus/tree/master/packages/bus-nestjs) - A NestJS module: handlers and workflows as providers, and the bus run by Nest's lifecycle diff --git a/docker-compose.yml b/docker-compose.yml index edb14d1c..28a76f69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,28 @@ services: ports: - '6432:5432' + redis: + image: redis:8 + ports: + - '6380:6379' + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 2s + timeout: 5s + retries: 30 + + # Optional: run the Redis transport's tests against Valkey with + # REDIS_URL=redis://localhost:6381 + valkey: + image: valkey/valkey:8 + ports: + - '6381:6379' + healthcheck: + test: ['CMD', 'valkey-cli', 'ping'] + interval: 2s + timeout: 5s + retries: 30 + mongo: image: mongo:7 # A single-node replica set, since the outbox's transactions need one. The diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 8207004f..7abdf33a 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -127,6 +127,7 @@ const infrastructure: DefaultTheme.SidebarItem[] = [ { text: 'Amazon SQS', link: '/transports/amazon-sqs' }, { text: 'SQS and Lambda', link: '/transports/sqs-lambda' }, { text: 'Postgres', link: '/transports/postgres' }, + { text: 'Redis', link: '/transports/redis' }, { text: 'Custom transports', link: '/transports/custom' } ] }, diff --git a/docs/guide/provisioning.md b/docs/guide/provisioning.md index 3975af95..c10da059 100644 --- a/docs/guide/provisioning.md +++ b/docs/guide/provisioning.md @@ -16,6 +16,7 @@ Everything is worked out from the bus' configuration: its transport, persistence | [Amazon SQS](/transports/amazon-sqs#provisioning) | An SNS topic per message, the service queue and dead letter queue, a subscription per handled message, and the queue's access policy | | [RabbitMQ](/transports/rabbitmq#provisioning) | A fanout exchange per message, the service queue with its retry and dead letter queues, and the bindings | | [Postgres transport](/transports/postgres#provisioning) | The schema, the messages, queues, subscriptions and dead letters tables, the service's queue, and a subscription per handled message | +| [Redis](/transports/redis#provisioning) | The service queue's stream and its consumer group, and the queue in the subscription set of each handled message | | [Postgres](/persistence/postgres#provisioning) | The schema, a table per workflow state with indexes on its lookups, and the outgoing messages table | | [MongoDB](/persistence/mongodb#provisioning) | A collection per workflow state with indexes on its lookups, and the outgoing messages collection | @@ -118,7 +119,7 @@ SqsTransport (7 resources) - `adapters` has an entry for each transport or persistence that provisions anything, in the order they ran: the persistence first, then the transport. - Each resource has a `type`, its `name` (an ARN for AWS resources) and, for some, the `properties` it's created with. The types are listed on each adapter's page. -- `runtimePermissions` is only included with `--permissions`. Its `format` says what `document` is: `iam-policy` (an IAM policy document), `rabbitmq-permissions` (the `configure`, `write` and `read` expressions for the vhost), `sql` (a list of grant statements) or `mongodb-privileges` (the privileges of a role). The `ProvisionReport` type, exported by `@node-ts/bus-cli`, describes it. +- `runtimePermissions` is only included with `--permissions`. Its `format` says what `document` is: `iam-policy` (an IAM policy document), `rabbitmq-permissions` (the `configure`, `write` and `read` expressions for the vhost), `sql` (a list of grant statements), `redis-acl` (the ACL rules of a user) or `mongodb-privileges` (the privileges of a role). The `ProvisionReport` type, exported by `@node-ts/bus-cli`, describes it. ## Checking resources at startup @@ -158,5 +159,5 @@ A [custom transport](/transports/custom) or [persistence](/persistence/custom) i ## See also -- [Amazon SQS](/transports/amazon-sqs#provisioning), [RabbitMQ](/transports/rabbitmq#provisioning), the [Postgres transport](/transports/postgres#provisioning), [Postgres](/persistence/postgres#provisioning) and [MongoDB](/persistence/mongodb#provisioning), for what each provisions and the permissions it needs +- [Amazon SQS](/transports/amazon-sqs#provisioning), [RabbitMQ](/transports/rabbitmq#provisioning), the [Postgres transport](/transports/postgres#provisioning), [Redis](/transports/redis#provisioning), [Postgres](/persistence/postgres#provisioning) and [MongoDB](/persistence/mongodb#provisioning), for what each provisions and the permissions it needs - [`BusInstance.provision`](/api/bus-core/classes/BusInstance#provision), [`ProvisioningPlan`](/api/bus-core/interfaces/ProvisioningPlan) and [`ResourcesNotProvisioned`](/api/bus-core/classes/ResourcesNotProvisioned) in the API reference diff --git a/docs/guide/recoverability.md b/docs/guide/recoverability.md index af8c4389..a9eb48e0 100644 --- a/docs/guide/recoverability.md +++ b/docs/guide/recoverability.md @@ -83,13 +83,14 @@ A message that was failed with `failMessage()` without anything throwing has a ` | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | | [RabbitMQ](/transports/rabbitmq) | an AMQP header on the dead-lettered message | | [Amazon SQS](/transports/amazon-sqs) | an SQS message attribute on the dead-lettered message. The SNS envelope in its body, with the message's attributes, is kept | +| [Redis](/transports/redis) | a `bus-failure` field of the entry in the queue's dead letter stream | | In-memory queue | the dead-lettered message's `raw.headers` | Transports also add it to messages they dead-letter themselves because they can't be parsed. `bus-failure` is reserved, so outgoing middleware can't set it. ## Replaying dead-lettered messages -Once the cause is fixed, move the messages back to the service queue with the broker's own tools: a [dead letter queue redrive](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-configure-dead-letter-queue-redrive.html) on Amazon SQS, or a [shovel](https://www.rabbitmq.com/docs/shovel) on RabbitMQ. The RabbitMQ transport leaves its attempt count off dead-lettered messages, so a replayed message gets all its attempts again. +Once the cause is fixed, move the messages back to the service queue with the broker's own tools: a [dead letter queue redrive](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-configure-dead-letter-queue-redrive.html) on Amazon SQS, a [shovel](https://www.rabbitmq.com/docs/shovel) on RabbitMQ, or the [redrive script](/transports/redis#retries-and-dead-letters) on Redis. The RabbitMQ transport leaves its attempt count off dead-lettered messages, so a replayed message gets all its attempts again. ## Receivers diff --git a/docs/package.json b/docs/package.json index 0ea4102c..eae9b325 100644 --- a/docs/package.json +++ b/docs/package.json @@ -28,6 +28,7 @@ "@node-ts/bus-opentelemetry": "workspace:^", "@node-ts/bus-postgres": "workspace:^", "@node-ts/bus-rabbitmq": "workspace:^", + "@node-ts/bus-redis": "workspace:^", "@node-ts/bus-sqs": "workspace:^", "@node-ts/bus-sqs-lambda": "workspace:^", "@node-ts/bus-test": "workspace:^", diff --git a/docs/snippets/redis.ts b/docs/snippets/redis.ts new file mode 100644 index 00000000..a8c5f21a --- /dev/null +++ b/docs/snippets/redis.ts @@ -0,0 +1,24 @@ +import { Bus } from '@node-ts/bus-core' +import { RedisTransport, RedisTransportConfiguration } from '@node-ts/bus-redis' +import { reserveRoomHandler } from './handlers/reserve-room-handler' +import { messageTypes } from './message-types.generated' + +const redisConfiguration: RedisTransportConfiguration = { + queueName: 'reservations-service', + connection: { url: 'redis://localhost:6379' }, + // Longer than the slowest handler takes, or a message still being handled is received again + visibilityTimeoutMs: 30_000 +} +const redisTransport = new RedisTransport(redisConfiguration) + +const bus = Bus.configure() + .withMessageTypes(messageTypes) + .withTransport(redisTransport) + .withHandler(reserveRoomHandler) + // For local development: creates the queue's stream, its consumer group and its subscriptions when the bus + // initializes. In production, create them at deploy time with `bus provision` instead. + .withAutoProvision() + .build() + +await bus.initialize() +await bus.start() diff --git a/docs/transports.md b/docs/transports.md index e9bfbe2e..88a11e1d 100644 --- a/docs/transports.md +++ b/docs/transports.md @@ -14,6 +14,7 @@ By default the bus uses an in-memory queue, `InMemoryQueue`, which needs nothing AWS' managed queues, with SNS topics for events. `bus provision` creates the topics, queues and subscriptions, or you manage them. Handle SQS messages in AWS Lambda, which reads the queue and passes batches to the bus. Queues in the Postgres database you already have, with no broker to run. Suits one service, or services that share a database. + Queues in Redis Streams, on the Redis or Valkey you may already run. Retries and dead letters are built on top, with no modules. Adapt another broker by implementing the Transport interface, and check it with the conformance suite. diff --git a/docs/transports/redis.md b/docs/transports/redis.md new file mode 100644 index 00000000..e5568b66 --- /dev/null +++ b/docs/transports/redis.md @@ -0,0 +1,219 @@ +--- +title: Redis +description: Run @node-ts/bus on Redis or Valkey, with each service's queue in a Redis Stream, using RedisTransport from @node-ts/bus-redis. +--- + +# Redis + +`RedisTransport` from `@node-ts/bus-redis` keeps each service's queue in a [Redis Stream](https://redis.io/docs/latest/develop/data-types/streams/), read by a consumer group, on Redis or Valkey. Fan-out to subscribers, delayed retries and dead letters are built on top of streams, with no Redis modules. This page covers when to use it, configuring and provisioning it, how messages are received and retried, keeping them safe, and moving from the old `@node-ts/bus-redis` 0.x. + + + +## When to use it + +Use it when you already run Redis or Valkey, and want queues without running another broker. You can move to another transport later without changing your messages, handlers or workflows. + +Redis keeps everything in memory, and by default it doesn't write every change to disk, so it can lose messages that a broker such as [RabbitMQ](/transports/rabbitmq) or [Amazon SQS](/transports/amazon-sqs) would keep. Read [Keeping messages safe](#keeping-messages-safe) before you use it in production. + +It needs Redis 7.0 or later, or Valkey 7.2 or later, run as a single primary, optionally with replicas. Redis Cluster and Sentinel aren't supported yet, though the keys are laid out so they can be. It uses [node-redis](https://github.com/redis/node-redis) 6, which you install alongside it. + +## Installation + +::: code-group + +```sh [npm] +npm i @node-ts/bus-redis @node-ts/bus-core redis +``` + +```sh [pnpm] +pnpm add @node-ts/bus-redis @node-ts/bus-core redis +``` + +```sh [yarn] +yarn add @node-ts/bus-redis @node-ts/bus-core redis +``` + +::: + +Configure a `RedisTransport` and pass it to the bus configuration: + +<<< @/snippets/redis.ts + +## Configuration + +| Option | Default | Description | +| ----------------------- | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `queueName` | | The queue this service receives from. It's part of every key of the queue, so it can't contain `{` or `}`. | +| `connection` | `{}` (localhost) | node-redis' [`createClient()` options](https://github.com/redis/node-redis/blob/master/docs/client-configuration.md), such as `url`, `username`, `password`, `database` and `socket` for TLS. | +| `keyPrefix` | `bus` | What every key starts with. Services that send each other messages must use the same prefix. It can't contain `{` or `}`. | +| `visibilityTimeoutMs` | `30000` | How long a handler has, in milliseconds, before another receiver takes the message over and it's handled again. Set it above how long your slowest handler takes. | +| `deadLetterRetentionMs` | `1209600000` (14 d) | How long dead-lettered messages are kept, in milliseconds. Older ones are trimmed, roughly, as new ones are dead-lettered. `0` or `Infinity` keeps them until you remove them. | + +The transport makes two connections: one to send and settle messages, and one that waits for new ones. It always speaks RESP2, whatever node-redis' default. An invalid `queueName` or `keyPrefix` throws `InvalidRedisKeyName`, and an invalid duration throws `InvalidRedisTransportDuration`. + +## Sending and receiving + +Provisioning a service creates its queue's stream, `bus:{}:queue`, with a consumer group named after the queue, and adds the queue to the subscription set, `bus:subscriptions:<$name>`, of each message it handles. Sending a command or publishing an event adds a copy of it to the stream of every queue in its subscription set, in one transaction, so every service that handles it gets a copy, or none does. A message no queue is subscribed to is dropped, with a warning: provision the service that handles it first. A queue whose stream doesn't exist is skipped, with a warning, rather than its stream being created with nothing to read it. + +Every instance of a service reads its queue as the same consumer group, so each message goes to one of them. A message that's been read stays pending to its receiver until it's handled, and is then acknowledged and removed from the stream, so streams only hold what's waiting or being handled. If it isn't settled within `visibilityTimeoutMs`, such as when the process stops, another receiver takes it over and handles it again, counting a failed attempt. That also happens if a handler takes longer than the timeout, while the first receipt is still being handled. The first receipt can then no longer delete, retry or dead-letter the message, which is logged as a warning. Make handlers idempotent, or use [`withOutbox()`](/guide/outbox), whose inbox skips a copy of a message that was already handled. + +Each process reads its queue once at a time, for as many messages as it has workers waiting, waiting on the server for up to a second for one to arrive. A message is received as soon as it's added. Messages are received in the order they were added, but a retried message goes to the back of the queue, and there's no strict ordering between instances. + +When a process stops, it gives back any message it read but didn't hand to a worker, without counting an attempt, and once its workers have settled what they were handling, it leaves the consumer group. A process that crashed leaves its consumer behind: it's removed by another instance once it has nothing pending and has been idle for a day. + +### Messages from other systems + +A system that isn't on @node-ts/bus can send a message to the services that handle it, such as a [custom handler](/guide/messages/system-messages)'s `topicIdentifier`, by adding it to the stream of each queue in the subscription set: + +```sh +redis-cli SMEMBERS 'bus:subscriptions:billing.order-paid' +# For each queue it returns, such as reservations-service: +redis-cli XADD 'bus:{reservations-service}:queue' NOMKSTREAM '*' \ + body '{"orderId":"o-1"}' \ + attributes '{"messageId":"a-unique-id","attributes":{},"stickyAttributes":{}}' \ + headers '{}' +``` + +`body` is the message as JSON, which the bus' serializer reads and the custom handler's resolver then matches, and `attributes` holds its [attributes](/guide/message-attributes). Give each message its own `messageId`, so the [inbox](/guide/outbox#the-inbox) can recognise a copy. + +## Retries and dead letters + +When the [recoverability policy](/guide/recoverability) retries a message, a copy that counts one more failed attempt waits in the queue's delayed set, `bus:{}:delayed`, scored by when it's due, and the received message is removed. Before each read, due messages are moved back to the stream. A message retried with no delay goes straight back to the stream. `failedAttempts` is the attempts the message was retried after, plus each earlier receipt that was taken over after the visibility timeout ended. + +A dead-lettered message is moved to the queue's dead letter stream, `bus:{}:dead-letter`, in one script, with its [failure metadata](/guide/recoverability#failure-metadata) in a `bus-failure` field and without its attempt count, so a replayed message gets all its attempts again. Entries older than `deadLetterRetentionMs` are trimmed as new ones are added. Trimming is approximate, so a few older entries may stay a while. A message that can't be parsed is dead-lettered straight away, with the parse error as its failure. + +To move a queue's dead letters back to it, save this script as `redrive.lua`: + +```lua +local entries = redis.call('XRANGE', KEYS[1], '-', '+', 'COUNT', 1000) +for _, entry in ipairs(entries) do + local fields = {} + for i = 1, #entry[2], 2 do + if entry[2][i] ~= 'bus-failure' then + fields[#fields + 1] = entry[2][i] + fields[#fields + 1] = entry[2][i + 1] + end + end + redis.call('XADD', KEYS[2], '*', unpack(fields)) + redis.call('XDEL', KEYS[1], entry[1]) +end +return #entries +``` + +and run it, again until it returns 0: + +```sh +redis-cli --eval redrive.lua 'bus:{reservations-service}:dead-letter' 'bus:{reservations-service}:queue' +``` + +To read dead letters, use `XRANGE 'bus:{reservations-service}:dead-letter' - +`, and to remove them all, `DEL 'bus:{reservations-service}:dead-letter'`. + +## Delayed delivery + +Messages sent with [`deliverAfter` or `deliverAt`](/guide/delayed-delivery) are stored by the persistence until they're due, as on any transport, so use a persistence that stores them, such as [Postgres](/persistence/postgres) or [MongoDB](/persistence/mongodb). Retries are delayed by the transport itself. + +## Keeping messages safe + +Redis keeps the queues in memory. Set it up so it keeps them: + +- **Set `maxmemory-policy` to `noeviction`.** Under any other policy, Redis removes keys when it runs out of memory, which can be a queue with everything in it. With `noeviction`, sends fail instead, and the bus reports them. +- **Turn on the append-only file** (`appendonly yes`). Without it, a restart loses everything since the last snapshot. With `appendfsync everysec`, the default, a crash can lose about the last second of messages; `appendfsync always` loses none, but every write waits for the disk. +- **Replication is asynchronous.** If the primary fails, a replica that's promoted may not have the last messages sent to it, or the last settlements, so some messages are lost and some are handled again. + +Managed services that keep a transaction log, such as Amazon MemoryDB, don't lose acknowledged writes on a failover. + +## Keys + +| Key | Type | Holds | +| --------------------------- | ---------- | -------------------------------------------------------------------------------- | +| `bus:{}:queue` | Stream | The queue's messages until they're handled, read by the consumer group `` | +| `bus:{}:delayed` | Sorted set | Retried messages until they're due, scored by when | +| `bus:{}:dead-letter` | Stream | Dead-lettered messages, with their `bus-failure` field | +| `bus:subscriptions:<$name>` | Set | The queues a message is sent to | + +The queue's name is a hash tag (`{}`), so a queue's keys are in one hash slot, and each script that changes them is one atomic step. To see a queue's backlog, use `XLEN 'bus:{reservations-service}:queue'`, and for what's being handled, `XPENDING 'bus:{reservations-service}:queue' reservations-service`. + +## Provisioning + +The transport creates nothing when the service starts. Create its consumer group and subscriptions at deploy time with [`bus provision`](/guide/provisioning), or with `withAutoProvision()` for local development and tests. It provisions: + +| Type | Resource | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `redis-stream` | The queue's stream, `bus:{}:queue` | +| `redis-consumer-group` | The consumer group the service reads it as, named after the queue, from the start of the stream | +| `redis-subscription` | The queue in the subscription set of each message the bus handles, and of each `topicIdentifier` of a [custom handler](/guide/messages/system-messages) | + +A send-only bus provisions nothing, since sending needs no resources of its own. The delayed set and the dead letter stream are created when they're first used. Provisioning never removes anything, so a subscription for a message the service no longer handles stays until you remove it with `SREM`. Its messages are then discarded by the service, as it has no handler for them. Provisioning needs the `XGROUP CREATE` and `SADD` commands. + +At `initialize()`, unless the bus only sends, the transport checks that its stream and consumer group exist (`XINFO GROUPS`), and that its queue is in the subscription set of each message it handles (`SISMEMBER`). It throws `ResourcesNotProvisioned` naming each one that's missing. `withResourceVerification(false)` turns the check off. + +### Runtime permissions + +`bus provision --dry-run --permissions` prints the [ACL rules](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/) the service needs, as a `redis-acl` document of a `user` and its `rules`. For a service whose queue is `reservations-service`: + +```txt +~bus:{reservations-service}:* %W~bus:{*}:queue %R~bus:subscriptions:* ++xadd +smembers +multi +exec +xreadgroup +xack +xdel +xpending +xclaim +xinfo|groups +xinfo|consumers ++xgroup|delconsumer +zadd +zrange +zrangebyscore +zrem +sismember +time +evalsha +eval +``` + +The service can read and write its own queue's keys, add messages to any queue's stream (to send, publish and reply) without reading them, and read the subscription sets. It runs its scripts with `EVALSHA`, and `EVAL` to load them, and the commands they run are checked too. A send-only bus only needs the second and third key patterns and `+xadd +smembers +multi +exec`. Create the user with the rules, such as: + +```sh +redis-cli ACL SETUSER reservations-service on '>a-long-password' resetkeys resetchannels -@all \ + '~bus:{reservations-service}:*' '%W~bus:{*}:queue' '%R~bus:subscriptions:*' +xadd +smembers ... +``` + +and connect with its `username` and `password`. Key permissions (`%R~`, `%W~`) need Redis 7.0 or later. + +## Replies + +A [reply](/guide/workflows/request-reply) from `ctx.reply()` is added straight to the stream of the requester's queue, the request's return address, which is the `queueName` of the bus that sent it. A reply to a queue whose stream doesn't exist throws `EndpointNotFound`, and creates nothing. + +## Message attributes + +Each message is a stream entry with the fields `body` (the message, as the bus' serializer writes it), `attributes` (its attributes, as JSON) and `headers` (its headers from [outgoing middleware](/guide/middleware#outgoing-middleware), as JSON), and `failedAttempts` once it's been retried. Only the `bus-failure` header is reserved. The `TransportMessage.id` is the entry's id, which is different for each queue's copy of a published message, and changes when the message is retried: the [`messageId`](/guide/message-attributes/message-id) stays the same. + +## Running Redis locally + +```sh +docker run -d -p 6379:6379 redis:8 +``` + +Valkey works the same way: + +```sh +docker run -d -p 6379:6379 valkey/valkey:8 +``` + +## Migrating from @node-ts/bus-redis 0.x + +The 0.x versions of `@node-ts/bus-redis`, from the `node-ts/bus-redis` repository, ran on bus-core 1.x with inversify, and kept queues in Redis lists. This version is a new transport on Redis Streams. First move your services to the current bus-core, following [Upgrading to 2.0](/upgrading/v2), then change the transport's configuration: + +| 0.x option | Now | +| ------------------------ | --------------------------------------------------------------------------------------------------------------------- | +| `BusRedisModule` | `new RedisTransport(configuration)`, passed to `Bus.configure().withTransport()` | +| `queueName` | `queueName` | +| `connectionString` | `connection: { url }` | +| `maxRetries` | The bus' [recoverability policy](/guide/recoverability): `withRecoverability(defaultRecoverability({ maxAttempts }))` | +| `visibilityTimeout` | `visibilityTimeoutMs` | +| `withScheduler` | Removed. Every instance takes over messages left pending past the visibility timeout. | +| `subscriptionsKeyPrefix` | `keyPrefix`. The keys are laid out differently, so the old prefix doesn't carry over. | + +Messages aren't moved from the old lists to the new streams, and services on the two versions can't send each other messages. Switch every service that sends a message and every service that handles it together: + + + +1. **Stop sending.** Stop the services, or the parts of them, that send messages. +2. **Drain the old queues.** Let the services on 0.x handle what's left, until their lists are empty. +3. **Provision.** Run [`bus provision`](/guide/provisioning) for each service on the new version, which creates its stream, consumer group and subscriptions. +4. **Deploy.** Start the services on the new version. +5. **Clean up.** Remove the old keys, which start with the old `subscriptionsKeyPrefix` (`node-ts:bus-redis:subscriptions:` by default) and the old queue names. + + + +## See also + +- [Provisioning](/guide/provisioning) +- [Recoverability](/guide/recoverability) +- [`RedisTransportConfiguration`](/api/bus-redis/interfaces/RedisTransportConfiguration) in the API reference diff --git a/docs/typedoc.json b/docs/typedoc.json index 22403eea..cc83510d 100644 --- a/docs/typedoc.json +++ b/docs/typedoc.json @@ -10,6 +10,7 @@ "../packages/bus-sqs-lambda", "../packages/bus-rabbitmq", "../packages/bus-postgres", + "../packages/bus-redis", "../packages/bus-mongodb", "../packages/bus-opentelemetry", "../packages/bus-nestjs", diff --git a/packages/bus-redis/CLAUDE.md b/packages/bus-redis/CLAUDE.md new file mode 100644 index 00000000..dca3651a --- /dev/null +++ b/packages/bus-redis/CLAUDE.md @@ -0,0 +1,20 @@ +# bus-redis + +A Redis Streams transport (node-redis 6, a peer dependency), replacing the archived `node-ts/bus-redis` 0.x (Redis lists via modest-queue, bus-core 1.x). The design was agreed on #347 (see the "Accepted design" comment). Read the root `CLAUDE.md` first. + +## Design + +- **Config** (`src/redis-transport-configuration.ts`): `queueName`, `connection` (node-redis `createClient()` options minus `RESP`/modules/scripts/typeMapping, default `{}`), `keyPrefix` (default `bus`), `visibilityTimeoutMs` (30 s), `deadLetterRetentionMs` (14 days; `0`/`Infinity` keep everything). No `deadLetterQueueName`: dead letters are per queue. `queueName`/`keyPrefix` with `{`/`}` or empty throw `InvalidRedisKeyName`; bad durations throw `InvalidRedisTransportDuration`. Using it before `connect()` throws `RedisTransportNotConnected`. +- **Connections**: `connect()` makes the command client (`createClient({ ...connection, RESP: 2 })`, so raw replies always have RESP2 shapes; everything goes through `sendCommand`, `multi().addCommand()` or scripts). `start()` makes the reader with `duplicate()` and a command timeout of the block time + 10 s. Every client has an `error` listener. Only standalone Redis/Valkey (no Cluster or Sentinel in v1), but keys stay Cluster-ready. +- **Keys** (`src/redis-keys.ts`, internal): `:{}:queue` (stream, consumer group ``), `:{}:delayed` (zset), `:{}:dead-letter` (stream), `:subscriptions:<$name>` (set of queue names). Every script touches one queue's keys only, which share the `{}` hash tag. `escapeGlob` escapes ACL key patterns. +- **Entries**: fields `body` (serializer output), `attributes` (JSON), `headers` (JSON) and `failedAttempts` once returned; the dead letter copy adds `bus-failure` and drops `failedAttempts`. Only `bus-failure` is a reserved header. `TransportMessage.id` is the entry id (new on each retry). +- **Sending**: `SMEMBERS` the subscription set, then one `MULTI` of `XADD NOMKSTREAM * …` per subscriber (all or nothing on a single primary). No subscribers warns and drops; a null reply (stream missing) warns. `sendToAddress` is `XADD NOMKSTREAM`, `EndpointNotFound` on null or an address with braces. +- **Scripts** (`src/redis-scripts.ts`, run with `EVALSHA`, falling back to `EVAL` on `NOSCRIPT`; they use the server's `TIME` and only Redis 6.2/7.0 commands so Valkey 7.2+ works; don't use `XACKDEL`, `XNACK` or `XREADGROUP CLAIM`): every settle checks `isPendingTo` (pending to this consumer with the delivery count it was read with, the lease token) and returns 0 if not, which the transport logs as taken over. `DELETE` = XACK + XDEL. `RETURN` = ZADD a `cjson` copy `{id, fields}` at now + delay (or XADD at once when the delay is 0) + XACK + XDEL. `FAIL` = XADD to the dead letter stream with `MINID ~ now - retention` + XACK + XDEL. `MAINTAIN` (before each read) moves up to 100 due zset entries back with XADD, returns ms until the next is due, reclaims with `XPENDING … IDLE ` + `XCLAIM` (once per second, for as many reads as are waiting), and once a minute removes other consumers with nothing pending and idle for a day. `RELEASE` (stop) re-claims a read-but-unhandled entry with `IDLE RETRYCOUNT JUSTID`, so another receiver takes it straight over without an attempt counted. `LEAVE` (on `disconnect()`, after the bus' workers settled) `XGROUP DELCONSUMER`s this consumer only when nothing is pending to it: DELCONSUMER drops pending entries, so never run it otherwise. +- **Receiving**: one fetch at a time per transport (`fetchForWaiters`/`fetch`) for every waiting `readNextMessage()`: MAINTAIN, then reclaimed entries, or else `XREADGROUP … COUNT BLOCK min(1000, next due) STREAMS >` on the reader. It loops until a message or `stop()`; a fetch error rejects the waiting reads (`NOGROUP` becomes `ResourcesNotProvisioned`), so the bus logs it and backs off. `failedAttempts = failedAttempts field + deliveries - 1` (new reads have 1 delivery; a reclaimed entry has its XPENDING count + 1), so expired receipts count like SQS and Postgres. `stop()` waits for the fetch in flight (≤ 1 s), releases unhandled entries and resolves waiting reads with `undefined`; handlers still settle until `disconnect()`. The consumer name is `--`, per transport. Unparseable bodies are dead-lettered on read. +- **Provisioning**: `provision()` (unless send-only or a dry run) runs `XGROUP CREATE 0 MKSTREAM` (`BUSYGROUP` is success) and `SADD`s every handled `$name` and custom-handler topic. `messageNames` is unused (no per-message resources). Plan types: `redis-stream`, `redis-consumer-group`, `redis-subscription`; `runtimePermissions` is `{ format: 'redis-acl', document: { user: '', rules } }`: own keys RW, `%W~:{*}:queue`, `%R~:subscriptions:*`, and the commands the transport and its scripts run (send-only: just the two patterns + `xadd smembers multi exec`). The integration test runs a bus as an ACL user with exactly these rules, so keep the list in step with the scripts. `initialize()` with `verifyResources` (not send-only) checks `XINFO GROUPS` and `SISMEMBER`s and throws `ResourcesNotProvisioned`. +- Delayed sends use the persistence's outgoing messages and dispatcher (no core change). The transport doesn't join the `withOutbox()` transaction. + +## Tests + +- Integration tests need Redis at `redis://localhost:6380` (override with `REDIS_URL`): `docker compose up -d --wait redis` from the repo root (port 6380, since 6379 is often taken). Valkey is on 6381 (`docker compose up -d --wait valkey`, `REDIS_URL=redis://localhost:6381`); CI runs the integration file against both. Tests use a random `keyPrefix`, remove only their own keys with `SCAN`/`UNLINK`, and must never `FLUSHDB`/`FLUSHALL`: the server may be shared. The ACL test creates and deletes its own user. +- The suites use bus-test's fixtures and message types, so this package has no `generate:message-types` script. diff --git a/packages/bus-redis/README.md b/packages/bus-redis/README.md new file mode 100644 index 00000000..23bd9a2d --- /dev/null +++ b/packages/bus-redis/README.md @@ -0,0 +1,72 @@ +# @node-ts/bus-redis + +A [Redis Streams](https://redis.io/docs/latest/develop/data-types/streams/) transport for [@node-ts/bus](https://node-ts.github.io/bus), on Redis or Valkey. Each service's queue is a stream read by a consumer group, with fan-out to subscribers, delayed retries and dead letters built on top. + +[![npm](https://img.shields.io/npm/v/@node-ts/bus-redis)](https://www.npmjs.com/package/@node-ts/bus-redis) + +**[Documentation](https://node-ts.github.io/bus/transports/redis)** · [Changelog](https://github.com/node-ts/bus/blob/master/packages/bus-redis/CHANGELOG.md) + +## Installation + +Requires Node.js 24 or later. + +It runs on Redis 7.0 or later, or Valkey 7.2 or later, as a single primary. + +```sh +npm i @node-ts/bus-redis @node-ts/bus-core redis +``` + +## Usage + +Configure a `RedisTransport` and pass it to the bus configuration: + + + +```ts +import { Bus } from '@node-ts/bus-core' +import { RedisTransport, RedisTransportConfiguration } from '@node-ts/bus-redis' +import { reserveRoomHandler } from './handlers/reserve-room-handler' +import { messageTypes } from './message-types.generated' + +const redisConfiguration: RedisTransportConfiguration = { + queueName: 'reservations-service', + connection: { url: 'redis://localhost:6379' }, + // Longer than the slowest handler takes, or a message still being handled is received again + visibilityTimeoutMs: 30_000 +} +const redisTransport = new RedisTransport(redisConfiguration) + +const bus = Bus.configure() + .withMessageTypes(messageTypes) + .withTransport(redisTransport) + .withHandler(reserveRoomHandler) + // For local development: creates the queue's stream, its consumer group and its subscriptions when the bus + // initializes. In production, create them at deploy time with `bus provision` instead. + .withAutoProvision() + .build() + +await bus.initialize() +await bus.start() +``` + +The example uses top-level `await`, so it runs as an ES module. In CommonJS, wrap it in an `async` function. + +> **Set `maxmemory-policy noeviction` and turn on the append-only file in production.** Otherwise Redis can evict a queue when it runs out of memory, and a restart loses the messages written since the last snapshot. + +## Configuration + +| Option | Default | Description | +| ----------------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `queueName` | | The queue this service receives from. It can't contain `{` or `}`. | +| `connection` | `{}` (localhost) | node-redis' `createClient()` options, such as `url`, `username`, `password` and `socket` for TLS. | +| `keyPrefix` | `bus` | What every key starts with. Services that send each other messages must use the same prefix. | +| `visibilityTimeoutMs` | `30000` | How long a handler has, in milliseconds, before another receiver takes the message over and it's handled again. Set it above how long your slowest handler takes. | +| `deadLetterRetentionMs` | `1209600000` (14 d) | How long dead-lettered messages are kept, in milliseconds. `0` or `Infinity` keeps them until you remove them. | + +Retried messages wait in a sorted set until they're due, and then go back to the queue's stream. How many attempts a message gets, and how long it waits between them, is up to the bus' recoverability policy. Dead-lettered messages go to a stream of the queue's own, with why they failed in a `bus-failure` field. + +## Learn more + +- [Redis](https://node-ts.github.io/bus/transports/redis): the keys, what `bus provision` creates, the ACL rules it needs at runtime, keeping messages safe, and moving from `@node-ts/bus-redis` 0.x +- [Provisioning](https://node-ts.github.io/bus/guide/provisioning) +- [Recoverability](https://node-ts.github.io/bus/guide/recoverability) diff --git a/packages/bus-redis/package.json b/packages/bus-redis/package.json new file mode 100644 index 00000000..c0494f96 --- /dev/null +++ b/packages/bus-redis/package.json @@ -0,0 +1,76 @@ +{ + "name": "@node-ts/bus-redis", + "version": "0.0.0", + "description": "A Redis Streams transport adapter for @node-ts/bus-core.", + "homepage": "https://node-ts.github.io/bus/transports/redis", + "license": "MIT", + "engines": { + "node": ">=24" + }, + "type": "commonjs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + }, + "require": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "./package.json": "./package.json" + }, + "files": [ + "dist", + "CHANGELOG.md" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/node-ts/bus.git", + "directory": "packages/bus-redis" + }, + "bugs": { + "url": "https://github.com/node-ts/bus/issues" + }, + "scripts": { + "clean": "rm -rf dist", + "build": "tsc", + "build:watch": "pnpm run build --incremental --watch --preserveWatchOutput" + }, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@node-ts/bus-messages": "workspace:^", + "serialize-error": "8", + "tslib": "^2.8.1" + }, + "devDependencies": { + "@node-ts/bus-core": "workspace:^", + "@node-ts/bus-test": "workspace:^", + "redis": "^6.3.0", + "typemoq": "^2.1.0", + "typescript": "^6.0.3" + }, + "peerDependencies": { + "@node-ts/bus-core": "^1.3.3", + "redis": "^6.0.0" + }, + "keywords": [ + "esb", + "redis", + "valkey", + "redis streams", + "queue", + "transport", + "typescript", + "enterprise integration patterns", + "bus", + "messaging", + "microservices", + "distributed systems" + ] +} diff --git a/packages/bus-redis/src/error/index.ts b/packages/bus-redis/src/error/index.ts new file mode 100644 index 00000000..e00c74fc --- /dev/null +++ b/packages/bus-redis/src/error/index.ts @@ -0,0 +1,3 @@ +export * from './invalid-redis-key-name' +export * from './invalid-redis-transport-duration' +export * from './redis-transport-not-connected' diff --git a/packages/bus-redis/src/error/invalid-redis-key-name.ts b/packages/bus-redis/src/error/invalid-redis-key-name.ts new file mode 100644 index 00000000..150850a1 --- /dev/null +++ b/packages/bus-redis/src/error/invalid-redis-key-name.ts @@ -0,0 +1,23 @@ +/** + * Thrown when `RedisTransport` is configured with a `queueName` or `keyPrefix` it can't build Redis keys from, such as + * one that's empty or contains `{` or `}`. Every key of a queue shares the queue's name as a hash tag (`{}`), + * so braces in it would change which part of the key Redis Cluster hashes. + */ +export class InvalidRedisKeyName extends Error { + readonly help: string + + /** + * @param setting the configuration setting that's invalid + * @param value the value it was given + */ + constructor( + readonly setting: 'queueName' | 'keyPrefix', + readonly value: string + ) { + super( + `RedisTransport's ${setting} must be a non-empty string without "{" or "}", but was "${value}"` + ) + this.help = `Change ${setting} in the RedisTransport configuration to a name without braces, such as "order-booking-service"` + Object.setPrototypeOf(this, new.target.prototype) + } +} diff --git a/packages/bus-redis/src/error/invalid-redis-transport-duration.ts b/packages/bus-redis/src/error/invalid-redis-transport-duration.ts new file mode 100644 index 00000000..884d2cbd --- /dev/null +++ b/packages/bus-redis/src/error/invalid-redis-transport-duration.ts @@ -0,0 +1,27 @@ +/** + * Thrown when `RedisTransport` is configured with a duration it can't use: a `visibilityTimeoutMs` that isn't a + * positive, finite number of milliseconds, or a `deadLetterRetentionMs` that's negative or not a number + */ +export class InvalidRedisTransportDuration extends Error { + readonly help: string + + /** + * @param setting the configuration setting that's invalid + * @param value the value it was given + */ + constructor( + readonly setting: 'visibilityTimeoutMs' | 'deadLetterRetentionMs', + readonly value: unknown + ) { + super( + setting === 'visibilityTimeoutMs' + ? `RedisTransport's visibilityTimeoutMs must be a positive number of milliseconds, but was ${String(value)}` + : `RedisTransport's deadLetterRetentionMs must be 0 or more milliseconds, or Infinity, but was ${String(value)}` + ) + this.help = + setting === 'visibilityTimeoutMs' + ? 'Set visibilityTimeoutMs in the RedisTransport configuration to a number above 0, or leave it out to use the default' + : 'Set deadLetterRetentionMs in the RedisTransport configuration to how long to keep dead letters, 0 or Infinity to keep them until removed, or leave it out to use the default' + Object.setPrototypeOf(this, new.target.prototype) + } +} diff --git a/packages/bus-redis/src/error/redis-transport-not-connected.ts b/packages/bus-redis/src/error/redis-transport-not-connected.ts new file mode 100644 index 00000000..dcda7c05 --- /dev/null +++ b/packages/bus-redis/src/error/redis-transport-not-connected.ts @@ -0,0 +1,14 @@ +/** + * Thrown when `RedisTransport` is used before it has connected, such as when it sends before the bus that owns it + * has been initialized + */ +export class RedisTransportNotConnected extends Error { + readonly help: string + + constructor() { + super('RedisTransport is not connected to Redis') + this.help = + 'Initialize the bus that uses the transport (await bus.initialize()) before sending, or provision it with bus.provision(), which connects first' + Object.setPrototypeOf(this, new.target.prototype) + } +} diff --git a/packages/bus-redis/src/index.mts b/packages/bus-redis/src/index.mts new file mode 100644 index 00000000..e2c8de2d --- /dev/null +++ b/packages/bus-redis/src/index.mts @@ -0,0 +1,5 @@ +// The ESM entry point re-exports the CommonJS build instead of being a second +// build. `import` and `require` then share one module instance, so +// `instanceof` checks on error and message classes still work when an app +// loads this package both ways. +export * from './index.js' diff --git a/packages/bus-redis/src/index.ts b/packages/bus-redis/src/index.ts new file mode 100644 index 00000000..fe6d6564 --- /dev/null +++ b/packages/bus-redis/src/index.ts @@ -0,0 +1,4 @@ +export * from './error' +export * from './redis-transport' +export * from './redis-transport-configuration' +export * from './redis-transport-message' diff --git a/packages/bus-redis/src/redis-keys.spec.ts b/packages/bus-redis/src/redis-keys.spec.ts new file mode 100644 index 00000000..3ce4239f --- /dev/null +++ b/packages/bus-redis/src/redis-keys.spec.ts @@ -0,0 +1,49 @@ +import { InvalidRedisKeyName } from './error' +import { escapeGlob, isValidKeyName, RedisKeys } from './redis-keys' + +describe('RedisKeys', () => { + describe('when naming the keys of a queue', () => { + const sut = new RedisKeys('bus') + + it('should put the queue name in a hash tag', () => { + expect([ + sut.queue('orders'), + sut.delayed('orders'), + sut.deadLetter('orders') + ]).toEqual([ + 'bus:{orders}:queue', + 'bus:{orders}:delayed', + 'bus:{orders}:dead-letter' + ]) + }) + + it('should name the subscription set of a message', () => { + expect(sut.subscriptions('@app/order-placed')).toEqual( + 'bus:subscriptions:@app/order-placed' + ) + }) + }) + + describe('when constructed with a prefix that has braces', () => { + it('should throw InvalidRedisKeyName', () => { + expect(() => new RedisKeys('bus{')).toThrow(InvalidRedisKeyName) + }) + }) + + describe('when checking a name', () => { + it('should only accept non-empty names without braces', () => { + expect(['orders', '', 'a{b', 'a}b'].map(isValidKeyName)).toEqual([ + true, + false, + false, + false + ]) + }) + }) + + describe('when escaping a glob', () => { + it('should escape the characters a Redis glob treats as special', () => { + expect(escapeGlob('a*b?c[d]e\\f')).toEqual('a\\*b\\?c\\[d\\]e\\\\f') + }) + }) +}) diff --git a/packages/bus-redis/src/redis-keys.ts b/packages/bus-redis/src/redis-keys.ts new file mode 100644 index 00000000..04257862 --- /dev/null +++ b/packages/bus-redis/src/redis-keys.ts @@ -0,0 +1,91 @@ +import { InvalidRedisKeyName } from './error' + +/** + * The prefix of every key, unless `keyPrefix` is set + */ +export const DEFAULT_KEY_PREFIX = 'bus' + +const HASH_TAG_BRACES = /[{}]/ + +/** + * Whether a name can be put in a key, inside a hash tag: it's not empty and has no braces + */ +export const isValidKeyName = (name: string): boolean => + typeof name === 'string' && name.length > 0 && !HASH_TAG_BRACES.test(name) + +/** + * Escapes the characters that are special in a Redis glob pattern (`*`, `?`, `[`, `]` and `\`), so an ACL key pattern + * matches the name exactly + */ +export const escapeGlob = (name: string): string => + name.replace(/[*?[\]\\]/g, match => `\\${match}`) + +/** + * The names of the keys `RedisTransport` uses. Every key of a queue has the queue's name as its Redis Cluster hash tag, + * so they're all in one hash slot, and a script can update them together. + * + * - `:{}:queue`: the stream of messages waiting to be handled, read by the consumer group `` + * - `:{}:delayed`: a sorted set of returned messages, scored by when they're due + * - `:{}:dead-letter`: the stream of dead-lettered messages + * - `:subscriptions:`: the set of queues a message is sent to + */ +export class RedisKeys { + /** + * @param prefix what every key starts with + * @throws InvalidRedisKeyName if the prefix is empty or has braces + */ + constructor(readonly prefix: string) { + if (!isValidKeyName(prefix)) { + throw new InvalidRedisKeyName('keyPrefix', prefix) + } + } + + /** + * The stream a queue's messages are sent to + */ + queue(queueName: string): string { + return `${this.prefix}:{${queueName}}:queue` + } + + /** + * The sorted set a queue's returned messages wait in until they're due + */ + delayed(queueName: string): string { + return `${this.prefix}:{${queueName}}:delayed` + } + + /** + * The stream a queue's dead-lettered messages are kept in + */ + deadLetter(queueName: string): string { + return `${this.prefix}:{${queueName}}:dead-letter` + } + + /** + * The set of the queues a message is sent to + */ + subscriptions(messageName: string): string { + return `${this.prefix}:subscriptions:${messageName}` + } + + /** + * An ACL key pattern for every key of one queue + */ + queueKeysPattern(queueName: string): string { + return `${escapeGlob(this.prefix)}:{${escapeGlob(queueName)}}:*` + } + + /** + * An ACL key pattern for the message stream of every queue + */ + anyQueuePattern(): string { + return `${escapeGlob(this.prefix)}:{*}:queue` + } + + /** + * An ACL key pattern for every subscription set + */ + subscriptionsPattern(): string { + return `${escapeGlob(this.prefix)}:subscriptions:*` + } +} diff --git a/packages/bus-redis/src/redis-scripts.ts b/packages/bus-redis/src/redis-scripts.ts new file mode 100644 index 00000000..b57dfe85 --- /dev/null +++ b/packages/bus-redis/src/redis-scripts.ts @@ -0,0 +1,232 @@ +import { createHash } from 'node:crypto' + +/** + * A Lua script the transport runs with `EVALSHA`, loading it with `EVAL` the first time a server doesn't have it + */ +export interface RedisScript { + /** + * What the script is for, for logs + */ + name: string + /** + * The Lua source + */ + source: string + /** + * The SHA1 digest of the source, which `EVALSHA` runs it by + */ + sha: string +} + +const script = (name: string, source: string): RedisScript => ({ + name, + source, + sha: createHash('sha1').update(source).digest('hex') +}) + +/** + * Lua helpers every script that settles a message starts with. + * + * `isPendingTo` checks a stream entry is still pending to a consumer with the delivery count it was received with, so + * a receipt that another receiver has taken over (after its visibility timeout ended) settles nothing. + * + * `nowMs` is the server's clock in milliseconds, so every process schedules by the same clock. + * + * `integer` formats a number without an exponent, which Lua's own formatting uses for large numbers. + */ +const PRELUDE = ` +local function isPendingTo(stream, group, id, consumer, deliveries) + local pending = redis.call('XPENDING', stream, group, id, id, 1) + local entry = pending[1] + return entry ~= nil and entry[2] == consumer and tonumber(entry[4]) == tonumber(deliveries) +end +local function nowMs() + local time = redis.call('TIME') + return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +end +local function integer(value) + return string.format('%.0f', value) +end +local function fieldsFrom(first) + local fields = {} + for i = first, #ARGV do + fields[#fields + 1] = ARGV[i] + end + return fields +end +` + +/** + * Deletes a message that's been handled: acknowledges it and removes it from the stream, so streams don't grow. + * + * KEYS: queue stream. ARGV: group, consumer, entry id, deliveries. + * Returns 1 if it was deleted, or 0 if the receipt no longer holds it. + */ +export const DELETE_SCRIPT = script( + 'delete', + `${PRELUDE} +if not isPendingTo(KEYS[1], ARGV[1], ARGV[3], ARGV[2], ARGV[4]) then + return 0 +end +redis.call('XACK', KEYS[1], ARGV[1], ARGV[3]) +redis.call('XDEL', KEYS[1], ARGV[3]) +return 1 +` +) + +/** + * Returns a message to be handled again after a delay: a copy with the given fields (which count one more failed + * attempt) is added to the delayed set, scored by when it's due, or straight to the stream when there's no delay, and + * the received entry is acknowledged and removed. + * + * KEYS: queue stream, delayed set. ARGV: group, consumer, entry id, deliveries, delay in ms, then the copy's fields + * and values. Returns 1 if it was returned, or 0 if the receipt no longer holds it. + */ +export const RETURN_SCRIPT = script( + 'return', + `${PRELUDE} +if not isPendingTo(KEYS[1], ARGV[1], ARGV[3], ARGV[2], ARGV[4]) then + return 0 +end +local fields = fieldsFrom(6) +local delay = tonumber(ARGV[5]) +if delay <= 0 then + redis.call('XADD', KEYS[1], '*', unpack(fields)) +else + redis.call('ZADD', KEYS[2], integer(nowMs() + delay), cjson.encode({ id = ARGV[3], fields = fields })) +end +redis.call('XACK', KEYS[1], ARGV[1], ARGV[3]) +redis.call('XDEL', KEYS[1], ARGV[3]) +return 1 +` +) + +/** + * Dead-letters a message: adds a copy with the given fields (including `bus-failure`) to the dead letter stream, + * trimming entries older than the retention, and acknowledges and removes the received entry. + * + * KEYS: queue stream, dead letter stream. ARGV: group, consumer, entry id, deliveries, retention in ms (0 keeps + * everything), then the copy's fields and values. Returns 1 if it was dead-lettered, or 0 if the receipt no longer + * holds it. + */ +export const FAIL_SCRIPT = script( + 'fail', + `${PRELUDE} +if not isPendingTo(KEYS[1], ARGV[1], ARGV[3], ARGV[2], ARGV[4]) then + return 0 +end +local fields = fieldsFrom(6) +local retention = tonumber(ARGV[5]) +if retention > 0 then + local minId = nowMs() - retention + if minId < 0 then + minId = 0 + end + redis.call('XADD', KEYS[2], 'MINID', '~', integer(minId), '*', unpack(fields)) +else + redis.call('XADD', KEYS[2], '*', unpack(fields)) +end +redis.call('XACK', KEYS[1], ARGV[1], ARGV[3]) +redis.call('XDEL', KEYS[1], ARGV[3]) +return 1 +` +) + +/** + * Run before each read. It: + * + * 1. moves returned messages that are due from the delayed set back to the stream, + * 2. when a reclaim count is given, takes over messages that have been pending for at least the visibility timeout, + * such as those of a process that stopped, returning each with its new delivery count, and + * 3. when a consumer idle time is given, removes consumers other than this one that have nothing pending and haven't + * read for that long, which processes that stopped leave behind. One with nothing pending loses nothing. + * + * KEYS: queue stream, delayed set. ARGV: group, consumer, visibility timeout in ms, reclaim count (0 to skip), how + * many due messages to move at most, consumer idle time in ms (0 to skip). + * Returns { ms until the next returned message is due, or -1 if there's none, { { id, fields, deliveries }, ... } }. + */ +export const MAINTAIN_SCRIPT = script( + 'maintain', + `${PRELUDE} +local now = nowMs() +local due = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', integer(now), 'LIMIT', 0, tonumber(ARGV[5])) +for _, member in ipairs(due) do + local entry = cjson.decode(member) + redis.call('XADD', KEYS[1], '*', unpack(entry.fields)) + redis.call('ZREM', KEYS[2], member) +end + +local nextDue = -1 +local first = redis.call('ZRANGE', KEYS[2], 0, 0, 'WITHSCORES') +if first[2] then + nextDue = tonumber(first[2]) - now + if nextDue < 0 then + nextDue = 0 + end +end + +local claimed = {} +if tonumber(ARGV[4]) > 0 then + local pending = redis.call('XPENDING', KEYS[1], ARGV[1], 'IDLE', ARGV[3], '-', '+', ARGV[4]) + for _, entry in ipairs(pending) do + local entries = redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[2], ARGV[3], entry[1]) + local claim = entries[1] + -- An entry that was removed from the stream isn't claimed, and is dropped from the pending list + if claim and claim[2] then + claimed[#claimed + 1] = { claim[1], claim[2], tonumber(entry[4]) + 1 } + end + end +end + +if tonumber(ARGV[6]) > 0 then + local consumers = redis.call('XINFO', 'CONSUMERS', KEYS[1], ARGV[1]) + for _, consumer in ipairs(consumers) do + local info = {} + for i = 1, #consumer, 2 do + info[consumer[i]] = consumer[i + 1] + end + if info['name'] ~= ARGV[2] and tonumber(info['pending']) == 0 and tonumber(info['idle']) >= tonumber(ARGV[6]) then + redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], info['name']) + end + end +end + +return { integer(nextDue), claimed } +` +) + +/** + * Gives back a message that was received but not handled, because the transport stopped: it stays pending, with the + * delivery count it had before it was received, and idle for the visibility timeout, so another receiver takes it + * over on its next read without counting a failed attempt. + * + * KEYS: queue stream. ARGV: group, consumer, entry id, deliveries, visibility timeout in ms. + */ +export const RELEASE_SCRIPT = script( + 'release', + `${PRELUDE} +if not isPendingTo(KEYS[1], ARGV[1], ARGV[3], ARGV[2], ARGV[4]) then + return 0 +end +redis.call('XCLAIM', KEYS[1], ARGV[1], ARGV[2], 0, ARGV[3], 'IDLE', ARGV[5], 'RETRYCOUNT', tonumber(ARGV[4]) - 1, 'JUSTID') +return 1 +` +) + +/** + * Removes this consumer from the group when nothing is pending to it, so stopped processes don't leave consumers + * behind. One with messages pending is left for other receivers to take them over. + * + * KEYS: queue stream. ARGV: group, consumer. Returns 1 if it was removed. + */ +export const LEAVE_SCRIPT = script( + 'leave', + ` +local pending = redis.call('XPENDING', KEYS[1], ARGV[1], '-', '+', 1, ARGV[2]) +if #pending == 0 then + redis.call('XGROUP', 'DELCONSUMER', KEYS[1], ARGV[1], ARGV[2]) + return 1 +end +return 0 +` +) diff --git a/packages/bus-redis/src/redis-transport-configuration.ts b/packages/bus-redis/src/redis-transport-configuration.ts new file mode 100644 index 00000000..c6a0948a --- /dev/null +++ b/packages/bus-redis/src/redis-transport-configuration.ts @@ -0,0 +1,58 @@ +import { TransportConfiguration } from '@node-ts/bus-core' +import { RedisClientOptions } from 'redis' + +/** + * How to connect to Redis: the options of node-redis' `createClient()`, such as `url`, `username`, `password`, + * `database` and `socket` (for TLS). The transport always speaks RESP2, so `RESP` can't be set. + * @example { url: 'redis://localhost:6379' } + */ +export type RedisConnectionOptions = Omit< + RedisClientOptions, + 'RESP' | 'modules' | 'functions' | 'scripts' | 'typeMapping' +> + +/** + * Configures a `RedisTransport`. Each queue keeps its dead letters in a stream of its own, so there's no dead letter + * queue to name. + */ +export interface RedisTransportConfiguration extends Omit< + TransportConfiguration, + 'deadLetterQueueName' +> { + /** + * The name of the queue this service receives from, which is its endpoint name. It's part of every key of the + * queue, as a Redis Cluster hash tag, so it can't contain `{` or `}`. + * @example order-booking-service + */ + queueName: string + + /** + * How to connect to Redis. The transport makes a connection for sending and settling messages, and another for + * waiting for new messages. + * @default {} (localhost:6379) + * @example { url: process.env.REDIS_URL } + */ + connection?: RedisConnectionOptions + + /** + * What every key the transport uses starts with. Services that send each other messages must use the same prefix. + * It can't contain `{` or `}`. + * @default bus + */ + keyPrefix?: string + + /** + * How long a message that's been received is left with its receiver while it's handled, in milliseconds. If it's + * neither deleted, returned nor dead-lettered by then, such as when the process stopped, another receiver takes it + * over and it's handled again, counting a failed attempt. Set it longer than your slowest handler takes. + * @default 30000 + */ + visibilityTimeoutMs?: number + + /** + * How long dead-lettered messages are kept, in milliseconds. Older ones are trimmed, roughly, as new ones are + * dead-lettered. `0` or `Infinity` keeps them until they're removed. + * @default 1209600000 (14 days) + */ + deadLetterRetentionMs?: number +} diff --git a/packages/bus-redis/src/redis-transport-message.ts b/packages/bus-redis/src/redis-transport-message.ts new file mode 100644 index 00000000..29ef44f5 --- /dev/null +++ b/packages/bus-redis/src/redis-transport-message.ts @@ -0,0 +1,40 @@ +import { TransportHeaders } from '@node-ts/bus-core' +import { MessageAttributes } from '@node-ts/bus-messages' + +/** + * A message as `RedisTransport` reads it from its queue's stream: the raw message of a `TransportMessage` + */ +export interface RedisTransportMessage { + /** + * The stream entry's id. A returned message is added to the stream again, so it gets a new one each retry. + * @example 1717171717171-0 + */ + id: string + + /** + * The message, as the bus' serializer wrote it + */ + body: string + + /** + * The attributes it was sent with + */ + attributes: MessageAttributes + + /** + * The native headers set by outgoing middleware + */ + headers: TransportHeaders + + /** + * How many times handling it failed before it was added to the stream again, from its `failedAttempts` field + */ + failedAttemptsBefore: number + + /** + * How many times this stream entry has been delivered, including this time. With the consumer that received it, + * it identifies this receipt: the message is only deleted, returned or dead-lettered while it's still pending to + * that consumer with this count, so a receipt that another receiver has taken over settles nothing. + */ + deliveries: number +} diff --git a/packages/bus-redis/src/redis-transport.integration.ts b/packages/bus-redis/src/redis-transport.integration.ts new file mode 100644 index 00000000..5a843837 --- /dev/null +++ b/packages/bus-redis/src/redis-transport.integration.ts @@ -0,0 +1,632 @@ +import { + Bus, + BusInstance, + deadLetter, + DefaultHandlerRegistry, + EndpointNotFound, + FAILURE_HEADER, + fromFailureHeader, + handlerFor, + JsonSerializer, + Logger, + MessageFailure, + MessageSerializer, + ProvisioningPlan, + RecoverabilityPolicy, + ResourcesNotProvisioned, + retry +} from '@node-ts/bus-core' +import { Message, MessageAttributes } from '@node-ts/bus-messages' +import { + messageTypes, + TestCommand, + TestSystemMessage, + transportTests +} from '@node-ts/bus-test' +import { randomUUID } from 'node:crypto' +import { createClient } from 'redis' +import { Mock } from 'typemoq' +import { RedisTransport } from './redis-transport' +import { RedisTransportConfiguration } from './redis-transport-configuration' + +jest.setTimeout(30_000) + +const connection = { + url: process.env.REDIS_URL || 'redis://localhost:6380' +} + +// Every key this run makes starts with it, so the run only ever touches its own keys +const keyPrefix = `node-ts-bus-test-${randomUUID()}` + +const configuration: RedisTransportConfiguration = { + queueName: '@node-ts/bus-redis-transport-test', + keyPrefix, + connection +} + +const silentLogger = () => Mock.ofType().object + +/** + * Moves a queue's dead letters back to it, as the docs show + */ +const REDRIVE_SCRIPT = ` +local entries = redis.call('XRANGE', KEYS[1], '-', '+', 'COUNT', 1000) +for _, entry in ipairs(entries) do + local fields = {} + for i = 1, #entry[2], 2 do + if entry[2][i] ~= 'bus-failure' then + fields[#fields + 1] = entry[2][i] + fields[#fields + 1] = entry[2][i + 1] + end + end + redis.call('XADD', KEYS[2], '*', unpack(fields)) + redis.call('XDEL', KEYS[1], entry[1]) +end +return #entries +` + +/** + * Turns a stream entry's fields and values, one after the other, into an object + */ +const toFields = (values: string[]): Record => { + const fields: Record = {} + for (let i = 0; i < values.length; i += 2) { + fields[values[i]] = values[i + 1] + } + return fields +} + +describe('RedisTransport', () => { + const redis = createClient({ ...connection, RESP: 2 }) + const transport = new RedisTransport(configuration) + const messageSerializer = new MessageSerializer( + new JsonSerializer(), + new DefaultHandlerRegistry(), + { messages: {}, types: {} } + ) + const queueKey = (queueName: string) => `${keyPrefix}:{${queueName}}:queue` + const deadLetterKey = (queueName: string) => + `${keyPrefix}:{${queueName}}:dead-letter` + + const publishSystemMessage = async (systemMessage: string) => { + const queues = await redis.sendCommand([ + 'SMEMBERS', + `${keyPrefix}:subscriptions:${TestSystemMessage.NAME}` + ]) + for (const queue of queues) { + await redis.sendCommand([ + 'XADD', + queueKey(queue), + '*', + 'body', + JSON.stringify(new TestSystemMessage()), + 'attributes', + JSON.stringify({ + messageId: randomUUID(), + attributes: { systemMessage }, + stickyAttributes: {} + }), + 'headers', + '{}' + ]) + } + } + + /** + * Waits for a dead letter in a queue's dead letter stream, then reads and removes them all + */ + const takeDeadLetters = async (queueName: string) => { + const key = deadLetterKey(queueName) + while ((await redis.sendCommand(['XLEN', key])) === 0) { + await new Promise(resolve => setTimeout(resolve, 50)) + } + const entries = await redis.sendCommand<[string, string[]][]>([ + 'XRANGE', + key, + '-', + '+' + ]) + await redis.sendCommand(['XDEL', key, ...entries.map(([id]) => id)]) + return entries.map(([, values]) => toFields(values)) + } + + const readAllFromDeadLetterQueue = async () => + (await takeDeadLetters(configuration.queueName)).map(fields => ({ + message: messageSerializer.deserialize(fields.body) as Message, + attributes: JSON.parse(fields.attributes) as MessageAttributes, + failure: fromFailureHeader(fields[FAILURE_HEADER]) + })) + + beforeAll(async () => { + await redis.connect() + }) + + afterAll(async () => { + // Removes only this run's keys. Never FLUSHDB: the server may be shared. + let cursor = '0' + do { + const [next, keys] = await redis.sendCommand<[string, string[]]>([ + 'SCAN', + cursor, + 'MATCH', + `${keyPrefix}:*`, + 'COUNT', + '1000' + ]) + cursor = next + if (keys.length) { + await redis.sendCommand(['UNLINK', ...keys]) + } + } while (cursor !== '0') + await redis.close() + }) + + transportTests( + transport, + publishSystemMessage, + TestSystemMessage.NAME, + readAllFromDeadLetterQueue + ) + + describe('when reading the endpoint name', () => { + it('should be the queue name', () => { + expect(transport.endpointName).toEqual(configuration.queueName) + }) + }) + + describe('when a bus initializes before its resources are provisioned', () => { + const queueName = 'unprovisioned' + let error: unknown + + beforeAll(async () => { + const bus = Bus.configure() + .withLogger(silentLogger) + .withMessageTypes(messageTypes) + .withTransport(new RedisTransport({ ...configuration, queueName })) + .withHandler(handlerFor(TestCommand, () => undefined)) + .build() + try { + await bus.initialize() + } catch (e) { + error = e + } + await bus.dispose() + }) + + it('should fail with ResourcesNotProvisioned, naming the missing stream, group and subscription', () => { + expect(error).toBeInstanceOf(ResourcesNotProvisioned) + expect((error as ResourcesNotProvisioned).missingResources).toEqual([ + `Redis stream ${queueKey(queueName)}`, + `Redis consumer group ${queueName} on ${queueKey(queueName)}`, + `Redis subscription ${TestCommand.NAME} -> ${queueName}` + ]) + }) + }) + + describe('when provisioning as a dry run', () => { + const queueName = 'dry-run' + let plan: ProvisioningPlan[] + let streamExists: number + + beforeAll(async () => { + const bus = Bus.configure() + .withLogger(silentLogger) + .withMessageTypes(messageTypes) + .withTransport(new RedisTransport({ ...configuration, queueName })) + .withHandler(handlerFor(TestCommand, () => undefined)) + .build() + plan = await bus.provision({ dryRun: true }) + await bus.dispose() + streamExists = await redis.sendCommand([ + 'EXISTS', + queueKey(queueName) + ]) + }) + + it('should create nothing', () => { + expect(streamExists).toEqual(0) + }) + + it('should plan the stream, consumer group and subscription', () => { + const transportPlan = plan.find( + ({ adapter }) => adapter === 'RedisTransport' + )! + expect(transportPlan.resources).toEqual( + expect.arrayContaining([ + { + type: 'redis-consumer-group', + name: queueName, + properties: { stream: queueKey(queueName), startId: '0' } + }, + { + type: 'redis-subscription', + name: `${TestCommand.NAME} -> ${queueName}`, + properties: { + set: `${keyPrefix}:subscriptions:${TestCommand.NAME}`, + member: queueName + } + } + ]) + ) + }) + }) + + describe('when provisioning twice', () => { + it('should succeed both times', async () => { + const queueName = 'provisioned-twice' + for (let i = 0; i < 2; i++) { + const bus = Bus.configure() + .withLogger(silentLogger) + .withMessageTypes(messageTypes) + .withTransport(new RedisTransport({ ...configuration, queueName })) + .withHandler(handlerFor(TestCommand, () => undefined)) + .build() + await bus.provision() + await bus.dispose() + } + }) + }) + + describe('when receiving', () => { + /** + * Builds, initializes and starts a bus with its own queue, which records the failed attempts and time of each + * receipt + */ + const startBus = async ( + queueName: string, + transportConfiguration: Partial, + handle: (command: TestCommand) => Promise, + options: { + concurrency?: number + receipts?: { failedAttempts: number; at: number }[] + recoverability?: RecoverabilityPolicy + } = {} + ): Promise => { + const configure = Bus.configure() + .withLogger(silentLogger) + .withMessageTypes(messageTypes) + .withTransport( + new RedisTransport({ + ...configuration, + queueName, + ...transportConfiguration + }) + ) + .withConcurrency(options.concurrency ?? 1) + .withAutoProvision() + .withMiddleware({ + incoming: async (context, next) => { + options.receipts?.push({ + failedAttempts: context.transportMessage.failedAttempts, + at: Date.now() + }) + await next() + } + }) + .withHandler(handlerFor(TestCommand, async command => handle(command))) + if (options.recoverability) { + configure.withRecoverability(options.recoverability) + } + const bus = configure.build() + await bus.initialize() + await bus.start() + return bus + } + + describe('and a message is not settled before its visibility timeout ends', () => { + const receipts: { failedAttempts: number; at: number }[] = [] + let bus: BusInstance + let pending: number + + beforeAll(async () => { + const queueName = 'visibility-timeout' + const secondReceipt = Promise.withResolvers() + const firstHandled = Promise.withResolvers() + bus = await startBus( + queueName, + { visibilityTimeoutMs: 200 }, + async () => { + if (receipts.length === 1) { + // Still handling the first receipt when the second arrives + await secondReceipt.promise + firstHandled.resolve() + } else { + secondReceipt.resolve() + } + }, + { concurrency: 2, receipts } + ) + await bus.send(new TestCommand(randomUUID(), new Date())) + await firstHandled.promise + // The second receipt deleted it. The first receipt's delete settles nothing, since it was taken over. + while ( + (await redis.sendCommand(['XLEN', queueKey(queueName)])) > 0 + ) { + await new Promise(resolve => setTimeout(resolve, 50)) + } + const [count] = await redis.sendCommand<[number]>([ + 'XPENDING', + queueKey(queueName), + queueName + ]) + pending = count + }) + + afterAll(async () => bus.dispose()) + + it('should receive it again, counting the expired receipt as a failed attempt', () => { + expect(receipts.map(({ failedAttempts }) => failedAttempts)).toEqual([ + 0, 1 + ]) + }) + + it('should leave nothing pending', () => { + expect(pending).toEqual(0) + }) + }) + + describe('and a message is returned with a delay', () => { + const RETRY_DELAY = 1_500 + const receipts: { failedAttempts: number; at: number }[] = [] + let bus: BusInstance + + beforeAll(async () => { + const handledTwice = Promise.withResolvers() + bus = await startBus( + 'returned-with-delay', + {}, + async () => { + if (receipts.length === 1) { + throw new Error('Fails the first time') + } + handledTwice.resolve() + }, + { receipts, recoverability: () => retry(RETRY_DELAY) } + ) + await bus.send(new TestCommand(randomUUID(), new Date())) + await handledTwice.promise + }) + + afterAll(async () => bus.dispose()) + + it('should receive it again once the delay has passed', () => { + expect(receipts[1].at - receipts[0].at).toBeGreaterThanOrEqual( + RETRY_DELAY - 50 + ) + }) + + it('should count the failed attempt', () => { + expect(receipts.map(({ failedAttempts }) => failedAttempts)).toEqual([ + 0, 1 + ]) + }) + }) + + describe('and the bus is disposed once its messages are handled', () => { + const queueName = 'leaves-group' + let consumers: unknown[] + + beforeAll(async () => { + const handled = Promise.withResolvers() + const bus = await startBus(queueName, {}, async () => handled.resolve()) + await bus.send(new TestCommand(randomUUID(), new Date())) + await handled.promise + await bus.dispose() + consumers = await redis.sendCommand([ + 'XINFO', + 'CONSUMERS', + queueKey(queueName), + queueName + ]) + }) + + it('should leave the consumer group', () => { + expect(consumers).toEqual([]) + }) + }) + + describe('and a dead-lettered message is moved back to the queue', () => { + const queueName = 'redrive' + const receipts: { failedAttempts: number; at: number }[] = [] + let bus: BusInstance + let redriven: number + + beforeAll(async () => { + const handled = Promise.withResolvers() + let fixed = false + bus = await startBus( + queueName, + {}, + async () => { + if (!fixed) { + throw new Error('Fails until the cause is fixed') + } + handled.resolve() + }, + { receipts, recoverability: () => deadLetter() } + ) + await bus.send(new TestCommand(randomUUID(), new Date())) + // Waits for it to be dead-lettered, without removing it + while ( + (await redis.sendCommand([ + 'XLEN', + deadLetterKey(queueName) + ])) === 0 + ) { + await new Promise(resolve => setTimeout(resolve, 50)) + } + fixed = true + redriven = await redis.sendCommand([ + 'EVAL', + REDRIVE_SCRIPT, + '2', + deadLetterKey(queueName), + queueKey(queueName) + ]) + await handled.promise + }) + + afterAll(async () => bus.dispose()) + + it('should move it with the script the docs show', () => { + expect(redriven).toEqual(1) + }) + + it('should handle it again with all its attempts', () => { + expect(receipts.map(({ failedAttempts }) => failedAttempts)).toEqual([ + 0, 0 + ]) + }) + }) + + describe('and a message cannot be parsed', () => { + const queueName = 'unparseable' + let bus: BusInstance + let remaining: number + let failure: MessageFailure | undefined + + beforeAll(async () => { + bus = await startBus(queueName, {}, async () => undefined) + await redis.sendCommand([ + 'XADD', + queueKey(queueName), + '*', + 'body', + 'not json', + 'attributes', + '{}', + 'headers', + '{}' + ]) + const [deadLetter] = await takeDeadLetters(queueName) + failure = fromFailureHeader(deadLetter[FAILURE_HEADER]) + remaining = await redis.sendCommand([ + 'XLEN', + queueKey(queueName) + ]) + }) + + afterAll(async () => bus.dispose()) + + it('should remove it from the queue', () => { + expect(remaining).toEqual(0) + }) + + it('should move it to the dead letter stream with the parse error', () => { + expect(failure!.failedAttempts).toEqual(1) + expect(failure!.endpoint).toEqual(queueName) + }) + }) + }) + + describe('when sending to an address with no queue', () => { + let error: unknown + + beforeAll(async () => { + const sender = new RedisTransport({ + ...configuration, + queueName: 'address-sender' + }) + sender.prepare({ + loggerFactory: silentLogger, + messageSerializer + } as never) + await sender.connect() + try { + await sender.sendToAddress( + 'no-such-queue', + new TestCommand(randomUUID(), new Date()) + ) + } catch (e) { + error = e + } + await sender.dispose() + }) + + it('should throw EndpointNotFound', () => { + expect(error).toBeInstanceOf(EndpointNotFound) + }) + + it('should not create a stream for it', async () => { + expect( + await redis.sendCommand(['EXISTS', queueKey('no-such-queue')]) + ).toEqual(0) + }) + }) + + describe('when the bus runs as a user with only the runtime permissions in its plan', () => { + const queueName = 'acl' + const username = `node-ts-bus-test-${randomUUID()}` + const password = randomUUID() + const receipts: { failedAttempts: number; at: number }[] = [] + let deadLetters: Record[] + + beforeAll(async () => { + const busConfiguration = () => + Bus.configure() + .withLogger(silentLogger) + .withMessageTypes(messageTypes) + .withHandler( + handlerFor(TestCommand, async command => { + receipts.push({ failedAttempts: 0, at: Date.now() }) + if (command.value === 'fail') { + throw new Error('Fails every time') + } + }) + ) + .withRecoverability(({ failedAttempts }) => + failedAttempts < 2 ? retry(0) : deadLetter() + ) + + // Provisioned with the default user, as a deploy would + const deploy = busConfiguration() + .withTransport(new RedisTransport({ ...configuration, queueName })) + .build() + const [plan] = await deploy.provision() + await deploy.dispose() + const { rules } = plan.runtimePermissions!.document as { + rules: string[] + } + await redis.sendCommand([ + 'ACL', + 'SETUSER', + username, + 'on', + `>${password}`, + 'resetkeys', + 'resetchannels', + '-@all', + ...rules + ]) + + const bus = busConfiguration() + .withTransport( + new RedisTransport({ + ...configuration, + queueName, + connection: { ...connection, username, password } + }) + ) + .build() + await bus.initialize() + await bus.start() + await bus.send(new TestCommand('succeed', new Date())) + await bus.send(new TestCommand('fail', new Date())) + deadLetters = await takeDeadLetters(queueName) + await bus.dispose() + }) + + afterAll(async () => { + await redis.sendCommand(['ACL', 'DELUSER', username]) + }) + + it('should receive, retry and dead-letter messages', () => { + // One receipt of the message that succeeds, and two of the one that fails + expect(receipts).toHaveLength(3) + expect(deadLetters).toHaveLength(1) + expect(fromFailureHeader(deadLetters[0][FAILURE_HEADER])).toMatchObject({ + failedAttempts: 2 + }) + }) + }) +}) diff --git a/packages/bus-redis/src/redis-transport.spec.ts b/packages/bus-redis/src/redis-transport.spec.ts new file mode 100644 index 00000000..26230da3 --- /dev/null +++ b/packages/bus-redis/src/redis-transport.spec.ts @@ -0,0 +1,266 @@ +import { + CoreDependencies, + DefaultHandlerRegistry, + FAILURE_HEADER, + HandlerRegistry, + JsonSerializer, + Logger, + MessageSerializer, + ProvisioningPlan, + TransportHeaderReserved +} from '@node-ts/bus-core' +import { Command } from '@node-ts/bus-messages' +import { Mock } from 'typemoq' +import { + InvalidRedisKeyName, + InvalidRedisTransportDuration, + RedisTransportNotConnected +} from './error' +import { RedisTransport } from './redis-transport' + +class TestTransportCommand extends Command { + static NAME = '@node-ts/bus-redis/test-transport-command' + $name = TestTransportCommand.NAME + $version = 0 +} + +const configuration = { + queueName: 'orders', + connection: {} +} + +const handlerRegistryFor = ( + messageNames: string[], + externalTopics: string[] = [] +): HandlerRegistry => { + const registry = Mock.ofType() + registry.setup(r => r.getMessageNames()).returns(() => messageNames) + registry + .setup(r => r.getExternallyManagedTopicIdentifiers()) + .returns(() => externalTopics) + return registry.object +} + +describe('RedisTransport', () => { + let sut: RedisTransport + + const prepare = () => { + sut.prepare({ + loggerFactory: () => Mock.ofType().object, + messageSerializer: new MessageSerializer( + new JsonSerializer(), + new DefaultHandlerRegistry(), + { messages: {}, types: {} } + ) + } as unknown as CoreDependencies) + } + + describe('when a receiving bus provisions as a dry run', () => { + let plan: ProvisioningPlan + + beforeAll(async () => { + sut = new RedisTransport({ ...configuration, keyPrefix: 'app*' }) + prepare() + // Not connected, so it would throw RedisTransportNotConnected if it called Redis + plan = await sut.provision({ + handlerRegistry: handlerRegistryFor( + [TestTransportCommand.NAME], + ['external-topic'] + ), + sendOnly: false, + messageNames: [TestTransportCommand.NAME], + sendsAnyMessage: false, + dryRun: true + }) + }) + + it('should plan the stream, consumer group and a subscription per handled message and custom topic', () => { + expect(plan.resources.map(({ type, name }) => `${type} ${name}`)).toEqual( + [ + 'redis-stream app*:{orders}:queue', + 'redis-consumer-group orders', + `redis-subscription ${TestTransportCommand.NAME} -> orders`, + 'redis-subscription external-topic -> orders' + ] + ) + }) + + it('should list the ACL rules it needs at runtime, with key patterns escaped', () => { + expect(plan.runtimePermissions).toEqual({ + format: 'redis-acl', + document: { + user: '', + rules: [ + '~app\\*:{orders}:*', + '%W~app\\*:{*}:queue', + '%R~app\\*:subscriptions:*', + '+xadd', + '+smembers', + '+multi', + '+exec', + '+xreadgroup', + '+xack', + '+xdel', + '+xpending', + '+xclaim', + '+xinfo|groups', + '+xinfo|consumers', + '+xgroup|delconsumer', + '+zadd', + '+zrange', + '+zrangebyscore', + '+zrem', + '+sismember', + '+time', + '+evalsha', + '+eval' + ] + } + }) + }) + }) + + describe('when a send-only bus provisions', () => { + let plan: ProvisioningPlan + + beforeAll(async () => { + sut = new RedisTransport(configuration) + prepare() + // Not a dry run, but a send-only bus has nothing to create, so it doesn't call Redis + plan = await sut.provision({ + handlerRegistry: handlerRegistryFor([]), + sendOnly: true, + messageNames: [TestTransportCommand.NAME], + sendsAnyMessage: true, + dryRun: false + }) + }) + + it('should plan no resources', () => { + expect(plan.resources).toEqual([]) + }) + + it('should only allow sending at runtime', () => { + expect(plan.runtimePermissions?.document).toEqual({ + user: '', + rules: [ + '%W~bus:{*}:queue', + '%R~bus:subscriptions:*', + '+xadd', + '+smembers', + '+multi', + '+exec' + ] + }) + }) + }) + + describe('when constructed with a queue name or key prefix it cannot use', () => { + const errors: unknown[] = [] + + beforeAll(() => { + for (const settings of [ + { queueName: '' }, + { queueName: 'orders{1}' }, + { keyPrefix: 'bus}' } + ]) { + try { + new RedisTransport({ ...configuration, ...settings }) + } catch (error) { + errors.push(error) + } + } + }) + + it('should throw InvalidRedisKeyName naming the setting', () => { + expect(errors).toHaveLength(3) + errors.forEach(error => expect(error).toBeInstanceOf(InvalidRedisKeyName)) + expect((errors as InvalidRedisKeyName[]).map(e => e.setting)).toEqual([ + 'queueName', + 'queueName', + 'keyPrefix' + ]) + }) + }) + + describe('when constructed with a duration it cannot use', () => { + const errors: unknown[] = [] + + beforeAll(() => { + for (const settings of [ + { visibilityTimeoutMs: 0 }, + { visibilityTimeoutMs: Infinity }, + { deadLetterRetentionMs: -1 }, + { deadLetterRetentionMs: Number.NaN } + ]) { + try { + new RedisTransport({ ...configuration, ...settings }) + } catch (error) { + errors.push(error) + } + } + }) + + it('should throw InvalidRedisTransportDuration naming the setting', () => { + expect( + (errors as InvalidRedisTransportDuration[]).map(e => e.setting) + ).toEqual([ + 'visibilityTimeoutMs', + 'visibilityTimeoutMs', + 'deadLetterRetentionMs', + 'deadLetterRetentionMs' + ]) + }) + }) + + describe('when constructed with a dead letter retention of 0 or Infinity', () => { + it('should accept it', () => { + expect( + () => new RedisTransport({ ...configuration, deadLetterRetentionMs: 0 }) + ).not.toThrow() + expect( + () => + new RedisTransport({ + ...configuration, + deadLetterRetentionMs: Infinity + }) + ).not.toThrow() + }) + }) + + describe('when checking send options', () => { + beforeAll(() => { + sut = new RedisTransport(configuration) + }) + + it('should reject the bus-failure header', () => { + expect(() => + sut.assertSendOptions({ headers: { [FAILURE_HEADER]: 'x' } }) + ).toThrow(TransportHeaderReserved) + }) + + it('should accept other headers', () => { + expect(() => + sut.assertSendOptions({ headers: { 'x-tenant': 'a' } }) + ).not.toThrow() + }) + }) + + describe('when sending before it has connected', () => { + let error: unknown + + beforeAll(async () => { + sut = new RedisTransport(configuration) + prepare() + try { + await sut.send(new TestTransportCommand()) + } catch (e) { + error = e + } + }) + + it('should throw RedisTransportNotConnected', () => { + expect(error).toBeInstanceOf(RedisTransportNotConnected) + }) + }) +}) diff --git a/packages/bus-redis/src/redis-transport.ts b/packages/bus-redis/src/redis-transport.ts new file mode 100644 index 00000000..55ecbc79 --- /dev/null +++ b/packages/bus-redis/src/redis-transport.ts @@ -0,0 +1,1130 @@ +import { + CoreDependencies, + createMessageFailure, + EndpointNotFound, + FAILURE_HEADER, + Logger, + MessageFailure, + Milliseconds, + ProvisionedResource, + ProvisioningPlan, + ResourcesNotProvisioned, + toFailureHeader, + Transport, + TransportHeaderReserved, + TransportHeaders, + TransportInitializationOptions, + TransportMessage, + TransportProvisionOptions, + TransportSendOptions +} from '@node-ts/bus-core' +import { + Command, + Event, + Message, + MessageAttributes +} from '@node-ts/bus-messages' +import { randomUUID } from 'node:crypto' +import { hostname } from 'node:os' +import { createClient } from 'redis' +import { serializeError } from 'serialize-error' +import { + InvalidRedisKeyName, + InvalidRedisTransportDuration, + RedisTransportNotConnected +} from './error' +import { DEFAULT_KEY_PREFIX, isValidKeyName, RedisKeys } from './redis-keys' +import { + DELETE_SCRIPT, + FAIL_SCRIPT, + LEAVE_SCRIPT, + MAINTAIN_SCRIPT, + RedisScript, + RELEASE_SCRIPT, + RETURN_SCRIPT +} from './redis-scripts' +import { + RedisConnectionOptions, + RedisTransportConfiguration +} from './redis-transport-configuration' +import { RedisTransportMessage } from './redis-transport-message' + +/** + * How long a received message is left with its receiver before another takes it over, unless `visibilityTimeoutMs` + * is set + */ +export const DEFAULT_VISIBILITY_TIMEOUT_MS: Milliseconds = 30_000 + +/** + * How long dead-lettered messages are kept, unless `deadLetterRetentionMs` is set: 14 days + */ +export const DEFAULT_DEAD_LETTER_RETENTION_MS: Milliseconds = + 14 * 24 * 60 * 60 * 1_000 + +/** + * The longest a read waits on the server for a message. It's kept well below node-redis' default command timeout + * (5 s), and it's the longest `stop()` waits for a read in flight. + */ +const READ_BLOCK_MS = 1_000 + +/** + * How long the reading connection waits for the answer to a read, beyond how long the read blocks for + */ +const READ_TIMEOUT_MARGIN_MS = 10_000 + +/** + * How often messages left pending past the visibility timeout are looked for + */ +const RECLAIM_INTERVAL_MS = 1_000 + +/** + * How often consumers that stopped processes left behind are looked for, and how long one must have been idle, with + * nothing pending, to be removed + */ +const CONSUMER_CLEANUP_INTERVAL_MS = 60_000 +const CONSUMER_IDLE_MS = 24 * 60 * 60 * 1_000 + +/** + * The most returned messages moved from the delayed set to the stream before each read + */ +const DUE_MESSAGES_PER_READ = 100 + +/** + * The fields of a stream entry + */ +const BODY_FIELD = 'body' +const ATTRIBUTES_FIELD = 'attributes' +const HEADERS_FIELD = 'headers' +const FAILED_ATTEMPTS_FIELD = 'failedAttempts' + +/** + * A stream entry as Redis returns it: its id and its fields and values, one after the other + */ +type RawStreamEntry = [id: string, fields: string[] | null] + +/** + * A message the transport received and hasn't given to the bus yet + */ +interface ReceivedEntry { + id: string + fields: string[] + deliveries: number +} + +/** + * A read waiting for a message + */ +interface Waiter { + resolve: ( + message: TransportMessage | undefined + ) => void + reject: (error: unknown) => void +} + +/** + * Creates a client that speaks RESP2, so replies have the same shape whatever the client's defaults + */ +const createRedisClient = (options: RedisConnectionOptions) => + createClient({ ...options, RESP: 2 }) + +type RedisClient = ReturnType + +/** + * Checks that no header set by outgoing middleware is one the transport writes itself + * @throws TransportHeaderReserved if one is + */ +const assertHeadersNotReserved = (headers: TransportHeaders): void => { + if (Object.hasOwn(headers, FAILURE_HEADER)) { + throw new TransportHeaderReserved(FAILURE_HEADER, 'RedisTransport') + } +} + +/** + * Reads the fields of a stream entry, given as field and value one after the other, into an object + */ +const toFieldMap = (fields: string[]): Record => { + const map: Record = {} + for (let i = 0; i + 1 < fields.length; i += 2) { + map[fields[i]] = fields[i + 1] + } + return map +} + +/** + * Parses a field holding JSON, or returns `undefined` when there's none or it isn't JSON + */ +const parseJsonField = (value: string | undefined): T | undefined => { + if (value === undefined) { + return undefined + } + try { + return JSON.parse(value) as T + } catch { + return undefined + } +} + +const isNoGroupError = (error: unknown): boolean => + error instanceof Error && error.message.startsWith('NOGROUP') + +/** + * A transport that keeps each service's queue in a Redis Stream, read by a consumer group named after the queue, so + * every instance of a service shares its messages. Published events and sent commands are added to the stream of + * every queue subscribed to their name, which is kept in a set per message name. + * + * A received message stays pending to its receiver until it's deleted, returned or dead-lettered. If that doesn't + * happen within `visibilityTimeoutMs`, such as when the process stops, another receiver takes it over. Returned + * messages wait in a sorted set until they're due, and dead-lettered ones go to a stream of the queue's own. + * + * It needs Redis 7.0 or later, or Valkey 7.2 or later, run as a single primary (not Redis Cluster or Sentinel), with + * `maxmemory-policy noeviction`. Its consumer group and subscriptions are created by `provision()`, at deploy time or + * with `withAutoProvision()`. + * @example + * const transport = new RedisTransport({ + * queueName: 'order-booking-service', + * connection: { url: process.env.REDIS_URL } + * }) + */ +export class RedisTransport implements Transport { + /** + * The name of the queue the transport receives from, from `queueName` + */ + readonly endpointName: string + + private coreDependencies: CoreDependencies + private logger: Logger + private readonly keys: RedisKeys + private readonly visibilityTimeoutMs: Milliseconds + private readonly deadLetterRetentionMs: Milliseconds + /** + * The name this process reads the queue's consumer group as. It's unique to each transport, so the messages pending + * to it are only its own. + */ + private readonly consumerName = `${hostname()}-${process.pid}-${randomUUID()}` + private client: RedisClient | undefined + private reader: RedisClient | undefined + private isStarted = false + /** + * Whether the transport has received as a consumer of the queue's group, so it leaves the group when it disconnects + */ + private hasStarted = false + /** + * Reads waiting for a message. One read of the queue at a time fetches messages for all of them. + */ + private waiters: Waiter[] = [] + /** + * Messages received for reads that were no longer waiting + */ + private received: TransportMessage[] = [] + /** + * The read of the queue in flight, which `stop()` waits for + */ + private fetching: Promise | undefined + private lastReclaimAt = 0 + private lastConsumerCleanupAt = 0 + + /** + * @param configuration the queue, how to connect, and how messages are received and dead-lettered + * @throws InvalidRedisKeyName if `queueName` or `keyPrefix` is empty or contains `{` or `}` + * @throws InvalidRedisTransportDuration if `visibilityTimeoutMs` isn't a positive number, or + * `deadLetterRetentionMs` is negative + */ + constructor(private readonly configuration: RedisTransportConfiguration) { + if (!isValidKeyName(configuration.queueName)) { + throw new InvalidRedisKeyName('queueName', configuration.queueName) + } + this.endpointName = configuration.queueName + this.keys = new RedisKeys(configuration.keyPrefix ?? DEFAULT_KEY_PREFIX) + this.visibilityTimeoutMs = + configuration.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS + const visibilityTimeoutMs = this.visibilityTimeoutMs + if ( + typeof visibilityTimeoutMs !== 'number' || + !Number.isFinite(visibilityTimeoutMs) || + visibilityTimeoutMs <= 0 + ) { + throw new InvalidRedisTransportDuration( + 'visibilityTimeoutMs', + visibilityTimeoutMs + ) + } + const retention = + configuration.deadLetterRetentionMs ?? DEFAULT_DEAD_LETTER_RETENTION_MS + if ( + typeof retention !== 'number' || + Number.isNaN(retention) || + retention < 0 + ) { + throw new InvalidRedisTransportDuration( + 'deadLetterRetentionMs', + retention + ) + } + // 0 keeps dead letters until they're removed, as Infinity does + this.deadLetterRetentionMs = Number.isFinite(retention) + ? Math.round(retention) + : 0 + } + + /** + * Keeps the bus' serializer, which messages are written and read with, and creates the transport's logger + * @param coreDependencies the dependencies of the bus the transport belongs to + */ + prepare(coreDependencies: CoreDependencies): void { + this.coreDependencies = coreDependencies + this.logger = coreDependencies.loggerFactory( + '@node-ts/bus-redis:redis-transport' + ) + } + + /** + * Connects to Redis, with the connection that sends and settles messages. Receiving uses a connection of its own, + * made when the transport starts. + */ + async connect(): Promise { + if (this.client) { + return + } + this.logger.info('Connecting redis transport') + const client = createRedisClient(this.configuration.connection ?? {}) + // A client without an error listener crashes the process when its connection fails. node-redis reconnects. + client.on('error', error => + this.logger.warn('Redis connection error', { + error: serializeError(error) + }) + ) + await client.connect() + this.client = client + } + + /** + * Closes the connection to Redis. If the transport received messages and nothing is pending to it any more, it's + * removed from the queue's consumer group first, so stopped processes don't leave consumers behind. + */ + async disconnect(): Promise { + if (this.hasStarted) { + // The bus disconnects once its workers have settled the messages they were handling + await this.leaveConsumerGroup() + } + const client = this.client + this.client = undefined + if (client?.isOpen) { + await client.close() + } + } + + /** + * Checks, unless `verifyResources` is off or the bus only sends, that the queue's stream and consumer group exist, + * and that the queue is subscribed to each message it handles. It creates nothing. + * @param options the messages the bus handles, and whether to check its resources + * @throws ResourcesNotProvisioned if something the bus needs doesn't exist + */ + async initialize(options: TransportInitializationOptions): Promise { + this.logger.info('Initializing redis transport') + if (options.verifyResources && !options.sendOnly) { + const missing = await this.findMissingResources( + this.subscribedMessageNames(options) + ) + if (missing.length) { + throw new ResourcesNotProvisioned('RedisTransport', missing) + } + } + this.logger.info('Redis transport initialized') + } + + /** + * Unless the bus only sends, creates the queue's stream and its consumer group, and subscribes the queue to each + * message it handles, including the topics of custom handlers. What exists is left as it is, and nothing is + * removed, so a subscription for a message the service no longer handles stays until it's removed by hand. Sending + * needs nothing provisioned, since messages are added to the streams of the queues subscribed to them. + * + * It needs the `XGROUP CREATE` and `SADD` commands, which the transport doesn't need at runtime. + * @param options the messages the bus handles, and whether it's a dry run + * @returns the stream, consumer group and subscriptions, and the ACL rules the transport needs at runtime + */ + async provision( + options: TransportProvisionOptions + ): Promise { + const queueName = this.endpointName + const queueKey = this.keys.queue(queueName) + const subscriptions = options.sendOnly + ? [] + : this.subscribedMessageNames(options) + const plan: ProvisioningPlan = { + adapter: 'RedisTransport', + resources: options.sendOnly + ? [] + : [ + { + type: 'redis-stream', + name: queueKey, + properties: { holds: 'messages waiting to be handled' } + }, + { + type: 'redis-consumer-group', + name: queueName, + properties: { stream: queueKey, startId: '0' } + }, + ...subscriptions.map((messageName): ProvisionedResource => ({ + type: 'redis-subscription', + name: `${messageName} -> ${queueName}`, + properties: { + set: this.keys.subscriptions(messageName), + member: queueName + } + })) + ], + runtimePermissions: { + format: 'redis-acl', + document: { + user: '', + rules: this.runtimeAclRules(options.sendOnly) + } + } + } + if (options.dryRun || options.sendOnly) { + return plan + } + + this.logger.info('Provisioning redis transport', { + resources: plan.resources.length + }) + const client = this.getClient() + try { + // From 0, so messages already in the stream are received too + await client.sendCommand([ + 'XGROUP', + 'CREATE', + queueKey, + queueName, + '0', + 'MKSTREAM' + ]) + } catch (error) { + // Already created, by an earlier provision or another process + if (!(error instanceof Error && error.message.startsWith('BUSYGROUP'))) { + throw error + } + } + for (const messageName of subscriptions) { + await client.sendCommand([ + 'SADD', + this.keys.subscriptions(messageName), + queueName + ]) + } + return plan + } + + /** + * Checks the headers set by outgoing middleware before the bus buffers or sends the message + * @param sendOptions the options the message will be sent with + * @throws TransportHeaderReserved if a header is named `bus-failure` + */ + assertSendOptions(sendOptions: TransportSendOptions): void { + assertHeadersNotReserved(sendOptions.headers ?? {}) + } + + /** + * Publishes an event to every queue subscribed to its name, atomically. A message with no subscribers is dropped, + * with a warning. + * @param event the event to publish + * @param messageAttributes the attributes to publish it with + * @param sendOptions native headers from outgoing middleware + * @throws TransportHeaderReserved if a header is named `bus-failure` + */ + async publish( + event: TEvent, + messageAttributes?: MessageAttributes, + sendOptions?: TransportSendOptions + ): Promise { + await this.sendToSubscribers(event, messageAttributes, sendOptions) + } + + /** + * Sends a command to every queue subscribed to its name, which is normally the one service that handles it. A + * message with no subscribers is dropped, with a warning. + * @param command the command to send + * @param messageAttributes the attributes to send it with + * @param sendOptions native headers from outgoing middleware + * @throws TransportHeaderReserved if a header is named `bus-failure` + */ + async send( + command: TCommand, + messageAttributes?: MessageAttributes, + sendOptions?: TransportSendOptions + ): Promise { + await this.sendToSubscribers(command, messageAttributes, sendOptions) + } + + /** + * Sends a message straight to a queue's stream, without going through the subscriptions. The bus calls it for + * `ctx.reply()`. + * @param address the name of the queue, which is the `queueName` of the transport that reads it + * @param message the command or event to send + * @param messageAttributes the attributes to send it with + * @param sendOptions native headers from outgoing middleware + * @throws TransportHeaderReserved if a header is named `bus-failure` + * @throws EndpointNotFound if there's no stream for that queue, because it hasn't been provisioned + */ + async sendToAddress( + address: string, + message: Message, + messageAttributes?: MessageAttributes, + sendOptions?: TransportSendOptions + ): Promise { + const fields = this.toFields(message, messageAttributes, sendOptions) + if (!isValidKeyName(address)) { + throw new EndpointNotFound(address, 'RedisTransport') + } + // NOMKSTREAM adds nothing to a stream that doesn't exist, rather than creating one nothing reads + const id = await this.getClient().sendCommand([ + 'XADD', + this.keys.queue(address), + 'NOMKSTREAM', + '*', + ...fields + ]) + if (id === null) { + throw new EndpointNotFound(address, 'RedisTransport') + } + } + + /** + * Starts receiving, on a connection of its own, since a read waits on the server for a message + */ + async start(): Promise { + if (this.isStarted) { + return + } + const reader = this.getClient().duplicate({ + commandOptions: { timeout: READ_BLOCK_MS + READ_TIMEOUT_MARGIN_MS } + }) + reader.on('error', error => + this.logger.warn('Redis connection error while receiving', { + error: serializeError(error) + }) + ) + await reader.connect() + this.reader = reader + this.isStarted = true + this.hasStarted = true + } + + /** + * Stops receiving. It waits for a read in flight, which waits on the server for at most a second, gives back any + * message received but not handled, so another receiver takes it over without counting a failed attempt, and + * releases the reads still waiting. Messages being handled can still be settled until the transport disconnects. + */ + async stop(): Promise { + if (!this.isStarted) { + return + } + this.isStarted = false + await this.fetching + const waiters = this.waiters + this.waiters = [] + waiters.forEach(waiter => waiter.resolve(undefined)) + const received = this.received + this.received = [] + for (const message of received) { + await this.release(message.raw) + } + const reader = this.reader + this.reader = undefined + if (reader?.isOpen) { + await reader.close() + } + } + + /** + * Receives the next message. While there's none, it waits until one arrives or the transport stops. Returned + * messages that are due are moved back to the queue first, and messages left pending past the visibility timeout + * are taken over. A message that can't be parsed is dead-lettered. + * @throws the connection's error if the queue can't be read, or ResourcesNotProvisioned if its consumer group + * doesn't exist + */ + async readNextMessage(): Promise< + TransportMessage | undefined + > { + if (!this.isStarted) { + return undefined + } + const message = this.received.shift() + if (message) { + return message + } + return new Promise((resolve, reject) => { + this.waiters.push({ resolve, reject }) + this.fetchForWaiters() + }) + } + + /** + * Deletes a message that's been handled. If another receiver has taken it over since, nothing is deleted, and the + * message is handled again. + */ + async deleteMessage( + message: TransportMessage + ): Promise { + const raw = message.raw + const settled = await this.runScript( + DELETE_SCRIPT, + [this.keys.queue(this.endpointName)], + this.receiptArguments(raw) + ) + this.warnIfTakenOver(settled, raw, 'deleted') + } + + /** + * Returns a message to be received again after `delay`, with one more failed attempt. It waits in the queue's + * delayed set until it's due. + * @param message the message to return + * @param delay how long until it can be received again, in milliseconds + */ + async returnMessage( + message: TransportMessage, + delay: Milliseconds + ): Promise { + const raw = message.raw as RedisTransportMessage + const settled = await this.runScript( + RETURN_SCRIPT, + [ + this.keys.queue(this.endpointName), + this.keys.delayed(this.endpointName) + ], + [ + ...this.receiptArguments(raw), + String(Math.max(0, Math.round(delay))), + ...this.fieldsOf(raw), + FAILED_ATTEMPTS_FIELD, + String(failedAttemptsOf(raw) + 1) + ] + ) + this.warnIfTakenOver(settled, raw, 'returned') + } + + /** + * Moves a message to the queue's dead letter stream, with its failure metadata in a `bus-failure` field, in one + * script + * @param transportMessage the message to dead-letter + * @param failure why and where it failed + */ + async fail( + transportMessage: TransportMessage, + failure: MessageFailure + ): Promise { + await this.deadLetter( + transportMessage.raw as RedisTransportMessage, + failure + ) + } + + /** + * Stops receiving and closes the connections + */ + async dispose(): Promise { + await this.stop() + await this.disconnect() + } + + private async sendToSubscribers( + message: Message, + messageAttributes: MessageAttributes | undefined, + sendOptions: TransportSendOptions | undefined + ): Promise { + const fields = this.toFields(message, messageAttributes, sendOptions) + const client = this.getClient() + const subscribers = await client.sendCommand([ + 'SMEMBERS', + this.keys.subscriptions(message.$name) + ]) + const queues = subscribers.filter(queue => { + if (isValidKeyName(queue)) { + return true + } + this.logger.warn('Skipping a subscription with an invalid queue name', { + messageName: message.$name, + queue + }) + return false + }) + if (!queues.length) { + this.logger.warn( + 'No queue is subscribed to the message, so it was dropped. Provision the service that handles it.', + { messageName: message.$name } + ) + return + } + // In one transaction, so every subscriber gets the message or none does + const transaction = client.multi() + queues.forEach(queue => + transaction.addCommand([ + 'XADD', + this.keys.queue(queue), + 'NOMKSTREAM', + '*', + ...fields + ]) + ) + const ids = (await transaction.exec()) as unknown[] + ids.forEach((id, index) => { + if (id === null) { + this.logger.warn( + 'A queue is subscribed to the message but has no stream, so it was skipped. Provision the queue, or remove the subscription.', + { messageName: message.$name, queue: queues[index] } + ) + } + }) + } + + /** + * Serializes a message and its attributes to a stream entry's fields and values + * @throws TransportHeaderReserved if a header is named `bus-failure` + */ + private toFields( + message: Message, + messageAttributes: MessageAttributes | undefined, + sendOptions: TransportSendOptions | undefined + ): string[] { + const headers = sendOptions?.headers ?? {} + assertHeadersNotReserved(headers) + const attributes: MessageAttributes = { + ...messageAttributes, + // The bus always sets a messageId. This only covers the transport being called directly + messageId: messageAttributes?.messageId ?? randomUUID(), + attributes: messageAttributes?.attributes ?? {}, + stickyAttributes: messageAttributes?.stickyAttributes ?? {} + } + return [ + BODY_FIELD, + this.coreDependencies.messageSerializer.serialize(message), + ATTRIBUTES_FIELD, + JSON.stringify(attributes), + HEADERS_FIELD, + JSON.stringify(headers) + ] + } + + /** + * The fields of a received message to copy when it's returned or dead-lettered, without its failed attempts + */ + private fieldsOf(raw: RedisTransportMessage): string[] { + return [ + BODY_FIELD, + raw.body, + ATTRIBUTES_FIELD, + JSON.stringify(raw.attributes), + HEADERS_FIELD, + JSON.stringify(raw.headers) + ] + } + + /** + * Reads the queue for the reads that are waiting, unless a read is already in flight. When it finishes, it reads + * again for any reads still waiting. + */ + private fetchForWaiters(): void { + if (this.fetching || !this.isStarted || !this.waiters.length) { + return + } + this.fetching = this.fetch() + .catch(error => { + // Each waiting read fails, so the bus logs it and waits before reading again + const waiters = this.waiters + this.waiters = [] + const failure = isNoGroupError(error) + ? new ResourcesNotProvisioned('RedisTransport', [ + this.consumerGroupDescription() + ]) + : error + waiters.forEach(waiter => waiter.reject(failure)) + }) + .finally(() => { + this.fetching = undefined + this.fetchForWaiters() + }) + } + + /** + * Reads messages for the waiting reads until it has at least one or the transport stops + */ + private async fetch(): Promise { + while (this.isStarted && this.waiters.length) { + const wanted = this.waiters.length + const now = Date.now() + const reclaim = now - this.lastReclaimAt >= RECLAIM_INTERVAL_MS + const cleanUpConsumers = + now - this.lastConsumerCleanupAt >= CONSUMER_CLEANUP_INTERVAL_MS + const [nextDue, claimed] = (await this.runScript( + MAINTAIN_SCRIPT, + [ + this.keys.queue(this.endpointName), + this.keys.delayed(this.endpointName) + ], + [ + this.endpointName, + this.consumerName, + String(this.visibilityTimeoutMs), + String(reclaim ? wanted : 0), + String(DUE_MESSAGES_PER_READ), + String(cleanUpConsumers ? CONSUMER_IDLE_MS : 0) + ] + )) as [string, [string, string[], number][]] + if (reclaim) { + this.lastReclaimAt = now + } + if (cleanUpConsumers) { + this.lastConsumerCleanupAt = now + } + + let entries: ReceivedEntry[] = claimed.map( + ([id, fields, deliveries]) => ({ + id, + fields, + deliveries + }) + ) + if (entries.length) { + this.logger.debug( + 'Took over messages left pending past the visibility timeout', + { ids: entries.map(({ id }) => id) } + ) + } else { + const dueIn = Number(nextDue) + entries = await this.readNew( + wanted, + dueIn >= 0 + ? Math.min(READ_BLOCK_MS, Math.max(1, dueIn)) + : READ_BLOCK_MS + ) + } + for (const entry of entries) { + const message = await this.toTransportMessage(entry) + if (!message) { + continue + } + if (!this.isStarted) { + // Received while stopping. It's given back for another receiver. + await this.release(message.raw) + continue + } + const waiter = this.waiters.shift() + if (waiter) { + waiter.resolve(message) + } else { + this.received.push(message) + } + } + } + } + + /** + * Reads messages that haven't been delivered to any receiver, waiting up to `blockMs` for one + */ + private async readNew( + count: number, + blockMs: Milliseconds + ): Promise { + const reader = this.reader + if (!reader) { + return [] + } + const reply = await reader.sendCommand< + [stream: string, entries: RawStreamEntry[]][] | null + >([ + 'XREADGROUP', + 'GROUP', + this.endpointName, + this.consumerName, + 'COUNT', + String(count), + 'BLOCK', + String(blockMs), + 'STREAMS', + this.keys.queue(this.endpointName), + '>' + ]) + if (!reply) { + return [] + } + return reply.flatMap(([, entries]) => + entries + .filter((entry): entry is [string, string[]] => entry[1] !== null) + .map(([id, fields]) => ({ id, fields, deliveries: 1 })) + ) + } + + /** + * Converts a received stream entry to a message for the bus, or dead-letters it if it can't be parsed + */ + private async toTransportMessage( + entry: ReceivedEntry + ): Promise | undefined> { + const fields = toFieldMap(entry.fields) + const storedAttributes = + parseJsonField>(fields[ATTRIBUTES_FIELD]) ?? {} + const attributes: MessageAttributes = { + ...storedAttributes, + attributes: storedAttributes.attributes ?? {}, + stickyAttributes: storedAttributes.stickyAttributes ?? {} + } + const failedAttemptsBefore = Number(fields[FAILED_ATTEMPTS_FIELD] ?? 0) + const raw: RedisTransportMessage = { + id: entry.id, + body: fields[BODY_FIELD] ?? '', + attributes, + headers: parseJsonField(fields[HEADERS_FIELD]) ?? {}, + failedAttemptsBefore: Number.isFinite(failedAttemptsBefore) + ? failedAttemptsBefore + : 0, + deliveries: entry.deliveries + } + try { + return { + id: entry.id, + domainMessage: this.coreDependencies.messageSerializer.deserialize( + raw.body + ), + raw, + attributes, + failedAttempts: failedAttemptsOf(raw) + } + } catch (error) { + // Parsing fails the same way every time, so retrying can't help + this.logger.warn( + 'Could not parse message. It will be moved to the dead letter stream', + { id: entry.id, error: serializeError(error) } + ) + await this.deadLetter( + raw, + createMessageFailure(error, { + failedAttempts: failedAttemptsOf(raw) + 1, + endpoint: this.endpointName, + messageId: attributes.messageId + }) + ) + return undefined + } + } + + private async deadLetter( + raw: RedisTransportMessage, + failure: MessageFailure + ): Promise { + const settled = await this.runScript( + FAIL_SCRIPT, + [ + this.keys.queue(this.endpointName), + this.keys.deadLetter(this.endpointName) + ], + [ + ...this.receiptArguments(raw), + String(this.deadLetterRetentionMs), + ...this.fieldsOf(raw), + FAILURE_HEADER, + toFailureHeader(failure) + ] + ) + this.warnIfTakenOver(settled, raw, 'dead-lettered') + } + + /** + * Gives back a message received but not handled, so another receiver takes it over without counting a failed + * attempt. A failure is only logged: the message is taken over after the visibility timeout anyway. + */ + private async release(raw: RedisTransportMessage): Promise { + try { + await this.runScript( + RELEASE_SCRIPT, + [this.keys.queue(this.endpointName)], + [...this.receiptArguments(raw), String(this.visibilityTimeoutMs)] + ) + } catch (error) { + this.logger.warn( + 'Could not give back a message received while stopping. Another receiver takes it over after the visibility timeout.', + { id: raw.id, error: serializeError(error) } + ) + } + } + + /** + * Removes this receiver from the consumer group, if nothing is pending to it. A failure is only logged. + */ + private async leaveConsumerGroup(): Promise { + if (!this.client?.isOpen) { + return + } + try { + await this.runScript( + LEAVE_SCRIPT, + [this.keys.queue(this.endpointName)], + [this.endpointName, this.consumerName] + ) + } catch (error) { + this.logger.debug('Could not leave the consumer group', { + error: serializeError(error) + }) + } + } + + /** + * The arguments that identify a receipt to the scripts that settle it: the group, this consumer, the entry and + * its delivery count + */ + private receiptArguments(raw: RedisTransportMessage): string[] { + return [ + this.endpointName, + this.consumerName, + raw.id, + String(raw.deliveries) + ] + } + + /** + * Runs a script by its digest, sending its source the first time the server doesn't have it + */ + private async runScript( + redisScript: RedisScript, + keys: string[], + args: string[] + ): Promise { + const client = this.getClient() + const tail = [String(keys.length), ...keys, ...args] + try { + return await client.sendCommand(['EVALSHA', redisScript.sha, ...tail]) + } catch (error) { + if (!(error instanceof Error && error.message.startsWith('NOSCRIPT'))) { + throw error + } + return client.sendCommand(['EVAL', redisScript.source, ...tail]) + } + } + + private warnIfTakenOver( + settled: unknown, + raw: RedisTransportMessage, + action: string + ): void { + if (Number(settled) !== 1) { + this.logger.warn( + `Message could not be ${action}, because its visibility timeout ended and another receiver has taken it over, or it was removed. It may be handled again. Raise visibilityTimeoutMs above how long your handlers take.`, + { id: raw.id, messageId: raw.attributes.messageId } + ) + } + } + + private getClient(): RedisClient { + if (!this.client) { + throw new RedisTransportNotConnected() + } + return this.client + } + + /** + * The message names the bus' queue is subscribed to: those it handles and the topics of its custom handlers + */ + private subscribedMessageNames({ + handlerRegistry + }: Pick): string[] { + return [ + ...new Set([ + ...handlerRegistry.getMessageNames(), + ...handlerRegistry.getExternallyManagedTopicIdentifiers() + ]) + ] + } + + private consumerGroupDescription(): string { + return `Redis consumer group ${this.endpointName} on ${this.keys.queue(this.endpointName)}` + } + + /** + * Finds what a receiving bus needs that doesn't exist: its stream, consumer group and subscriptions + * @returns a description of each one that's missing + */ + private async findMissingResources( + subscriptions: string[] + ): Promise { + const client = this.getClient() + const queueKey = this.keys.queue(this.endpointName) + const missing: string[] = [] + try { + const groups = await client.sendCommand([ + 'XINFO', + 'GROUPS', + queueKey + ]) + const hasGroup = groups.some( + group => toFieldMap(group).name === this.endpointName + ) + if (!hasGroup) { + missing.push(this.consumerGroupDescription()) + } + } catch (error) { + if (!(error instanceof Error && /no such key/i.test(error.message))) { + throw error + } + missing.push(`Redis stream ${queueKey}`, this.consumerGroupDescription()) + } + for (const messageName of subscriptions) { + const isMember = await client.sendCommand([ + 'SISMEMBER', + this.keys.subscriptions(messageName), + this.endpointName + ]) + if (isMember !== 1) { + missing.push( + `Redis subscription ${messageName} -> ${this.endpointName}` + ) + } + } + return missing + } + + /** + * The ACL rules the transport needs at runtime. A receiving bus reads and writes its own queue's keys, adds + * messages to the stream of any queue (to publish, send and reply) and reads the subscription sets. It runs its + * scripts with `EVALSHA`, and `EVAL` to load them, and the commands they run must be allowed too. + */ + private runtimeAclRules(sendOnly: boolean): string[] { + const sending = [ + `%W~${this.keys.anyQueuePattern()}`, + `%R~${this.keys.subscriptionsPattern()}`, + '+xadd', + '+smembers', + '+multi', + '+exec' + ] + if (sendOnly) { + return sending + } + return [ + `~${this.keys.queueKeysPattern(this.endpointName)}`, + ...sending, + '+xreadgroup', + '+xack', + '+xdel', + '+xpending', + '+xclaim', + '+xinfo|groups', + '+xinfo|consumers', + '+xgroup|delconsumer', + '+zadd', + '+zrange', + '+zrangebyscore', + '+zrem', + '+sismember', + '+time', + '+evalsha', + '+eval' + ] + } +} + +/** + * How many times handling a message failed before this delivery: the failed attempts it was returned with, plus each + * earlier delivery of this stream entry that was never settled, such as one to a process that stopped + */ +const failedAttemptsOf = (raw: RedisTransportMessage): number => + raw.failedAttemptsBefore + raw.deliveries - 1 diff --git a/packages/bus-redis/tsconfig.json b/packages/bus-redis/tsconfig.json new file mode 100644 index 00000000..55d02cd4 --- /dev/null +++ b/packages/bus-redis/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*.ts", "src/index.mts"], + "exclude": ["src/**/*.spec.ts", "src/**/*.integration.ts", "src/**/test"], + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 504a8907..949197c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -268,6 +268,9 @@ importers: '@node-ts/bus-rabbitmq': specifier: workspace:^ version: link:../packages/bus-rabbitmq + '@node-ts/bus-redis': + specifier: workspace:^ + version: link:../packages/bus-redis '@node-ts/bus-sqs': specifier: workspace:^ version: link:../packages/bus-sqs @@ -610,6 +613,34 @@ importers: specifier: ^6.0.3 version: 6.0.3 + packages/bus-redis: + dependencies: + '@node-ts/bus-messages': + specifier: workspace:^ + version: link:../bus-messages + serialize-error: + specifier: '8' + version: 8.1.0 + tslib: + specifier: ^2.8.1 + version: 2.8.1 + devDependencies: + '@node-ts/bus-core': + specifier: workspace:^ + version: link:../bus-core + '@node-ts/bus-test': + specifier: workspace:^ + version: link:../bus-test + redis: + specifier: ^6.3.0 + version: 6.3.0(@opentelemetry/api@1.9.1) + typemoq: + specifier: ^2.1.0 + version: 2.1.0 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + packages/bus-sqs: dependencies: '@aws-sdk/client-sns': @@ -2024,6 +2055,42 @@ packages: resolution: {integrity: sha512-4EDEmvxWtgsCnnVeBvtFIFZtUhPPt1+bA9JrSwU4Sa//6oKtzCSlGGXYJr44OD9aGISymbieJ4mCKHUygUDU+g==} engines: {node: '>=18'} + '@redis/bloom@6.3.0': + resolution: {integrity: sha512-NQ5poYpZr0jv6zazN1cy6JY5kIipKFZ3R9GTL4kOUTX3a9YAfY29U8bsE4lWhVQEM+G4V0BXVC57mubIdDijeA==} + engines: {node: '>= 20.0.0'} + peerDependencies: + '@redis/client': ^6.3.0 + + '@redis/client@6.3.0': + resolution: {integrity: sha512-fHrjwGCBvAANGi+TuRuwKi83sIOzgHr8eh+Uf48w3fSX7Hhrz174NrWqfJhAC2vnN0d/tfaZpdr6+o6JC3QHwg==} + engines: {node: '>= 20.0.0'} + peerDependencies: + '@node-rs/xxhash': ^1.1.0 + '@opentelemetry/api': '>=1 <2' + peerDependenciesMeta: + '@node-rs/xxhash': + optional: true + '@opentelemetry/api': + optional: true + + '@redis/json@6.3.0': + resolution: {integrity: sha512-yt6vRfPBXtW/3qm53cF81TSQpLMjWdW4/2E9WQ2GgbvsKkuCKCFQIcMiKlil1ZMnezK+IQuS795xKU5psbaegw==} + engines: {node: '>= 20.0.0'} + peerDependencies: + '@redis/client': ^6.3.0 + + '@redis/search@6.3.0': + resolution: {integrity: sha512-RPXAZmKjZZG7Tm7m1w9keFdXzbL91PiSHQBfQRe9xwW40jNsTRbG0FwhIz4QlHPU4gKDAsCy8IQHj4yZl4H9rw==} + engines: {node: '>= 20.0.0'} + peerDependencies: + '@redis/client': ^6.3.0 + + '@redis/time-series@6.3.0': + resolution: {integrity: sha512-+fJOB8mN1z5WRakCJNh5lVdz1KqvHGSOistsKeitD24QsCtPwyRQXWC0iq6T/i/ubLE/YcguZqPm8F598MKk4A==} + engines: {node: '>= 20.0.0'} + peerDependencies: + '@redis/client': ^6.3.0 + '@rollup/rollup-android-arm-eabi@4.63.6': resolution: {integrity: sha512-G6xF9OVRWsbHadMfoPNUDwW/Jt70QYG+ZDUvtYCFxjf3SPhqVAO0u9ShXZ7Cf5silVoRWyN4bDjCEKnJkXkw0w==} cpu: [arm] @@ -4228,6 +4295,10 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + redis@6.3.0: + resolution: {integrity: sha512-XFQbPie1lGpKeUZ8ySYY43yLnQy/iIQtHNvIr/1XdaABpb96rUSdm3Mfl+98FEVbv9ZJIzINMno5s0Nh8E5LEw==} + engines: {node: '>= 20.0.0'} + reflect-metadata@0.2.2: resolution: {integrity: sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==} @@ -6346,6 +6417,26 @@ snapshots: dependencies: tinyexec: 1.3.1 + '@redis/bloom@6.3.0(@redis/client@6.3.0(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 6.3.0(@opentelemetry/api@1.9.1) + + '@redis/client@6.3.0(@opentelemetry/api@1.9.1)': + optionalDependencies: + '@opentelemetry/api': 1.9.1 + + '@redis/json@6.3.0(@redis/client@6.3.0(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 6.3.0(@opentelemetry/api@1.9.1) + + '@redis/search@6.3.0(@redis/client@6.3.0(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 6.3.0(@opentelemetry/api@1.9.1) + + '@redis/time-series@6.3.0(@redis/client@6.3.0(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 6.3.0(@opentelemetry/api@1.9.1) + '@rollup/rollup-android-arm-eabi@4.63.6': optional: true @@ -8824,6 +8915,17 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + redis@6.3.0(@opentelemetry/api@1.9.1): + dependencies: + '@redis/bloom': 6.3.0(@redis/client@6.3.0(@opentelemetry/api@1.9.1)) + '@redis/client': 6.3.0(@opentelemetry/api@1.9.1) + '@redis/json': 6.3.0(@redis/client@6.3.0(@opentelemetry/api@1.9.1)) + '@redis/search': 6.3.0(@redis/client@6.3.0(@opentelemetry/api@1.9.1)) + '@redis/time-series': 6.3.0(@redis/client@6.3.0(@opentelemetry/api@1.9.1)) + transitivePeerDependencies: + - '@node-rs/xxhash' + - '@opentelemetry/api' + reflect-metadata@0.2.2: {} regex-recursion@6.0.2: diff --git a/test.env b/test.env index d581bafd..c5f79baf 100644 --- a/test.env +++ b/test.env @@ -17,4 +17,5 @@ NODE_OPTIONS="--experimental-vm-modules --disable-warning=ExperimentalWarning" # RABBITMQ_URL=amqp://guest:guest@0.0.0.0 # RABBITMQ_MANAGEMENT_URL=http://127.0.0.1:15672 # POSTGRES_URL=postgres://postgres:password@localhost:6432/postgres +# REDIS_URL=redis://localhost:6380 # MONGODB_URL=mongodb://localhost:27017/workflows?directConnection=true From ad4e2eb53d808e0cc71481d2af73220dd407c8c1 Mon Sep 17 00:00:00 2001 From: Andrew den Hertog Date: Thu, 8 Oct 2026 14:20:21 +1000 Subject: [PATCH 2/4] bound redis reads and shutdown on a frozen connection, require whole-ms durations Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/redis-streams-transport.md | 2 +- MIGRATING.md | 2 +- docs/transports/redis.md | 17 +- packages/bus-redis/CLAUDE.md | 12 +- .../error/invalid-redis-transport-duration.ts | 11 +- .../src/redis-transport-configuration.ts | 4 +- .../src/redis-transport.integration.ts | 50 ++++ .../bus-redis/src/redis-transport.spec.ts | 34 ++- packages/bus-redis/src/redis-transport.ts | 228 ++++++++++++++---- .../bus-redis/src/test/freezable-proxy.ts | 75 ++++++ packages/bus-redis/src/test/index.ts | 1 + 11 files changed, 352 insertions(+), 84 deletions(-) create mode 100644 packages/bus-redis/src/test/freezable-proxy.ts create mode 100644 packages/bus-redis/src/test/index.ts diff --git a/.changeset/redis-streams-transport.md b/.changeset/redis-streams-transport.md index 42d33eda..ea7e3419 100644 --- a/.changeset/redis-streams-transport.md +++ b/.changeset/redis-streams-transport.md @@ -4,4 +4,4 @@ Add `RedisTransport`, a Redis Streams transport for Redis 7+ and Valkey 7.2+ (#347). Each service's queue is a stream read by a consumer group, with fan-out through subscription sets, delayed retries in a sorted set, messages taken over from stopped processes after `visibilityTimeoutMs`, a dead letter stream per queue with `deadLetterRetentionMs`, `bus provision` support with a `redis-acl` runtime permission plan, and replies. It needs `redis` (node-redis) 6 as a peer dependency. -**Breaking:** this replaces `@node-ts/bus-redis` 0.x, the Redis lists transport for bus-core 1.x, which shared its name. It's a new transport with a new configuration and key layout, and messages in 0.x queues aren't moved: see "Migrating from @node-ts/bus-redis 0.x" at https://node-ts.github.io/bus/transports/redis. +**Breaking:** this replaces `@node-ts/bus-redis` 0.x, the Redis lists transport that shared its name: 0.1.8 (`latest`, for bus-core 1.x) and 0.1.1–0.1.7 and 0.1.9 (the `0.x` tag, for bus-core 0.6 with inversify). It's a new transport with a new configuration and key layout, and messages in 0.x queues aren't moved: see "Migrating from @node-ts/bus-redis 0.x" at https://node-ts.github.io/bus/transports/redis. diff --git a/MIGRATING.md b/MIGRATING.md index 9549d540..da72713f 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -162,7 +162,7 @@ Two things behave differently: ## @node-ts/bus-redis -- **It's a new transport on Redis Streams**, replacing the 0.x Redis lists transport from the `node-ts/bus-redis` repository. `BusRedisModule` and inversify are gone: pass `new RedisTransport({ queueName, connection: { url } })` to `withTransport()`. It needs Redis 7.0+ or Valkey 7.2+, and `redis` (node-redis) 6 as a peer dependency. +- **It's a new transport on Redis Streams**, replacing the 0.x Redis lists transport from the `node-ts/bus-redis` repository: 0.1.8 (npm's `latest`, for bus-core 1.x, configured with `new RedisTransport({ queueName, connectionString })`), and 0.1.1–0.1.7 and 0.1.9 (the `0.x` tag, for bus-core 0.6, loaded as the inversify `BusRedisModule`). Pass `new RedisTransport({ queueName, connection: { url } })` to `withTransport()`: `connectionString` is now `connection.url`, and inversify isn't used. It needs Redis 7.0+ or Valkey 7.2+, and `redis` (node-redis) 6 as a peer dependency. - **`maxRetries` is removed**: use the bus' recoverability policy, `defaultRecoverability({ maxAttempts })`. `visibilityTimeout` is now `visibilityTimeoutMs`, `subscriptionsKeyPrefix` is now `keyPrefix`, and `withScheduler` is removed. - **Messages in 0.x queues aren't moved**, and the two versions can't send each other messages. Drain the old queues, then provision and deploy every service that exchanges messages together. See [Migrating from @node-ts/bus-redis 0.x](https://node-ts.github.io/bus/transports/redis#migrating-from-node-ts-bus-redis-0-x). diff --git a/docs/transports/redis.md b/docs/transports/redis.md index e5568b66..0627cd5c 100644 --- a/docs/transports/redis.md +++ b/docs/transports/redis.md @@ -49,7 +49,7 @@ Configure a `RedisTransport` and pass it to the bus configuration: | `visibilityTimeoutMs` | `30000` | How long a handler has, in milliseconds, before another receiver takes the message over and it's handled again. Set it above how long your slowest handler takes. | | `deadLetterRetentionMs` | `1209600000` (14 d) | How long dead-lettered messages are kept, in milliseconds. Older ones are trimmed, roughly, as new ones are dead-lettered. `0` or `Infinity` keeps them until you remove them. | -The transport makes two connections: one to send and settle messages, and one that waits for new ones. It always speaks RESP2, whatever node-redis' default. An invalid `queueName` or `keyPrefix` throws `InvalidRedisKeyName`, and an invalid duration throws `InvalidRedisTransportDuration`. +The transport makes two connections: one to send and settle messages, and one that waits for new ones. It always speaks RESP2, whatever node-redis' default. Each connection sends a `PING` every 5 seconds, and one that receives nothing for 15 seconds is closed and reconnected, so a server or network that stops answering doesn't leave the bus waiting forever; `stop()` and `dispose()` wait a few seconds at most for it. Set `socket.socketTimeout`, `pingInterval` or `socket.reconnectStrategy` in `connection` to change that. An invalid `queueName` or `keyPrefix` throws `InvalidRedisKeyName`, and a duration that isn't a whole number of milliseconds throws `InvalidRedisTransportDuration`. ## Sending and receiving @@ -57,7 +57,7 @@ Provisioning a service creates its queue's stream, `bus:{}:queue`, wi Every instance of a service reads its queue as the same consumer group, so each message goes to one of them. A message that's been read stays pending to its receiver until it's handled, and is then acknowledged and removed from the stream, so streams only hold what's waiting or being handled. If it isn't settled within `visibilityTimeoutMs`, such as when the process stops, another receiver takes it over and handles it again, counting a failed attempt. That also happens if a handler takes longer than the timeout, while the first receipt is still being handled. The first receipt can then no longer delete, retry or dead-letter the message, which is logged as a warning. Make handlers idempotent, or use [`withOutbox()`](/guide/outbox), whose inbox skips a copy of a message that was already handled. -Each process reads its queue once at a time, for as many messages as it has workers waiting, waiting on the server for up to a second for one to arrive. A message is received as soon as it's added. Messages are received in the order they were added, but a retried message goes to the back of the queue, and there's no strict ordering between instances. +Each process reads its queue one read at a time, for as many messages as it has workers waiting, waiting on the server for up to a second for one to arrive. A message is received as soon as it's added. Messages are received in the order they were added, but a retried message goes to the back of the queue, and there's no strict ordering between instances. When a process stops, it gives back any message it read but didn't hand to a worker, without counting an attempt, and once its workers have settled what they were handling, it leaves the consumer group. A process that crashed leaves its consumer behind: it's removed by another instance once it has nothing pending and has been idle for a day. @@ -153,11 +153,11 @@ At `initialize()`, unless the bus only sends, the transport checks that its stre ```txt ~bus:{reservations-service}:* %W~bus:{*}:queue %R~bus:subscriptions:* -+xadd +smembers +multi +exec +xreadgroup +xack +xdel +xpending +xclaim +xinfo|groups +xinfo|consumers ++xadd +smembers +multi +exec +ping +xreadgroup +xack +xdel +xpending +xclaim +xinfo|groups +xinfo|consumers +xgroup|delconsumer +zadd +zrange +zrangebyscore +zrem +sismember +time +evalsha +eval ``` -The service can read and write its own queue's keys, add messages to any queue's stream (to send, publish and reply) without reading them, and read the subscription sets. It runs its scripts with `EVALSHA`, and `EVAL` to load them, and the commands they run are checked too. A send-only bus only needs the second and third key patterns and `+xadd +smembers +multi +exec`. Create the user with the rules, such as: +The service can read and write its own queue's keys, add messages to any queue's stream (to send, publish and reply) without reading them, and read the subscription sets. It runs its scripts with `EVALSHA`, and `EVAL` to load them, and the commands they run are checked too. A send-only bus only needs the second and third key patterns and `+xadd +smembers +multi +exec +ping`. Create the user with the rules, such as: ```sh redis-cli ACL SETUSER reservations-service on '>a-long-password' resetkeys resetchannels -@all \ @@ -188,11 +188,16 @@ docker run -d -p 6379:6379 valkey/valkey:8 ## Migrating from @node-ts/bus-redis 0.x -The 0.x versions of `@node-ts/bus-redis`, from the `node-ts/bus-redis` repository, ran on bus-core 1.x with inversify, and kept queues in Redis lists. This version is a new transport on Redis Streams. First move your services to the current bus-core, following [Upgrading to 2.0](/upgrading/v2), then change the transport's configuration: +The 0.x versions of `@node-ts/bus-redis`, from the `node-ts/bus-redis` repository, kept queues in Redis lists. There were two lines of them: + +- **0.1.8**, npm's `latest`, for bus-core 1.x. It's configured with `new RedisTransport({ queueName, connectionString, ... })`, passed to `Bus.configure().withTransport()`. +- **0.1.1 to 0.1.7, and 0.1.9** (the `0.x` tag), for bus-core 0.6 with inversify. It's loaded as `BusRedisModule`, with its configuration bound to `BUS_REDIS_SYMBOLS.TransportConfiguration`. + +This version is a new transport on Redis Streams. First move your services to the current bus-core, following [Upgrading to 2.0](/upgrading/v2), after moving to bus-core 1.x if they're on 0.6. Then change the transport's configuration: | 0.x option | Now | | ------------------------ | --------------------------------------------------------------------------------------------------------------------- | -| `BusRedisModule` | `new RedisTransport(configuration)`, passed to `Bus.configure().withTransport()` | +| `BusRedisModule` (0.6) | `new RedisTransport(configuration)`, passed to `Bus.configure().withTransport()`, as 0.1.8 already does | | `queueName` | `queueName` | | `connectionString` | `connection: { url }` | | `maxRetries` | The bus' [recoverability policy](/guide/recoverability): `withRecoverability(defaultRecoverability({ maxAttempts }))` | diff --git a/packages/bus-redis/CLAUDE.md b/packages/bus-redis/CLAUDE.md index dca3651a..6d760a32 100644 --- a/packages/bus-redis/CLAUDE.md +++ b/packages/bus-redis/CLAUDE.md @@ -1,20 +1,20 @@ # bus-redis -A Redis Streams transport (node-redis 6, a peer dependency), replacing the archived `node-ts/bus-redis` 0.x (Redis lists via modest-queue, bus-core 1.x). The design was agreed on #347 (see the "Accepted design" comment). Read the root `CLAUDE.md` first. +A Redis Streams transport (node-redis 6, a peer dependency), replacing the archived `node-ts/bus-redis` 0.x (Redis lists via modest-queue: 0.1.8 for bus-core 1.x with `new RedisTransport({ connectionString })`, and 0.1.1–0.1.7/0.1.9 for bus-core 0.6 with inversify's `BusRedisModule`). The design was agreed on #347 (see the "Accepted design" comment). Read the root `CLAUDE.md` first. ## Design -- **Config** (`src/redis-transport-configuration.ts`): `queueName`, `connection` (node-redis `createClient()` options minus `RESP`/modules/scripts/typeMapping, default `{}`), `keyPrefix` (default `bus`), `visibilityTimeoutMs` (30 s), `deadLetterRetentionMs` (14 days; `0`/`Infinity` keep everything). No `deadLetterQueueName`: dead letters are per queue. `queueName`/`keyPrefix` with `{`/`}` or empty throw `InvalidRedisKeyName`; bad durations throw `InvalidRedisTransportDuration`. Using it before `connect()` throws `RedisTransportNotConnected`. -- **Connections**: `connect()` makes the command client (`createClient({ ...connection, RESP: 2 })`, so raw replies always have RESP2 shapes; everything goes through `sendCommand`, `multi().addCommand()` or scripts). `start()` makes the reader with `duplicate()` and a command timeout of the block time + 10 s. Every client has an `error` listener. Only standalone Redis/Valkey (no Cluster or Sentinel in v1), but keys stay Cluster-ready. +- **Config** (`src/redis-transport-configuration.ts`): `queueName`, `connection` (node-redis `createClient()` options minus `RESP`/modules/scripts/typeMapping, default `{}`), `keyPrefix` (default `bus`), `visibilityTimeoutMs` (30 s), `deadLetterRetentionMs` (14 days; `0`/`Infinity` keep everything). No `deadLetterQueueName`: dead letters are per queue. `queueName`/`keyPrefix` with `{`/`}` or empty throw `InvalidRedisKeyName`; durations must be whole milliseconds (Redis rejects a fractional `IDLE`), else `InvalidRedisTransportDuration`. Using it before `connect()` throws `RedisTransportNotConnected`. +- **Connections**: `connect()` makes the command client (`createClient({ ...connection, RESP: 2 })`, so raw replies always have RESP2 shapes; everything goes through `sendCommand`, `multi().addCommand()` or scripts). node-redis' command timeout stops covering a command once it's written to the socket (6.3, `commands-queue.js`), so a frozen or half-open connection would leave a read or script pending forever. So every client gets `socket.socketTimeout` (15 s; the reader's is at least the block time + 10 s), `pingInterval` 5 s (so idle connections aren't timed out; hence `+ping` in the ACL) and a `reconnectStrategy` that also reconnects after a socket timeout (node-redis' default doesn't); settings in `connection` win. `stop()` races the fetch in flight against 3 s and then `destroy()`s the reader, and bounds giving back messages to 3 s; `disconnect()` bounds leaving the group and `close()` to 3 s each, then `destroy()`s (`settleWithin`/`closeClient`). The integration test freezes a TCP proxy (`src/test/freezable-proxy.ts`) during a blocked read to cover it. `start()` makes the reader with `duplicate()`. Every client has an `error` listener. Only standalone Redis/Valkey (no Cluster or Sentinel in v1), but keys stay Cluster-ready. - **Keys** (`src/redis-keys.ts`, internal): `:{}:queue` (stream, consumer group ``), `:{}:delayed` (zset), `:{}:dead-letter` (stream), `:subscriptions:<$name>` (set of queue names). Every script touches one queue's keys only, which share the `{}` hash tag. `escapeGlob` escapes ACL key patterns. - **Entries**: fields `body` (serializer output), `attributes` (JSON), `headers` (JSON) and `failedAttempts` once returned; the dead letter copy adds `bus-failure` and drops `failedAttempts`. Only `bus-failure` is a reserved header. `TransportMessage.id` is the entry id (new on each retry). - **Sending**: `SMEMBERS` the subscription set, then one `MULTI` of `XADD NOMKSTREAM * …` per subscriber (all or nothing on a single primary). No subscribers warns and drops; a null reply (stream missing) warns. `sendToAddress` is `XADD NOMKSTREAM`, `EndpointNotFound` on null or an address with braces. - **Scripts** (`src/redis-scripts.ts`, run with `EVALSHA`, falling back to `EVAL` on `NOSCRIPT`; they use the server's `TIME` and only Redis 6.2/7.0 commands so Valkey 7.2+ works; don't use `XACKDEL`, `XNACK` or `XREADGROUP CLAIM`): every settle checks `isPendingTo` (pending to this consumer with the delivery count it was read with, the lease token) and returns 0 if not, which the transport logs as taken over. `DELETE` = XACK + XDEL. `RETURN` = ZADD a `cjson` copy `{id, fields}` at now + delay (or XADD at once when the delay is 0) + XACK + XDEL. `FAIL` = XADD to the dead letter stream with `MINID ~ now - retention` + XACK + XDEL. `MAINTAIN` (before each read) moves up to 100 due zset entries back with XADD, returns ms until the next is due, reclaims with `XPENDING … IDLE ` + `XCLAIM` (once per second, for as many reads as are waiting), and once a minute removes other consumers with nothing pending and idle for a day. `RELEASE` (stop) re-claims a read-but-unhandled entry with `IDLE RETRYCOUNT JUSTID`, so another receiver takes it straight over without an attempt counted. `LEAVE` (on `disconnect()`, after the bus' workers settled) `XGROUP DELCONSUMER`s this consumer only when nothing is pending to it: DELCONSUMER drops pending entries, so never run it otherwise. -- **Receiving**: one fetch at a time per transport (`fetchForWaiters`/`fetch`) for every waiting `readNextMessage()`: MAINTAIN, then reclaimed entries, or else `XREADGROUP … COUNT BLOCK min(1000, next due) STREAMS >` on the reader. It loops until a message or `stop()`; a fetch error rejects the waiting reads (`NOGROUP` becomes `ResourcesNotProvisioned`), so the bus logs it and backs off. `failedAttempts = failedAttempts field + deliveries - 1` (new reads have 1 delivery; a reclaimed entry has its XPENDING count + 1), so expired receipts count like SQS and Postgres. `stop()` waits for the fetch in flight (≤ 1 s), releases unhandled entries and resolves waiting reads with `undefined`; handlers still settle until `disconnect()`. The consumer name is `--`, per transport. Unparseable bodies are dead-lettered on read. -- **Provisioning**: `provision()` (unless send-only or a dry run) runs `XGROUP CREATE 0 MKSTREAM` (`BUSYGROUP` is success) and `SADD`s every handled `$name` and custom-handler topic. `messageNames` is unused (no per-message resources). Plan types: `redis-stream`, `redis-consumer-group`, `redis-subscription`; `runtimePermissions` is `{ format: 'redis-acl', document: { user: '', rules } }`: own keys RW, `%W~:{*}:queue`, `%R~:subscriptions:*`, and the commands the transport and its scripts run (send-only: just the two patterns + `xadd smembers multi exec`). The integration test runs a bus as an ACL user with exactly these rules, so keep the list in step with the scripts. `initialize()` with `verifyResources` (not send-only) checks `XINFO GROUPS` and `SISMEMBER`s and throws `ResourcesNotProvisioned`. +- **Receiving**: one fetch at a time per transport (`fetchForWaiters`/`fetch`) for every waiting `readNextMessage()`: MAINTAIN, then reclaimed entries, or else `XREADGROUP … COUNT BLOCK min(1000, next due) STREAMS >` on the reader. If converting or dead-lettering an entry of a batch throws, the entries not yet given to a read are released (`deliver`). It loops until a message or `stop()`; a fetch error rejects the waiting reads (`NOGROUP` becomes `ResourcesNotProvisioned`), so the bus logs it and backs off. `failedAttempts = failedAttempts field + deliveries - 1` (new reads have 1 delivery; a reclaimed entry has its XPENDING count + 1), so expired receipts count like SQS and Postgres. `stop()` waits for the fetch in flight (≤ 1 s), releases unhandled entries and resolves waiting reads with `undefined`; handlers still settle until `disconnect()`. The consumer name is `--`, per transport. Unparseable bodies are dead-lettered on read. +- **Provisioning**: `provision()` (unless send-only or a dry run) runs `XGROUP CREATE 0 MKSTREAM` (`BUSYGROUP` is success) and `SADD`s every handled `$name` and custom-handler topic. `messageNames` is unused (no per-message resources). Plan types: `redis-stream`, `redis-consumer-group`, `redis-subscription`; `runtimePermissions` is `{ format: 'redis-acl', document: { user: '', rules } }`: own keys RW, `%W~:{*}:queue`, `%R~:subscriptions:*`, and the commands the transport and its scripts run (send-only: just the two patterns + `xadd smembers multi exec ping`). The integration test runs a bus as an ACL user with exactly these rules, so keep the list in step with the scripts. `initialize()` with `verifyResources` (not send-only) checks `XINFO GROUPS` and `SISMEMBER`s and throws `ResourcesNotProvisioned`. - Delayed sends use the persistence's outgoing messages and dispatcher (no core change). The transport doesn't join the `withOutbox()` transaction. ## Tests - Integration tests need Redis at `redis://localhost:6380` (override with `REDIS_URL`): `docker compose up -d --wait redis` from the repo root (port 6380, since 6379 is often taken). Valkey is on 6381 (`docker compose up -d --wait valkey`, `REDIS_URL=redis://localhost:6381`); CI runs the integration file against both. Tests use a random `keyPrefix`, remove only their own keys with `SCAN`/`UNLINK`, and must never `FLUSHDB`/`FLUSHALL`: the server may be shared. The ACL test creates and deletes its own user. -- The suites use bus-test's fixtures and message types, so this package has no `generate:message-types` script. +- The suites use bus-test's fixtures and message types, so this package has no `generate:message-types` script. `src/test` only holds the freezable proxy. diff --git a/packages/bus-redis/src/error/invalid-redis-transport-duration.ts b/packages/bus-redis/src/error/invalid-redis-transport-duration.ts index 884d2cbd..e68e6ef7 100644 --- a/packages/bus-redis/src/error/invalid-redis-transport-duration.ts +++ b/packages/bus-redis/src/error/invalid-redis-transport-duration.ts @@ -1,6 +1,7 @@ /** * Thrown when `RedisTransport` is configured with a duration it can't use: a `visibilityTimeoutMs` that isn't a - * positive, finite number of milliseconds, or a `deadLetterRetentionMs` that's negative or not a number + * positive whole number of milliseconds, or a `deadLetterRetentionMs` that isn't 0 or more whole milliseconds, or + * `Infinity`. Redis takes these as integers. */ export class InvalidRedisTransportDuration extends Error { readonly help: string @@ -15,13 +16,13 @@ export class InvalidRedisTransportDuration extends Error { ) { super( setting === 'visibilityTimeoutMs' - ? `RedisTransport's visibilityTimeoutMs must be a positive number of milliseconds, but was ${String(value)}` - : `RedisTransport's deadLetterRetentionMs must be 0 or more milliseconds, or Infinity, but was ${String(value)}` + ? `RedisTransport's visibilityTimeoutMs must be a positive whole number of milliseconds, but was ${String(value)}` + : `RedisTransport's deadLetterRetentionMs must be 0 or more whole milliseconds, or Infinity, but was ${String(value)}` ) this.help = setting === 'visibilityTimeoutMs' - ? 'Set visibilityTimeoutMs in the RedisTransport configuration to a number above 0, or leave it out to use the default' - : 'Set deadLetterRetentionMs in the RedisTransport configuration to how long to keep dead letters, 0 or Infinity to keep them until removed, or leave it out to use the default' + ? 'Set visibilityTimeoutMs in the RedisTransport configuration to a whole number above 0, or leave it out to use the default' + : 'Set deadLetterRetentionMs in the RedisTransport configuration to how long to keep dead letters in whole milliseconds, 0 or Infinity to keep them until removed, or leave it out to use the default' Object.setPrototypeOf(this, new.target.prototype) } } diff --git a/packages/bus-redis/src/redis-transport-configuration.ts b/packages/bus-redis/src/redis-transport-configuration.ts index c6a0948a..6a5eadd0 100644 --- a/packages/bus-redis/src/redis-transport-configuration.ts +++ b/packages/bus-redis/src/redis-transport-configuration.ts @@ -28,7 +28,9 @@ export interface RedisTransportConfiguration extends Omit< /** * How to connect to Redis. The transport makes a connection for sending and settling messages, and another for - * waiting for new messages. + * waiting for new messages. Unless set here, each sends a `PING` every 5 s (`pingInterval`), is closed and + * reconnected when it receives nothing for 15 s (`socket.socketTimeout`), and reconnects with a backoff of up to + * 2 s (`socket.reconnectStrategy`). * @default {} (localhost:6379) * @example { url: process.env.REDIS_URL } */ diff --git a/packages/bus-redis/src/redis-transport.integration.ts b/packages/bus-redis/src/redis-transport.integration.ts index 5a843837..8ef0946e 100644 --- a/packages/bus-redis/src/redis-transport.integration.ts +++ b/packages/bus-redis/src/redis-transport.integration.ts @@ -28,6 +28,7 @@ import { createClient } from 'redis' import { Mock } from 'typemoq' import { RedisTransport } from './redis-transport' import { RedisTransportConfiguration } from './redis-transport-configuration' +import { FreezableProxy } from './test' jest.setTimeout(30_000) @@ -479,6 +480,55 @@ describe('RedisTransport', () => { }) }) + describe('and Redis stops answering', () => { + const proxy = new FreezableProxy({ + host: new URL(connection.url).hostname, + port: Number(new URL(connection.url).port || 6379) + }) + let stopMs: number + let disposeMs: number + + beforeAll(async () => { + await proxy.start() + const handled = Promise.withResolvers() + const bus = await startBus( + 'frozen', + { connection: { url: `redis://127.0.0.1:${proxy.port}` } }, + async () => handled.resolve() + ) + await bus.send(new TestCommand(randomUUID(), new Date())) + await handled.promise + // Freezes while the worker waits on the server for its next message, so that read is in flight + while ( + !(await redis.sendCommand(['CLIENT', 'LIST'])) + .split('\n') + .some( + client => /cmd=xreadgroup/.test(client) && /flags=b/.test(client) + ) + ) { + await new Promise(resolve => setTimeout(resolve, 20)) + } + proxy.freeze() + const stopping = Date.now() + await bus.stop() + stopMs = Date.now() - stopping + const disposing = Date.now() + await bus.dispose() + disposeMs = Date.now() - disposing + }) + + afterAll(async () => proxy.close()) + + it('should stop without waiting for it', () => { + // stop() gives a read in flight 3 s, then closes its connection + expect(stopMs).toBeLessThan(8_000) + }) + + it('should dispose without waiting for it', () => { + expect(disposeMs).toBeLessThan(10_000) + }) + }) + describe('and a message cannot be parsed', () => { const queueName = 'unparseable' let bus: BusInstance diff --git a/packages/bus-redis/src/redis-transport.spec.ts b/packages/bus-redis/src/redis-transport.spec.ts index 26230da3..dfc30d22 100644 --- a/packages/bus-redis/src/redis-transport.spec.ts +++ b/packages/bus-redis/src/redis-transport.spec.ts @@ -98,6 +98,7 @@ describe('RedisTransport', () => { '+smembers', '+multi', '+exec', + '+ping', '+xreadgroup', '+xack', '+xdel', @@ -149,7 +150,8 @@ describe('RedisTransport', () => { '+xadd', '+smembers', '+multi', - '+exec' + '+exec', + '+ping' ] }) }) @@ -190,8 +192,11 @@ describe('RedisTransport', () => { for (const settings of [ { visibilityTimeoutMs: 0 }, { visibilityTimeoutMs: Infinity }, + // Redis takes idle times as integers + { visibilityTimeoutMs: 1500.5 }, { deadLetterRetentionMs: -1 }, - { deadLetterRetentionMs: Number.NaN } + { deadLetterRetentionMs: Number.NaN }, + { deadLetterRetentionMs: 1500.5 } ]) { try { new RedisTransport({ ...configuration, ...settings }) @@ -207,6 +212,8 @@ describe('RedisTransport', () => { ).toEqual([ 'visibilityTimeoutMs', 'visibilityTimeoutMs', + 'visibilityTimeoutMs', + 'deadLetterRetentionMs', 'deadLetterRetentionMs', 'deadLetterRetentionMs' ]) @@ -214,17 +221,20 @@ describe('RedisTransport', () => { }) describe('when constructed with a dead letter retention of 0 or Infinity', () => { + const errors: unknown[] = [] + + beforeAll(() => { + for (const deadLetterRetentionMs of [0, Infinity]) { + try { + new RedisTransport({ ...configuration, deadLetterRetentionMs }) + } catch (error) { + errors.push(error) + } + } + }) + it('should accept it', () => { - expect( - () => new RedisTransport({ ...configuration, deadLetterRetentionMs: 0 }) - ).not.toThrow() - expect( - () => - new RedisTransport({ - ...configuration, - deadLetterRetentionMs: Infinity - }) - ).not.toThrow() + expect(errors).toEqual([]) }) }) diff --git a/packages/bus-redis/src/redis-transport.ts b/packages/bus-redis/src/redis-transport.ts index 55ecbc79..c26b0cb7 100644 --- a/packages/bus-redis/src/redis-transport.ts +++ b/packages/bus-redis/src/redis-transport.ts @@ -62,15 +62,32 @@ export const DEFAULT_DEAD_LETTER_RETENTION_MS: Milliseconds = 14 * 24 * 60 * 60 * 1_000 /** - * The longest a read waits on the server for a message. It's kept well below node-redis' default command timeout - * (5 s), and it's the longest `stop()` waits for a read in flight. + * The longest a read waits on the server for a message */ const READ_BLOCK_MS = 1_000 /** - * How long the reading connection waits for the answer to a read, beyond how long the read blocks for + * How long a connection may go without receiving anything before it's closed as lost, and reconnected. node-redis' + * command timeout only covers a command waiting to be written, not one waiting for its reply, so without it a + * connection that stops answering, such as through a frozen proxy or a half-open socket, would leave reads and + * scripts waiting forever. Each connection sends a PING every `PING_INTERVAL_MS`, so an idle one isn't closed. The + * receiving connection's is longer than a read blocks for. */ -const READ_TIMEOUT_MARGIN_MS = 10_000 +const SOCKET_TIMEOUT_MS = 15_000 +const READ_SOCKET_TIMEOUT_MS = READ_BLOCK_MS + 10_000 +const PING_INTERVAL_MS = 5_000 + +/** + * The longest `stop()` waits for a read in flight before it closes the receiving connection, and for the messages + * it gives back + */ +const STOP_TIMEOUT_MS = READ_BLOCK_MS + 2_000 + +/** + * The longest `disconnect()` waits to leave the consumer group, and for the connection to close cleanly, before it + * closes it at once + */ +const DISCONNECT_TIMEOUT_MS = 3_000 /** * How often messages left pending past the visibility timeout are looked for @@ -111,6 +128,12 @@ interface ReceivedEntry { deliveries: number } +/** + * What identifies a receipt of a message: its stream entry, and how many times it had been delivered when it was + * received + */ +type Receipt = Pick + /** * A read waiting for a message */ @@ -122,10 +145,67 @@ interface Waiter { } /** - * Creates a client that speaks RESP2, so replies have the same shape whatever the client's defaults + * Reconnects with a backoff of up to 2 s, like node-redis' default, but also after a socket timeout, which node-redis + * doesn't reconnect after by default + */ +const reconnectWithBackoff = (retries: number): number => + Math.min(2 ** retries * 50, 2_000) + Math.floor(Math.random() * 200) + +/** + * Creates a client that speaks RESP2, so replies have the same shape whatever the client's defaults, and that + * notices a connection that stopped answering (see `SOCKET_TIMEOUT_MS`). Settings given in `options` win. */ const createRedisClient = (options: RedisConnectionOptions) => - createClient({ ...options, RESP: 2 }) + createClient({ + pingInterval: PING_INTERVAL_MS, + ...options, + socket: { + socketTimeout: SOCKET_TIMEOUT_MS, + reconnectStrategy: reconnectWithBackoff, + ...options.socket + }, + RESP: 2 + }) + +const TIMED_OUT = Symbol('timed out') + +/** + * Waits for a promise, but no longer than `ms` + * @returns what the promise resolves with, or `TIMED_OUT` + */ +const settleWithin = async ( + promise: Promise, + ms: Milliseconds +): Promise => { + let timer: NodeJS.Timeout | undefined + try { + return await Promise.race([ + promise, + new Promise(resolve => { + timer = setTimeout(() => resolve(TIMED_OUT), ms) + }) + ]) + } finally { + clearTimeout(timer) + } +} + +/** + * Closes a client cleanly, or at once if that takes longer than `DISCONNECT_TIMEOUT_MS`, such as when its connection + * stopped answering + */ +const closeClient = async (client: RedisClient): Promise => { + if (!client.isOpen) { + return + } + const closed = await settleWithin( + client.close().catch(() => undefined), + DISCONNECT_TIMEOUT_MS + ) + if (closed === TIMED_OUT) { + client.destroy() + } +} type RedisClient = ReturnType @@ -226,8 +306,8 @@ export class RedisTransport implements Transport { /** * @param configuration the queue, how to connect, and how messages are received and dead-lettered * @throws InvalidRedisKeyName if `queueName` or `keyPrefix` is empty or contains `{` or `}` - * @throws InvalidRedisTransportDuration if `visibilityTimeoutMs` isn't a positive number, or - * `deadLetterRetentionMs` is negative + * @throws InvalidRedisTransportDuration if `visibilityTimeoutMs` isn't a positive whole number, or + * `deadLetterRetentionMs` isn't 0 or more whole milliseconds, or `Infinity` */ constructor(private readonly configuration: RedisTransportConfiguration) { if (!isValidKeyName(configuration.queueName)) { @@ -238,11 +318,8 @@ export class RedisTransport implements Transport { this.visibilityTimeoutMs = configuration.visibilityTimeoutMs ?? DEFAULT_VISIBILITY_TIMEOUT_MS const visibilityTimeoutMs = this.visibilityTimeoutMs - if ( - typeof visibilityTimeoutMs !== 'number' || - !Number.isFinite(visibilityTimeoutMs) || - visibilityTimeoutMs <= 0 - ) { + // A whole number, since Redis takes idle times as integers + if (!Number.isInteger(visibilityTimeoutMs) || visibilityTimeoutMs <= 0) { throw new InvalidRedisTransportDuration( 'visibilityTimeoutMs', visibilityTimeoutMs @@ -251,9 +328,8 @@ export class RedisTransport implements Transport { const retention = configuration.deadLetterRetentionMs ?? DEFAULT_DEAD_LETTER_RETENTION_MS if ( - typeof retention !== 'number' || - Number.isNaN(retention) || - retention < 0 + retention !== Infinity && + (!Number.isInteger(retention) || retention < 0) ) { throw new InvalidRedisTransportDuration( 'deadLetterRetentionMs', @@ -261,9 +337,7 @@ export class RedisTransport implements Transport { ) } // 0 keeps dead letters until they're removed, as Infinity does - this.deadLetterRetentionMs = Number.isFinite(retention) - ? Math.round(retention) - : 0 + this.deadLetterRetentionMs = Number.isFinite(retention) ? retention : 0 } /** @@ -304,12 +378,12 @@ export class RedisTransport implements Transport { async disconnect(): Promise { if (this.hasStarted) { // The bus disconnects once its workers have settled the messages they were handling - await this.leaveConsumerGroup() + await settleWithin(this.leaveConsumerGroup(), DISCONNECT_TIMEOUT_MS) } const client = this.client this.client = undefined - if (client?.isOpen) { - await client.close() + if (client) { + await closeClient(client) } } @@ -497,8 +571,17 @@ export class RedisTransport implements Transport { if (this.isStarted) { return } - const reader = this.getClient().duplicate({ - commandOptions: { timeout: READ_BLOCK_MS + READ_TIMEOUT_MARGIN_MS } + const client = this.getClient() + const socket = client.options.socket ?? {} + const reader = client.duplicate({ + // Longer than a read blocks for, so only a connection that stopped answering times out + socket: { + ...socket, + socketTimeout: Math.max( + socket.socketTimeout ?? 0, + READ_SOCKET_TIMEOUT_MS + ) + } }) reader.on('error', error => this.logger.warn('Redis connection error while receiving', { @@ -515,25 +598,40 @@ export class RedisTransport implements Transport { * Stops receiving. It waits for a read in flight, which waits on the server for at most a second, gives back any * message received but not handled, so another receiver takes it over without counting a failed attempt, and * releases the reads still waiting. Messages being handled can still be settled until the transport disconnects. + * + * It waits at most a few seconds for each step, so a connection that stopped answering can't hold it up: a read + * still in flight then has its connection closed, and a message it couldn't give back is taken over once its + * visibility timeout ends. */ async stop(): Promise { if (!this.isStarted) { return } this.isStarted = false - await this.fetching + const reader = this.reader + this.reader = undefined + const fetching = this.fetching + if ( + fetching && + (await settleWithin(fetching, STOP_TIMEOUT_MS)) === TIMED_OUT + ) { + this.logger.warn( + 'A read of the queue did not finish in time, so its connection was closed. Redis may not be answering.' + ) + // Fails the read in flight straight away. One waiting on the main connection is left to its socket timeout. + reader?.destroy() + } const waiters = this.waiters this.waiters = [] waiters.forEach(waiter => waiter.resolve(undefined)) const received = this.received this.received = [] - for (const message of received) { - await this.release(message.raw) - } - const reader = this.reader - this.reader = undefined - if (reader?.isOpen) { - await reader.close() + await settleWithin( + this.releaseAll(received.map(({ raw }) => raw)), + STOP_TIMEOUT_MS + ) + if (reader) { + await closeClient(reader) } } @@ -799,22 +897,37 @@ export class RedisTransport implements Transport { : READ_BLOCK_MS ) } - for (const entry of entries) { - const message = await this.toTransportMessage(entry) - if (!message) { - continue - } - if (!this.isStarted) { - // Received while stopping. It's given back for another receiver. - await this.release(message.raw) - continue - } - const waiter = this.waiters.shift() - if (waiter) { - waiter.resolve(message) - } else { - this.received.push(message) - } + await this.deliver(entries) + } + } + + /** + * Gives received entries to the waiting reads. If one can't be converted or dead-lettered, the entries not given + * to a read yet, including that one, are given back, so they're received again straight away rather than after the + * visibility timeout, and the error is thrown. + */ + private async deliver(entries: ReceivedEntry[]): Promise { + for (const [index, entry] of entries.entries()) { + let message: TransportMessage | undefined + try { + message = await this.toTransportMessage(entry) + } catch (error) { + await this.releaseAll(entries.slice(index)) + throw error + } + if (!message) { + continue + } + if (!this.isStarted) { + // Received while stopping. It's given back for another receiver. + await this.release(message.raw) + continue + } + const waiter = this.waiters.shift() + if (waiter) { + waiter.resolve(message) + } else { + this.received.push(message) } } } @@ -933,7 +1046,7 @@ export class RedisTransport implements Transport { * Gives back a message received but not handled, so another receiver takes it over without counting a failed * attempt. A failure is only logged: the message is taken over after the visibility timeout anyway. */ - private async release(raw: RedisTransportMessage): Promise { + private async release(raw: Receipt): Promise { try { await this.runScript( RELEASE_SCRIPT, @@ -942,12 +1055,21 @@ export class RedisTransport implements Transport { ) } catch (error) { this.logger.warn( - 'Could not give back a message received while stopping. Another receiver takes it over after the visibility timeout.', + 'Could not give back a message that was received but not handled. Another receiver takes it over after the visibility timeout.', { id: raw.id, error: serializeError(error) } ) } } + /** + * Gives back messages received but not handled, one at a time + */ + private async releaseAll(receipts: Receipt[]): Promise { + for (const receipt of receipts) { + await this.release(receipt) + } + } + /** * Removes this receiver from the consumer group, if nothing is pending to it. A failure is only logged. */ @@ -972,7 +1094,7 @@ export class RedisTransport implements Transport { * The arguments that identify a receipt to the scripts that settle it: the group, this consumer, the entry and * its delivery count */ - private receiptArguments(raw: RedisTransportMessage): string[] { + private receiptArguments(raw: Receipt): string[] { return [ this.endpointName, this.consumerName, @@ -1094,7 +1216,9 @@ export class RedisTransport implements Transport { '+xadd', '+smembers', '+multi', - '+exec' + '+exec', + // Each connection sends a PING when idle, so one that stopped answering is noticed + '+ping' ] if (sendOnly) { return sending diff --git a/packages/bus-redis/src/test/freezable-proxy.ts b/packages/bus-redis/src/test/freezable-proxy.ts new file mode 100644 index 00000000..46098227 --- /dev/null +++ b/packages/bus-redis/src/test/freezable-proxy.ts @@ -0,0 +1,75 @@ +import { createConnection, createServer, Server, Socket } from 'node:net' + +/** + * A TCP proxy in front of Redis that can be frozen: it then stops passing data either way, but keeps every connection + * open, as a hung server, a stalled network or a half-open socket would. New connections are accepted, and get no + * answer either. + */ +export class FreezableProxy { + private readonly server: Server + private readonly sockets = new Set() + private frozen = false + + /** + * @param target where Redis listens + */ + constructor(private readonly target: { host: string; port: number }) { + this.server = createServer(client => this.connect(client)) + } + + /** + * The port the proxy listens on, once it's started + */ + get port(): number { + const address = this.server.address() + if (!address || typeof address === 'string') { + throw new Error('The proxy has not started') + } + return address.port + } + + /** + * Starts listening on a free port of 127.0.0.1 + */ + async start(): Promise { + await new Promise(resolve => + this.server.listen(0, '127.0.0.1', resolve) + ) + } + + /** + * Stops passing data, without closing any connection + */ + freeze(): void { + this.frozen = true + } + + /** + * Closes every connection and stops listening + */ + async close(): Promise { + this.sockets.forEach(socket => socket.destroy()) + await new Promise(resolve => this.server.close(() => resolve())) + } + + private connect(client: Socket): void { + const upstream = createConnection(this.target) + for (const [from, to] of [ + [client, upstream], + [upstream, client] + ]) { + this.sockets.add(from) + from.on('data', data => { + if (!this.frozen) { + to.write(data) + } + }) + // The other side may already be gone, which the test doesn't care about + from.on('error', () => undefined) + from.on('close', () => { + this.sockets.delete(from) + to.destroy() + }) + } + } +} diff --git a/packages/bus-redis/src/test/index.ts b/packages/bus-redis/src/test/index.ts new file mode 100644 index 00000000..03388c87 --- /dev/null +++ b/packages/bus-redis/src/test/index.ts @@ -0,0 +1 @@ +export * from './freezable-proxy' From 8bb788dc7bcdd884ea9f09cbf3d1b324caef951e Mon Sep 17 00:00:00 2001 From: Andrew den Hertog Date: Thu, 8 Oct 2026 15:23:31 +1000 Subject: [PATCH 3/4] add a reply watchdog for redis connections, leave a failing batch entry to its visibility timeout Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/transports/redis.md | 10 +- packages/bus-redis/CLAUDE.md | 6 +- .../src/redis-transport.integration.ts | 221 +++++++++++++++++- .../bus-redis/src/redis-transport.spec.ts | 21 +- packages/bus-redis/src/redis-transport.ts | 176 +++++++++++--- .../bus-redis/src/test/freezable-proxy.ts | 8 + 6 files changed, 396 insertions(+), 46 deletions(-) diff --git a/docs/transports/redis.md b/docs/transports/redis.md index 0627cd5c..616dea13 100644 --- a/docs/transports/redis.md +++ b/docs/transports/redis.md @@ -49,7 +49,13 @@ Configure a `RedisTransport` and pass it to the bus configuration: | `visibilityTimeoutMs` | `30000` | How long a handler has, in milliseconds, before another receiver takes the message over and it's handled again. Set it above how long your slowest handler takes. | | `deadLetterRetentionMs` | `1209600000` (14 d) | How long dead-lettered messages are kept, in milliseconds. Older ones are trimmed, roughly, as new ones are dead-lettered. `0` or `Infinity` keeps them until you remove them. | -The transport makes two connections: one to send and settle messages, and one that waits for new ones. It always speaks RESP2, whatever node-redis' default. Each connection sends a `PING` every 5 seconds, and one that receives nothing for 15 seconds is closed and reconnected, so a server or network that stops answering doesn't leave the bus waiting forever; `stop()` and `dispose()` wait a few seconds at most for it. Set `socket.socketTimeout`, `pingInterval` or `socket.reconnectStrategy` in `connection` to change that. An invalid `queueName` or `keyPrefix` throws `InvalidRedisKeyName`, and a duration that isn't a whole number of milliseconds throws `InvalidRedisTransportDuration`. +The transport makes two connections: one to send and settle messages, and one that waits for new ones. It always speaks RESP2, whatever node-redis' default. So that a server or network that stops answering doesn't leave the bus waiting forever: + +- every 5 seconds, each connection is sent a `PING`, and if no reply comes within 15 seconds, it's closed, which fails the commands waiting on it, and replaced with a new one. So a connection that stops answering is noticed within 20 seconds, however often it's written to; +- a connection with no traffic at all for 15 seconds is closed and reconnected (`socket.socketTimeout`), and node-redis sends it a `PING` every 5 seconds (`pingInterval`), so an idle one stays open; +- `stop()` waits at most 3 seconds for a read in flight, and then closes its connection, and at most 3 more to give back messages it received but didn't hand out. The bus then waits for the messages being handled: on a connection that stopped answering, settling each one fails once the connection is replaced, up to 20 seconds later. `dispose()` then waits at most 3 seconds to leave the consumer group and 3 to close the connection. + +Set `socket.socketTimeout`, `pingInterval` or `socket.reconnectStrategy` in `connection` to change the connections' own settings. An invalid `queueName` or `keyPrefix` throws `InvalidRedisKeyName`, and a duration that isn't a whole number of milliseconds throws `InvalidRedisTransportDuration`. ## Sending and receiving @@ -188,7 +194,7 @@ docker run -d -p 6379:6379 valkey/valkey:8 ## Migrating from @node-ts/bus-redis 0.x -The 0.x versions of `@node-ts/bus-redis`, from the `node-ts/bus-redis` repository, kept queues in Redis lists. There were two lines of them: +The 0.x versions of `@node-ts/bus-redis`, from the `node-ts/bus-redis` repository, kept queues in Redis lists, through BullMQ in 0.1.1 and 0.1.2 and modest-queue after that. There were two lines of them: - **0.1.8**, npm's `latest`, for bus-core 1.x. It's configured with `new RedisTransport({ queueName, connectionString, ... })`, passed to `Bus.configure().withTransport()`. - **0.1.1 to 0.1.7, and 0.1.9** (the `0.x` tag), for bus-core 0.6 with inversify. It's loaded as `BusRedisModule`, with its configuration bound to `BUS_REDIS_SYMBOLS.TransportConfiguration`. diff --git a/packages/bus-redis/CLAUDE.md b/packages/bus-redis/CLAUDE.md index 6d760a32..d28b360b 100644 --- a/packages/bus-redis/CLAUDE.md +++ b/packages/bus-redis/CLAUDE.md @@ -1,16 +1,16 @@ # bus-redis -A Redis Streams transport (node-redis 6, a peer dependency), replacing the archived `node-ts/bus-redis` 0.x (Redis lists via modest-queue: 0.1.8 for bus-core 1.x with `new RedisTransport({ connectionString })`, and 0.1.1–0.1.7/0.1.9 for bus-core 0.6 with inversify's `BusRedisModule`). The design was agreed on #347 (see the "Accepted design" comment). Read the root `CLAUDE.md` first. +A Redis Streams transport (node-redis 6, a peer dependency), replacing the archived `node-ts/bus-redis` 0.x (Redis lists, via BullMQ in 0.1.1/0.1.2 and modest-queue later: 0.1.8 for bus-core 1.x with `new RedisTransport({ connectionString })`, and 0.1.1–0.1.7/0.1.9 for bus-core 0.6 with inversify's `BusRedisModule`). The design was agreed on #347 (see the "Accepted design" comment). Read the root `CLAUDE.md` first. ## Design - **Config** (`src/redis-transport-configuration.ts`): `queueName`, `connection` (node-redis `createClient()` options minus `RESP`/modules/scripts/typeMapping, default `{}`), `keyPrefix` (default `bus`), `visibilityTimeoutMs` (30 s), `deadLetterRetentionMs` (14 days; `0`/`Infinity` keep everything). No `deadLetterQueueName`: dead letters are per queue. `queueName`/`keyPrefix` with `{`/`}` or empty throw `InvalidRedisKeyName`; durations must be whole milliseconds (Redis rejects a fractional `IDLE`), else `InvalidRedisTransportDuration`. Using it before `connect()` throws `RedisTransportNotConnected`. -- **Connections**: `connect()` makes the command client (`createClient({ ...connection, RESP: 2 })`, so raw replies always have RESP2 shapes; everything goes through `sendCommand`, `multi().addCommand()` or scripts). node-redis' command timeout stops covering a command once it's written to the socket (6.3, `commands-queue.js`), so a frozen or half-open connection would leave a read or script pending forever. So every client gets `socket.socketTimeout` (15 s; the reader's is at least the block time + 10 s), `pingInterval` 5 s (so idle connections aren't timed out; hence `+ping` in the ACL) and a `reconnectStrategy` that also reconnects after a socket timeout (node-redis' default doesn't); settings in `connection` win. `stop()` races the fetch in flight against 3 s and then `destroy()`s the reader, and bounds giving back messages to 3 s; `disconnect()` bounds leaving the group and `close()` to 3 s each, then `destroy()`s (`settleWithin`/`closeClient`). The integration test freezes a TCP proxy (`src/test/freezable-proxy.ts`) during a blocked read to cover it. `start()` makes the reader with `duplicate()`. Every client has an `error` listener. Only standalone Redis/Valkey (no Cluster or Sentinel in v1), but keys stay Cluster-ready. +- **Connections**: `connect()` makes the command client (`createClient({ ...connection, RESP: 2 })`, so raw replies always have RESP2 shapes; everything goes through `sendCommand`, `multi().addCommand()` or scripts). node-redis' command timeout stops covering a command once it's written to the socket (6.3, `commands-queue.js`), so a frozen or half-open connection would leave a read or script pending forever. So every client gets `socket.socketTimeout` (15 s; the reader's is at least the block time + 10 s), `pingInterval` 5 s (so idle connections aren't timed out; hence `+ping` in the ACL) and a `reconnectStrategy` that also reconnects after a socket timeout (node-redis' default doesn't); settings in `connection` win. Writes count as socket activity, so a frozen connection that's written to often never times out: the watchdog (`startWatchdog`, from `connect()` to `disconnect()`, unref'd, never overlapping itself) sends each ready client a PING every 5 s and, without a reply in 15 s, `destroy()`s it (failing what's pending) and replaces it (`createMainClient`/`createReader`). A replacement made while Redis is frozen is stuck in its handshake until its socket timeout, so recovery after Redis answers again can take another 15 s. `stop()` races the fetch in flight against 3 s and then `destroy()`s the reader, and bounds giving back messages to 3 s; `disconnect()` bounds leaving the group and `close()` to 3 s each, then `destroy()`s (`settleWithin`/`closeClient`). The integration tests freeze a TCP proxy (`src/test/freezable-proxy.ts`) during a blocked read, and while sending every 2 s (about 70 s, the slowest test). `start()` makes the reader with `duplicate()`. Every client has an `error` listener. Only standalone Redis/Valkey (no Cluster or Sentinel in v1), but keys stay Cluster-ready. - **Keys** (`src/redis-keys.ts`, internal): `:{}:queue` (stream, consumer group ``), `:{}:delayed` (zset), `:{}:dead-letter` (stream), `:subscriptions:<$name>` (set of queue names). Every script touches one queue's keys only, which share the `{}` hash tag. `escapeGlob` escapes ACL key patterns. - **Entries**: fields `body` (serializer output), `attributes` (JSON), `headers` (JSON) and `failedAttempts` once returned; the dead letter copy adds `bus-failure` and drops `failedAttempts`. Only `bus-failure` is a reserved header. `TransportMessage.id` is the entry id (new on each retry). - **Sending**: `SMEMBERS` the subscription set, then one `MULTI` of `XADD NOMKSTREAM * …` per subscriber (all or nothing on a single primary). No subscribers warns and drops; a null reply (stream missing) warns. `sendToAddress` is `XADD NOMKSTREAM`, `EndpointNotFound` on null or an address with braces. - **Scripts** (`src/redis-scripts.ts`, run with `EVALSHA`, falling back to `EVAL` on `NOSCRIPT`; they use the server's `TIME` and only Redis 6.2/7.0 commands so Valkey 7.2+ works; don't use `XACKDEL`, `XNACK` or `XREADGROUP CLAIM`): every settle checks `isPendingTo` (pending to this consumer with the delivery count it was read with, the lease token) and returns 0 if not, which the transport logs as taken over. `DELETE` = XACK + XDEL. `RETURN` = ZADD a `cjson` copy `{id, fields}` at now + delay (or XADD at once when the delay is 0) + XACK + XDEL. `FAIL` = XADD to the dead letter stream with `MINID ~ now - retention` + XACK + XDEL. `MAINTAIN` (before each read) moves up to 100 due zset entries back with XADD, returns ms until the next is due, reclaims with `XPENDING … IDLE ` + `XCLAIM` (once per second, for as many reads as are waiting), and once a minute removes other consumers with nothing pending and idle for a day. `RELEASE` (stop) re-claims a read-but-unhandled entry with `IDLE RETRYCOUNT JUSTID`, so another receiver takes it straight over without an attempt counted. `LEAVE` (on `disconnect()`, after the bus' workers settled) `XGROUP DELCONSUMER`s this consumer only when nothing is pending to it: DELCONSUMER drops pending entries, so never run it otherwise. -- **Receiving**: one fetch at a time per transport (`fetchForWaiters`/`fetch`) for every waiting `readNextMessage()`: MAINTAIN, then reclaimed entries, or else `XREADGROUP … COUNT BLOCK min(1000, next due) STREAMS >` on the reader. If converting or dead-lettering an entry of a batch throws, the entries not yet given to a read are released (`deliver`). It loops until a message or `stop()`; a fetch error rejects the waiting reads (`NOGROUP` becomes `ResourcesNotProvisioned`), so the bus logs it and backs off. `failedAttempts = failedAttempts field + deliveries - 1` (new reads have 1 delivery; a reclaimed entry has its XPENDING count + 1), so expired receipts count like SQS and Postgres. `stop()` waits for the fetch in flight (≤ 1 s), releases unhandled entries and resolves waiting reads with `undefined`; handlers still settle until `disconnect()`. The consumer name is `--`, per transport. Unparseable bodies are dead-lettered on read. +- **Receiving**: one fetch at a time per transport (`fetchForWaiters`/`fetch`) for every waiting `readNextMessage()`: MAINTAIN, then reclaimed entries, or else `XREADGROUP … COUNT BLOCK min(1000, next due) STREAMS >` on the reader. If converting or dead-lettering an entry of a batch throws, the entries after it are released (`deliver`); the failing one is left to its visibility timeout, so one that fails every time is counted rather than looping. It loops until a message or `stop()`; a fetch error rejects the waiting reads (`NOGROUP` becomes `ResourcesNotProvisioned`), so the bus logs it and backs off. `failedAttempts = failedAttempts field + deliveries - 1` (new reads have 1 delivery; a reclaimed entry has its XPENDING count + 1), so expired receipts count like SQS and Postgres. `stop()` waits for the fetch in flight (≤ 3 s, then destroys the reader), releases unhandled entries and resolves waiting reads with `undefined`; handlers still settle until `disconnect()`. The consumer name is `--`, per transport. Unparseable bodies are dead-lettered on read. - **Provisioning**: `provision()` (unless send-only or a dry run) runs `XGROUP CREATE 0 MKSTREAM` (`BUSYGROUP` is success) and `SADD`s every handled `$name` and custom-handler topic. `messageNames` is unused (no per-message resources). Plan types: `redis-stream`, `redis-consumer-group`, `redis-subscription`; `runtimePermissions` is `{ format: 'redis-acl', document: { user: '', rules } }`: own keys RW, `%W~:{*}:queue`, `%R~:subscriptions:*`, and the commands the transport and its scripts run (send-only: just the two patterns + `xadd smembers multi exec ping`). The integration test runs a bus as an ACL user with exactly these rules, so keep the list in step with the scripts. `initialize()` with `verifyResources` (not send-only) checks `XINFO GROUPS` and `SISMEMBER`s and throws `ResourcesNotProvisioned`. - Delayed sends use the persistence's outgoing messages and dispatcher (no core change). The transport doesn't join the `withOutbox()` transaction. diff --git a/packages/bus-redis/src/redis-transport.integration.ts b/packages/bus-redis/src/redis-transport.integration.ts index 8ef0946e..e5718d1d 100644 --- a/packages/bus-redis/src/redis-transport.integration.ts +++ b/packages/bus-redis/src/redis-transport.integration.ts @@ -24,6 +24,7 @@ import { transportTests } from '@node-ts/bus-test' import { randomUUID } from 'node:crypto' +import { EventEmitter } from 'node:events' import { createClient } from 'redis' import { Mock } from 'typemoq' import { RedisTransport } from './redis-transport' @@ -493,7 +494,12 @@ describe('RedisTransport', () => { const handled = Promise.withResolvers() const bus = await startBus( 'frozen', - { connection: { url: `redis://127.0.0.1:${proxy.port}` } }, + { + connection: { + url: `redis://127.0.0.1:${proxy.port}`, + name: 'node-ts-bus-test-frozen' + } + }, async () => handled.resolve() ) await bus.send(new TestCommand(randomUUID(), new Date())) @@ -503,7 +509,10 @@ describe('RedisTransport', () => { !(await redis.sendCommand(['CLIENT', 'LIST'])) .split('\n') .some( - client => /cmd=xreadgroup/.test(client) && /flags=b/.test(client) + client => + client.includes('name=node-ts-bus-test-frozen ') && + /cmd=xreadgroup/.test(client) && + /flags=b/.test(client) ) ) { await new Promise(resolve => setTimeout(resolve, 20)) @@ -529,6 +538,106 @@ describe('RedisTransport', () => { }) }) + describe('and Redis stops answering while messages are still being sent', () => { + const proxy = new FreezableProxy({ + host: new URL(connection.url).hostname, + port: Number(new URL(connection.url).port || 6379) + }) + const handled = new EventEmitter() + let bus: BusInstance | undefined + let sending: NodeJS.Timeout | undefined + let recovered: boolean + let stopMs: number + let disposeMs: number + + /** + * Sends a message every 2 s, without waiting for it, as an API or a timer would. Each send writes to the + * connection, so its socket timeout never ends, and only a check that Redis answers notices it's frozen. + */ + const keepSending = (sender: BusInstance) => + setInterval(() => { + sender + .send(new TestCommand('sent-while-frozen', new Date())) + .catch(() => undefined) + }, 2_000) + + /** + * Sends a message until it's handled, or the deadline passes. A connection made while Redis was frozen is stuck + * until its socket timeout ends, so the first sends after it answers again may fail. + */ + const receivesWithin = async ( + sender: BusInstance, + ms: number + ): Promise => { + const value = randomUUID() + const deadline = Date.now() + ms + while (Date.now() < deadline) { + const received = new Promise(resolve => { + const timeout = setTimeout(() => resolve(false), 5_000) + handled.on('handled', handledValue => { + if (handledValue === value) { + clearTimeout(timeout) + resolve(true) + } + }) + }) + await sender + .send(new TestCommand(value, new Date())) + .catch(() => undefined) + if (await received) { + return true + } + } + return false + } + + beforeAll(async () => { + await proxy.start() + bus = await startBus( + 'frozen-while-sending', + { connection: { url: `redis://127.0.0.1:${proxy.port}` } }, + async command => { + handled.emit('handled', command.value) + } + ) + + proxy.freeze() + sending = keepSending(bus) + // Long enough for the watchdog to send a PING (every 5 s) and give up waiting for its answer (15 s) + await new Promise(resolve => setTimeout(resolve, 22_000)) + proxy.unfreeze() + clearInterval(sending) + recovered = await receivesWithin(bus, 45_000) + + proxy.freeze() + sending = keepSending(bus) + const stopping = Date.now() + await bus.stop() + stopMs = Date.now() - stopping + const disposing = Date.now() + await bus.dispose() + disposeMs = Date.now() - disposing + bus = undefined + }, 120_000) + + afterAll(async () => { + clearInterval(sending) + await bus?.dispose() + await proxy.close() + }) + + it('should replace the connections and receive again once Redis answers', () => { + expect(recovered).toEqual(true) + }) + + it('should stop and dispose without waiting for it', () => { + // The bus waits for the messages being handled, whose settling fails once the watchdog closes the connection + // (a PING every 5 s, answered within 15 s) + expect(stopMs).toBeLessThan(30_000) + expect(disposeMs).toBeLessThan(10_000) + }) + }) + describe('and a message cannot be parsed', () => { const queueName = 'unparseable' let bus: BusInstance @@ -569,6 +678,114 @@ describe('RedisTransport', () => { }) }) + describe('when a message in a batch cannot be dead-lettered', () => { + const queueName = 'batch-release' + const transport = new RedisTransport({ + ...configuration, + queueName, + // Long, so a message left pending isn't taken over during the test + visibilityTimeoutMs: 60_000 + }) + const ids: string[] = [] + let firstRound: PromiseSettledResult[] + let receivedAgain: { id: string; failedAttempts: number }[] + let pending: [string, string, number, number][] + + beforeAll(async () => { + await redis.sendCommand([ + 'XGROUP', + 'CREATE', + queueKey(queueName), + queueName, + '0', + 'MKSTREAM' + ]) + // Its dead letter stream is a string, so dead-lettering fails every time + await redis.sendCommand(['SET', deadLetterKey(queueName), 'not a stream']) + for (const body of [ + JSON.stringify(new TestCommand('1', new Date())), + 'not json', + JSON.stringify(new TestCommand('3', new Date())), + JSON.stringify(new TestCommand('4', new Date())) + ]) { + ids.push( + await redis.sendCommand([ + 'XADD', + queueKey(queueName), + '*', + 'body', + body, + 'attributes', + JSON.stringify({ + messageId: randomUUID(), + attributes: {}, + stickyAttributes: {} + }), + 'headers', + '{}' + ]) + ) + } + transport.prepare({ + loggerFactory: silentLogger, + messageSerializer + } as never) + await transport.connect() + await transport.start() + + // Four reads waiting, so all four messages are read in one batch + firstRound = await Promise.allSettled([ + transport.readNextMessage(), + transport.readNextMessage(), + transport.readNextMessage(), + transport.readNextMessage() + ]) + receivedAgain = [] + while (receivedAgain.length < 2) { + const message = await transport.readNextMessage() + if (message) { + receivedAgain.push({ + id: message.id!, + failedAttempts: message.failedAttempts + }) + await transport.deleteMessage(message) + } + } + pending = await redis.sendCommand<[string, string, number, number][]>([ + 'XPENDING', + queueKey(queueName), + queueName, + '-', + '+', + '10' + ]) + }) + + afterAll(async () => transport.dispose()) + + it('should give the message before it to a read, and fail the other reads', () => { + expect( + firstRound.map(result => + result.status === 'fulfilled' + ? (result.value as { id: string }).id + : 'rejected' + ) + ).toEqual([ids[0], 'rejected', 'rejected', 'rejected']) + }) + + it('should give back the messages after it straight away, without counting an attempt', () => { + expect(receivedAgain).toEqual([ + { id: ids[2], failedAttempts: 0 }, + { id: ids[3], failedAttempts: 0 } + ]) + }) + + it('should leave the message that failed pending until its visibility timeout ends', () => { + // The first message wasn't settled by the test, so it's pending too + expect(pending.map(([id]) => id)).toEqual([ids[0], ids[1]]) + }) + }) + describe('when sending to an address with no queue', () => { let error: unknown diff --git a/packages/bus-redis/src/redis-transport.spec.ts b/packages/bus-redis/src/redis-transport.spec.ts index dfc30d22..97df2d5c 100644 --- a/packages/bus-redis/src/redis-transport.spec.ts +++ b/packages/bus-redis/src/redis-transport.spec.ts @@ -239,20 +239,29 @@ describe('RedisTransport', () => { }) describe('when checking send options', () => { + let reservedError: unknown + let otherError: unknown + beforeAll(() => { sut = new RedisTransport(configuration) + try { + sut.assertSendOptions({ headers: { [FAILURE_HEADER]: 'x' } }) + } catch (error) { + reservedError = error + } + try { + sut.assertSendOptions({ headers: { 'x-tenant': 'a' } }) + } catch (error) { + otherError = error + } }) it('should reject the bus-failure header', () => { - expect(() => - sut.assertSendOptions({ headers: { [FAILURE_HEADER]: 'x' } }) - ).toThrow(TransportHeaderReserved) + expect(reservedError).toBeInstanceOf(TransportHeaderReserved) }) it('should accept other headers', () => { - expect(() => - sut.assertSendOptions({ headers: { 'x-tenant': 'a' } }) - ).not.toThrow() + expect(otherError).toBeUndefined() }) }) diff --git a/packages/bus-redis/src/redis-transport.ts b/packages/bus-redis/src/redis-transport.ts index c26b0cb7..c2fe6893 100644 --- a/packages/bus-redis/src/redis-transport.ts +++ b/packages/bus-redis/src/redis-transport.ts @@ -67,16 +67,26 @@ export const DEFAULT_DEAD_LETTER_RETENTION_MS: Milliseconds = const READ_BLOCK_MS = 1_000 /** - * How long a connection may go without receiving anything before it's closed as lost, and reconnected. node-redis' - * command timeout only covers a command waiting to be written, not one waiting for its reply, so without it a - * connection that stops answering, such as through a frozen proxy or a half-open socket, would leave reads and - * scripts waiting forever. Each connection sends a PING every `PING_INTERVAL_MS`, so an idle one isn't closed. The - * receiving connection's is longer than a read blocks for. + * How long a connection may go without any traffic before it's closed as lost, and reconnected. node-redis' command + * timeout only covers a command waiting to be written, not one waiting for its reply, so without it a connection + * that stops answering, such as through a frozen proxy or a half-open socket, would leave reads and scripts waiting + * forever. Each connection sends a PING every `PING_INTERVAL_MS`, so an idle one isn't closed. The receiving + * connection's is longer than a read blocks for. + * + * Writes count as traffic too, so a connection that's written to often never times out. The watchdog covers that. */ const SOCKET_TIMEOUT_MS = 15_000 const READ_SOCKET_TIMEOUT_MS = READ_BLOCK_MS + 10_000 const PING_INTERVAL_MS = 5_000 +/** + * How often the watchdog sends each connection a PING, and how long the connection has to answer before it's closed, + * failing whatever was waiting on it, and replaced. It notices a connection that stopped answering however often + * it's written to. + */ +const WATCHDOG_INTERVAL_MS = 5_000 +const REPLY_TIMEOUT_MS = 15_000 + /** * The longest `stop()` waits for a read in flight before it closes the receiving connection, and for the messages * it gives back @@ -300,6 +310,11 @@ export class RedisTransport implements Transport { * The read of the queue in flight, which `stop()` waits for */ private fetching: Promise | undefined + private watchdogTimer: NodeJS.Timeout | undefined + /** + * Whether the watchdog is checking the connections, so checks don't overlap + */ + private isCheckingConnections = false private lastReclaimAt = 0 private lastConsumerCleanupAt = 0 @@ -360,15 +375,10 @@ export class RedisTransport implements Transport { return } this.logger.info('Connecting redis transport') - const client = createRedisClient(this.configuration.connection ?? {}) - // A client without an error listener crashes the process when its connection fails. node-redis reconnects. - client.on('error', error => - this.logger.warn('Redis connection error', { - error: serializeError(error) - }) - ) + const client = this.createMainClient() await client.connect() this.client = client + this.startWatchdog() } /** @@ -376,6 +386,7 @@ export class RedisTransport implements Transport { * removed from the queue's consumer group first, so stopped processes don't leave consumers behind. */ async disconnect(): Promise { + this.stopWatchdog() if (this.hasStarted) { // The bus disconnects once its workers have settled the messages they were handling await settleWithin(this.leaveConsumerGroup(), DISCONNECT_TIMEOUT_MS) @@ -571,23 +582,8 @@ export class RedisTransport implements Transport { if (this.isStarted) { return } - const client = this.getClient() - const socket = client.options.socket ?? {} - const reader = client.duplicate({ - // Longer than a read blocks for, so only a connection that stopped answering times out - socket: { - ...socket, - socketTimeout: Math.max( - socket.socketTimeout ?? 0, - READ_SOCKET_TIMEOUT_MS - ) - } - }) - reader.on('error', error => - this.logger.warn('Redis connection error while receiving', { - error: serializeError(error) - }) - ) + this.getClient() + const reader = this.createReader() await reader.connect() this.reader = reader this.isStarted = true @@ -902,9 +898,10 @@ export class RedisTransport implements Transport { } /** - * Gives received entries to the waiting reads. If one can't be converted or dead-lettered, the entries not given - * to a read yet, including that one, are given back, so they're received again straight away rather than after the - * visibility timeout, and the error is thrown. + * Gives received entries to the waiting reads. If one can't be converted or dead-lettered, the entries after it are + * given back, so they're received again straight away rather than after the visibility timeout, and the error is + * thrown. The one that failed is left pending until its visibility timeout ends, and then counts a failed attempt, + * so one that fails every time doesn't loop without being counted. */ private async deliver(entries: ReceivedEntry[]): Promise { for (const [index, entry] of entries.entries()) { @@ -912,7 +909,7 @@ export class RedisTransport implements Transport { try { message = await this.toTransportMessage(entry) } catch (error) { - await this.releaseAll(entries.slice(index)) + await this.releaseAll(entries.slice(index + 1)) throw error } if (!message) { @@ -1136,6 +1133,119 @@ export class RedisTransport implements Transport { } } + /** + * Creates the connection that sends and settles messages + */ + private createMainClient(): RedisClient { + const client = createRedisClient(this.configuration.connection ?? {}) + // A client without an error listener crashes the process when its connection fails. node-redis reconnects. + client.on('error', error => + this.logger.warn('Redis connection error', { + error: serializeError(error) + }) + ) + return client + } + + /** + * Creates the connection that waits for new messages + */ + private createReader(): RedisClient { + const connection = this.configuration.connection ?? {} + const socket = connection.socket ?? {} + const reader = createRedisClient({ + ...connection, + // Longer than a read blocks for, so only a connection that stopped answering times out + socket: { + ...socket, + socketTimeout: Math.max( + socket.socketTimeout ?? 0, + READ_SOCKET_TIMEOUT_MS + ) + } + }) + reader.on('error', error => + this.logger.warn('Redis connection error while receiving', { + error: serializeError(error) + }) + ) + return reader + } + + private startWatchdog(): void { + clearInterval(this.watchdogTimer) + this.watchdogTimer = setInterval( + // checkConnections() handles its own errors + () => void this.checkConnections(), + WATCHDOG_INTERVAL_MS + ) + // The watchdog alone doesn't keep the process running + this.watchdogTimer.unref() + } + + private stopWatchdog(): void { + clearInterval(this.watchdogTimer) + this.watchdogTimer = undefined + } + + /** + * Checks each connection answers, unless a check is still in flight + */ + private async checkConnections(): Promise { + if (this.isCheckingConnections) { + return + } + this.isCheckingConnections = true + try { + await Promise.all([ + this.checkConnection('client'), + this.checkConnection('reader') + ]) + } catch (error) { + this.logger.warn('Could not check the connections to Redis', { + error: serializeError(error) + }) + } finally { + this.isCheckingConnections = false + } + } + + /** + * Sends a connection a PING. If it doesn't answer within `REPLY_TIMEOUT_MS`, it's closed at once, which fails the + * commands waiting on it, and replaced with a new connection. A connection that's reconnecting isn't checked, since + * node-redis is already dealing with it. + */ + private async checkConnection(role: 'client' | 'reader'): Promise { + const client = this[role] + if (!client?.isReady) { + return + } + const answer = await settleWithin( + client.sendCommand(['PING']).then( + () => 'answered', + () => 'failed' + ), + REPLY_TIMEOUT_MS + ) + // It may have been replaced, or closed by stop() or disconnect(), in the meantime + if (answer !== TIMED_OUT || this[role] !== client) { + return + } + this.logger.warn( + `Redis didn't answer within ${REPLY_TIMEOUT_MS} ms, so the connection was closed and is being replaced. Commands waiting on it fail.`, + { connection: role === 'client' ? 'sending' : 'receiving' } + ) + const replacement = + role === 'client' ? this.createMainClient() : this.createReader() + this[role] = replacement + client.destroy() + replacement.connect().catch(error => + this.logger.warn('Could not reconnect to Redis', { + error: serializeError(error) + }) + ) + } + private getClient(): RedisClient { if (!this.client) { throw new RedisTransportNotConnected() diff --git a/packages/bus-redis/src/test/freezable-proxy.ts b/packages/bus-redis/src/test/freezable-proxy.ts index 46098227..a59b7dce 100644 --- a/packages/bus-redis/src/test/freezable-proxy.ts +++ b/packages/bus-redis/src/test/freezable-proxy.ts @@ -44,6 +44,14 @@ export class FreezableProxy { this.frozen = true } + /** + * Passes data again. What was sent while frozen is lost, so the connections open then stay broken, as a client + * would find them after a network fault, and only new connections work. + */ + unfreeze(): void { + this.frozen = false + } + /** * Closes every connection and stops listening */ From 4d0c4d7db99c54679c183483f9227288ae9d45a1 Mon Sep 17 00:00:00 2001 From: Andrew den Hertog Date: Thu, 8 Oct 2026 15:49:52 +1000 Subject: [PATCH 4/4] name redis command timeouts, quiet expected reconnect failures, remove test listeners Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/bus-redis/CLAUDE.md | 2 +- packages/bus-redis/src/error/index.ts | 1 + .../src/error/redis-command-timed-out.ts | 27 +++++ .../src/redis-transport.integration.ts | 15 ++- packages/bus-redis/src/redis-transport.ts | 110 +++++++++++++----- 5 files changed, 117 insertions(+), 38 deletions(-) create mode 100644 packages/bus-redis/src/error/redis-command-timed-out.ts diff --git a/packages/bus-redis/CLAUDE.md b/packages/bus-redis/CLAUDE.md index d28b360b..95b8b7b9 100644 --- a/packages/bus-redis/CLAUDE.md +++ b/packages/bus-redis/CLAUDE.md @@ -5,7 +5,7 @@ A Redis Streams transport (node-redis 6, a peer dependency), replacing the archi ## Design - **Config** (`src/redis-transport-configuration.ts`): `queueName`, `connection` (node-redis `createClient()` options minus `RESP`/modules/scripts/typeMapping, default `{}`), `keyPrefix` (default `bus`), `visibilityTimeoutMs` (30 s), `deadLetterRetentionMs` (14 days; `0`/`Infinity` keep everything). No `deadLetterQueueName`: dead letters are per queue. `queueName`/`keyPrefix` with `{`/`}` or empty throw `InvalidRedisKeyName`; durations must be whole milliseconds (Redis rejects a fractional `IDLE`), else `InvalidRedisTransportDuration`. Using it before `connect()` throws `RedisTransportNotConnected`. -- **Connections**: `connect()` makes the command client (`createClient({ ...connection, RESP: 2 })`, so raw replies always have RESP2 shapes; everything goes through `sendCommand`, `multi().addCommand()` or scripts). node-redis' command timeout stops covering a command once it's written to the socket (6.3, `commands-queue.js`), so a frozen or half-open connection would leave a read or script pending forever. So every client gets `socket.socketTimeout` (15 s; the reader's is at least the block time + 10 s), `pingInterval` 5 s (so idle connections aren't timed out; hence `+ping` in the ACL) and a `reconnectStrategy` that also reconnects after a socket timeout (node-redis' default doesn't); settings in `connection` win. Writes count as socket activity, so a frozen connection that's written to often never times out: the watchdog (`startWatchdog`, from `connect()` to `disconnect()`, unref'd, never overlapping itself) sends each ready client a PING every 5 s and, without a reply in 15 s, `destroy()`s it (failing what's pending) and replaces it (`createMainClient`/`createReader`). A replacement made while Redis is frozen is stuck in its handshake until its socket timeout, so recovery after Redis answers again can take another 15 s. `stop()` races the fetch in flight against 3 s and then `destroy()`s the reader, and bounds giving back messages to 3 s; `disconnect()` bounds leaving the group and `close()` to 3 s each, then `destroy()`s (`settleWithin`/`closeClient`). The integration tests freeze a TCP proxy (`src/test/freezable-proxy.ts`) during a blocked read, and while sending every 2 s (about 70 s, the slowest test). `start()` makes the reader with `duplicate()`. Every client has an `error` listener. Only standalone Redis/Valkey (no Cluster or Sentinel in v1), but keys stay Cluster-ready. +- **Connections**: `connect()` makes the command client (`createClient({ ...connection, RESP: 2 })`, so raw replies always have RESP2 shapes; everything goes through `sendCommand`, `multi().addCommand()` or scripts). node-redis' command timeout stops covering a command once it's written to the socket (6.3, `commands-queue.js`), so a frozen or half-open connection would leave a read or script pending forever. So every client gets `socket.socketTimeout` (15 s; the reader's is at least the block time + 10 s), `pingInterval` 5 s (so idle connections aren't timed out; hence `+ping` in the ACL) and a `reconnectStrategy` that also reconnects after a socket timeout (node-redis' default doesn't); settings in `connection` win. Writes count as socket activity, so a frozen connection that's written to often never times out: the watchdog (`startWatchdog`, from `connect()` to `disconnect()`, unref'd, never overlapping itself) sends each ready client a PING every 5 s and, without a reply in 15 s, `destroy()`s it (failing what's pending) and replaces it (`createMainClient`/`createReader`). A replacement made while Redis is frozen is stuck in its handshake until its socket timeout, so recovery after Redis answers again can take another 15 s. node-redis' command timeout error has no message, so `nameTimeout` rethrows it as `RedisCommandTimedOut` naming the command (or script) and connection; a replacement that fails to connect after being closed or replaced again isn't logged. `stop()` races the fetch in flight against 3 s and then `destroy()`s the reader, and bounds giving back messages to 3 s; `disconnect()` bounds leaving the group and `close()` to 3 s each, then `destroy()`s (`settleWithin`/`closeClient`). The integration tests freeze a TCP proxy (`src/test/freezable-proxy.ts`) during a blocked read, and while sending every 2 s (about 70 s, the slowest test). `start()` makes the reader with `duplicate()`. Every client has an `error` listener. Only standalone Redis/Valkey (no Cluster or Sentinel in v1), but keys stay Cluster-ready. - **Keys** (`src/redis-keys.ts`, internal): `:{}:queue` (stream, consumer group ``), `:{}:delayed` (zset), `:{}:dead-letter` (stream), `:subscriptions:<$name>` (set of queue names). Every script touches one queue's keys only, which share the `{}` hash tag. `escapeGlob` escapes ACL key patterns. - **Entries**: fields `body` (serializer output), `attributes` (JSON), `headers` (JSON) and `failedAttempts` once returned; the dead letter copy adds `bus-failure` and drops `failedAttempts`. Only `bus-failure` is a reserved header. `TransportMessage.id` is the entry id (new on each retry). - **Sending**: `SMEMBERS` the subscription set, then one `MULTI` of `XADD NOMKSTREAM * …` per subscriber (all or nothing on a single primary). No subscribers warns and drops; a null reply (stream missing) warns. `sendToAddress` is `XADD NOMKSTREAM`, `EndpointNotFound` on null or an address with braces. diff --git a/packages/bus-redis/src/error/index.ts b/packages/bus-redis/src/error/index.ts index e00c74fc..446670f1 100644 --- a/packages/bus-redis/src/error/index.ts +++ b/packages/bus-redis/src/error/index.ts @@ -1,3 +1,4 @@ export * from './invalid-redis-key-name' export * from './invalid-redis-transport-duration' +export * from './redis-command-timed-out' export * from './redis-transport-not-connected' diff --git a/packages/bus-redis/src/error/redis-command-timed-out.ts b/packages/bus-redis/src/error/redis-command-timed-out.ts new file mode 100644 index 00000000..f4831b99 --- /dev/null +++ b/packages/bus-redis/src/error/redis-command-timed-out.ts @@ -0,0 +1,27 @@ +/** + * Thrown when a command to Redis times out, such as while a connection that stopped answering is being replaced, so + * the command couldn't be sent. node-redis' own timeout error has no message, so this one says which command and + * connection it was. + */ +export class RedisCommandTimedOut extends Error { + readonly help: string + + /** + * @param command the command, or the name of the transport's script, that timed out + * @param connection the transport's connection it was sent on: the one that sends and settles messages, or the + * one that receives them + * @param cause node-redis' timeout error + */ + constructor( + readonly command: string, + readonly connection: 'sending' | 'receiving', + readonly cause?: unknown + ) { + super( + `Redis command ${command} timed out on RedisTransport's ${connection} connection` + ) + this.help = + "Check that Redis is running and reachable. The transport replaces a connection that doesn't answer, and the message is received again once Redis answers." + Object.setPrototypeOf(this, new.target.prototype) + } +} diff --git a/packages/bus-redis/src/redis-transport.integration.ts b/packages/bus-redis/src/redis-transport.integration.ts index e5718d1d..1dbfb257 100644 --- a/packages/bus-redis/src/redis-transport.integration.ts +++ b/packages/bus-redis/src/redis-transport.integration.ts @@ -573,13 +573,18 @@ describe('RedisTransport', () => { const deadline = Date.now() + ms while (Date.now() < deadline) { const received = new Promise(resolve => { - const timeout = setTimeout(() => resolve(false), 5_000) - handled.on('handled', handledValue => { + const onHandled = (handledValue: string) => { if (handledValue === value) { - clearTimeout(timeout) - resolve(true) + settle(true) } - }) + } + const settle = (wasHandled: boolean) => { + clearTimeout(timeout) + handled.off('handled', onHandled) + resolve(wasHandled) + } + const timeout = setTimeout(() => settle(false), 5_000) + handled.on('handled', onHandled) }) await sender .send(new TestCommand(value, new Date())) diff --git a/packages/bus-redis/src/redis-transport.ts b/packages/bus-redis/src/redis-transport.ts index c2fe6893..091967a5 100644 --- a/packages/bus-redis/src/redis-transport.ts +++ b/packages/bus-redis/src/redis-transport.ts @@ -26,11 +26,12 @@ import { } from '@node-ts/bus-messages' import { randomUUID } from 'node:crypto' import { hostname } from 'node:os' -import { createClient } from 'redis' +import { createClient, TimeoutError } from 'redis' import { serializeError } from 'serialize-error' import { InvalidRedisKeyName, InvalidRedisTransportDuration, + RedisCommandTimedOut, RedisTransportNotConnected } from './error' import { DEFAULT_KEY_PREFIX, isValidKeyName, RedisKeys } from './redis-keys' @@ -177,6 +178,25 @@ const createRedisClient = (options: RedisConnectionOptions) => RESP: 2 }) +/** + * Waits for a command, replacing node-redis' timeout error, which has no message, with one that names the command + * and connection + * @throws RedisCommandTimedOut if the command timed out + */ +const nameTimeout = async ( + command: Promise, + name: string, + connection: 'sending' | 'receiving' +): Promise => { + try { + return await command + } catch (error) { + throw error instanceof TimeoutError + ? new RedisCommandTimedOut(name, connection, error) + : error + } +} + const TIMED_OUT = Symbol('timed out') /** @@ -563,13 +583,17 @@ export class RedisTransport implements Transport { throw new EndpointNotFound(address, 'RedisTransport') } // NOMKSTREAM adds nothing to a stream that doesn't exist, rather than creating one nothing reads - const id = await this.getClient().sendCommand([ + const id = await nameTimeout( + this.getClient().sendCommand([ + 'XADD', + this.keys.queue(address), + 'NOMKSTREAM', + '*', + ...fields + ]), 'XADD', - this.keys.queue(address), - 'NOMKSTREAM', - '*', - ...fields - ]) + 'sending' + ) if (id === null) { throw new EndpointNotFound(address, 'RedisTransport') } @@ -729,10 +753,14 @@ export class RedisTransport implements Transport { ): Promise { const fields = this.toFields(message, messageAttributes, sendOptions) const client = this.getClient() - const subscribers = await client.sendCommand([ + const subscribers = await nameTimeout( + client.sendCommand([ + 'SMEMBERS', + this.keys.subscriptions(message.$name) + ]), 'SMEMBERS', - this.keys.subscriptions(message.$name) - ]) + 'sending' + ) const queues = subscribers.filter(queue => { if (isValidKeyName(queue)) { return true @@ -761,7 +789,11 @@ export class RedisTransport implements Transport { ...fields ]) ) - const ids = (await transaction.exec()) as unknown[] + const ids = (await nameTimeout( + transaction.exec(), + 'MULTI', + 'sending' + )) as unknown[] ids.forEach((id, index) => { if (id === null) { this.logger.warn( @@ -940,21 +972,23 @@ export class RedisTransport implements Transport { if (!reader) { return [] } - const reply = await reader.sendCommand< - [stream: string, entries: RawStreamEntry[]][] | null - >([ + const reply = await nameTimeout( + reader.sendCommand<[stream: string, entries: RawStreamEntry[]][] | null>([ + 'XREADGROUP', + 'GROUP', + this.endpointName, + this.consumerName, + 'COUNT', + String(count), + 'BLOCK', + String(blockMs), + 'STREAMS', + this.keys.queue(this.endpointName), + '>' + ]), 'XREADGROUP', - 'GROUP', - this.endpointName, - this.consumerName, - 'COUNT', - String(count), - 'BLOCK', - String(blockMs), - 'STREAMS', - this.keys.queue(this.endpointName), - '>' - ]) + 'receiving' + ) if (!reply) { return [] } @@ -1110,13 +1144,22 @@ export class RedisTransport implements Transport { ): Promise { const client = this.getClient() const tail = [String(keys.length), ...keys, ...args] + const name = `script ${redisScript.name}` try { - return await client.sendCommand(['EVALSHA', redisScript.sha, ...tail]) + return await nameTimeout( + client.sendCommand(['EVALSHA', redisScript.sha, ...tail]), + name, + 'sending' + ) } catch (error) { if (!(error instanceof Error && error.message.startsWith('NOSCRIPT'))) { throw error } - return client.sendCommand(['EVAL', redisScript.source, ...tail]) + return nameTimeout( + client.sendCommand(['EVAL', redisScript.source, ...tail]), + name, + 'sending' + ) } } @@ -1239,11 +1282,14 @@ export class RedisTransport implements Transport { role === 'client' ? this.createMainClient() : this.createReader() this[role] = replacement client.destroy() - replacement.connect().catch(error => - this.logger.warn('Could not reconnect to Redis', { - error: serializeError(error) - }) - ) + replacement.connect().catch(error => { + // A replacement closed by disconnect() or stop(), or replaced in turn, fails to connect as expected + if (this[role] === replacement) { + this.logger.warn('Could not reconnect to Redis', { + error: serializeError(error) + }) + } + }) } private getClient(): RedisClient {