From 5b69e413be4bdb86f66d4aeb2f957ab65d2f77bf Mon Sep 17 00:00:00 2001 From: Nicolaus Weidner Date: Fri, 21 Aug 2026 16:53:25 +0200 Subject: [PATCH 1/8] RDSC-5610: Use actual source name in secret and env var names --- .../data-pipelines/pipeline-config.md | 23 +++++++++++------ .../data-pipelines/prepare-dbs/oracle.md | 8 +++--- .../data-pipelines/prepare-dbs/snowflake.md | 21 +++++++++------- .../data-pipelines/prepare-dbs/sql-server.md | 25 ++++++++++++------- .../data-pipelines/prepare-dbs/supabase.md | 13 +++++----- 5 files changed, 54 insertions(+), 36 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/pipeline-config.md b/content/integrate/redis-data-integration/data-pipelines/pipeline-config.md index 3d1072ff9e..f9d137e737 100644 --- a/content/integrate/redis-data-integration/data-pipelines/pipeline-config.md +++ b/content/integrate/redis-data-integration/data-pipelines/pipeline-config.md @@ -41,9 +41,10 @@ sources: type: mysql host: # e.g. localhost port: 3306 - # User and password are injected from the secrets. - user: ${SOURCE_DB_USERNAME} - password: ${SOURCE_DB_PASSWORD} + # User and password are injected from the secrets. The prefix comes from + # the source name above. + user: ${MYSQL_DB_USERNAME} + password: ${MYSQL_DB_PASSWORD} # Additional properties for the source collector: # List of databases to include (optional). # databases: @@ -194,11 +195,17 @@ and [`processors`](#processors). ### Sources -The `sources` section has a subsection for the source that -you need to configure. The source section starts with a unique name -to identify the source (in the example, there is a source -called `mysql` but you can choose any name you like). The example -configuration contains the following data: +The `sources` section has one subsection per source database. Each subsection starts +with the source name, which identifies the source and must be unique (in the example, +the source is called `mysql`). + +The source name also determines the environment variables that carry the source's +credentials, for example `${MYSQL_DB_USERNAME}` and `${MYSQL_DB_PASSWORD}` for a +source named `mysql`. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) +for the source naming rules and for capturing from more than one source database. + +The example configuration contains the following data: - `type`: The type of collector to use for the pipeline. Currently, the only types RDI supports are `cdc` and `external`. diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md index 5d8d2b09f7..d182a2841e 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md @@ -1034,8 +1034,8 @@ sources: type: oracle host: host.docker.internal port: 1521 - user: ${SOURCE_DB_USERNAME} - password: ${SOURCE_DB_PASSWORD} + user: ${ORACLE_DB_USERNAME} + password: ${ORACLE_DB_PASSWORD} advanced: source: database.dbname: ORCLCDB @@ -1118,8 +1118,8 @@ sources: type: oracle host: oracle port: 1521 - user: ${SOURCE_DB_USERNAME} - password: ${SOURCE_DB_PASSWORD} + user: ${ORACLE_DB_USERNAME} + password: ${ORACLE_DB_PASSWORD} database: ORCLCDB advanced: source: diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/snowflake.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/snowflake.md index 4b4550fc80..7b7e882e80 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/snowflake.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/snowflake.md @@ -146,11 +146,14 @@ Before deploying the RDI pipeline, configure the necessary secrets. ### Password authentication +The secret names and keys come from the source name, which for the source configured in +the next step is `snowflake`: + ```bash -kubectl create secret generic source-db \ +kubectl create secret generic snowflake-db \ --namespace=rdi \ - --from-literal=SOURCE_DB_USERNAME=your_username \ - --from-literal=SOURCE_DB_PASSWORD=your_password + --from-literal=SNOWFLAKE_DB_USERNAME=your_username \ + --from-literal=SNOWFLAKE_DB_PASSWORD=your_password ``` ### Private key authentication @@ -158,17 +161,17 @@ kubectl create secret generic source-db \ Create a secret with the private key file: ```bash -kubectl create secret generic source-db-ssl \ +kubectl create secret generic snowflake-db-ssl \ --namespace=rdi \ --from-file=client.key=/path/to/rsa_key.p8 ``` -Also create the source-db secret with the username: +Also create the `snowflake-db` secret with the username: ```bash -kubectl create secret generic source-db \ +kubectl create secret generic snowflake-db \ --namespace=rdi \ - --from-literal=SOURCE_DB_USERNAME=your_username + --from-literal=SNOWFLAKE_DB_USERNAME=your_username ``` ## 4. Configure RDI for Snowflake @@ -182,8 +185,8 @@ sources: connection: type: snowflake url: "jdbc:snowflake://myaccount.snowflakecomputing.com/" - user: "${SOURCE_DB_USERNAME}" - password: "${SOURCE_DB_PASSWORD}" # Omit for key-pair auth + user: "${SNOWFLAKE_DB_USERNAME}" + password: "${SNOWFLAKE_DB_PASSWORD}" # Omit for key-pair auth database: "MYDB" warehouse: "COMPUTE_WH" # role: "RDI_ROLE" # Optional: Snowflake role diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md index 19f91ca52f..ab3b41c2cd 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md @@ -610,8 +610,8 @@ sources: host: .database.windows.net port: 1433 database: - user: ${SOURCE_DB_USERNAME} - password: ${SOURCE_DB_PASSWORD} + user: ${SQLSERVER_DB_USERNAME} + password: ${SQLSERVER_DB_PASSWORD} logging: level: info schemas: @@ -646,20 +646,27 @@ Debezium SQL Server connector and JDBC driver. The Azure-specific values are: | `snapshot.mode` | The Debezium snapshot strategy. | `initial`. Captures a snapshot of the existing rows, then streams subsequent changes from the CDC tables. | For SQL authentication, omit the `driver.authentication` line and set -`${SOURCE_DB_USERNAME}` and `${SOURCE_DB_PASSWORD}` to the SQL user's credentials. +`${SQLSERVER_DB_USERNAME}` and `${SQLSERVER_DB_PASSWORD}` to the SQL user's credentials. Keep the other Azure-specific properties. #### Secret mapping -For Microsoft Entra service-principal authentication, the RDI source secret must +For Microsoft Entra service-principal authentication, the source's credentials must provide: -| Secret key | Value | +| Config reference | Value | | --- | --- | -| `SOURCE_DB_USERNAME` | The service principal's **Application (client) ID** (a GUID). | -| `SOURCE_DB_PASSWORD` | The service principal's **client secret**. | +| `${SQLSERVER_DB_USERNAME}` | The service principal's **Application (client) ID** (a GUID). | +| `${SQLSERVER_DB_PASSWORD}` | The service principal's **client secret**. | -{{< warning >}}The `SOURCE_DB_USERNAME` value is the client ID (a GUID), but the contained +Set them with the source name, which for the source in this example is `sqlserver`: + +```bash +redis-di set-secret USERNAME --db sqlserver +redis-di set-secret PASSWORD --db sqlserver +``` + +{{< warning >}}The username value is the client ID (a GUID), but the contained database user created in the previous section uses the service principal's **display name**. These are two different identifiers for the same principal — mixing them up is the most common cause of `Login failed for user ''` errors @@ -711,7 +718,7 @@ GO - **`Login failed for user ''`** — the contained database user was not created for this service principal, or it was created with the wrong identifier. Verify that the `CREATE USER ... FROM EXTERNAL PROVIDER` statement used - the service principal's display name, and that `SOURCE_DB_USERNAME` contains its + the service principal's display name, and that `${SQLSERVER_DB_USERNAME}` contains its client ID. Query `sys.database_principals` on the source database to see which principals exist. - **`SSL Server certificate validation failed` or hostname mismatch** — diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/supabase.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/supabase.md index bcc2e31454..1da4762516 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/supabase.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/supabase.md @@ -163,12 +163,13 @@ In the Supabase dashboard, go to 1. Enable **Enforce SSL on incoming connections**. 1. Download the Supabase CA certificate. -Store the database username, password, and CA certificate as RDI secrets: +Store the database username, password, and CA certificate as RDI secrets. Pass the +source name with `--db`; the source configured in the next step is named `supabase`: ```bash -redis-di set-secret SOURCE_DB_USERNAME rdi_replication -redis-di set-secret SOURCE_DB_PASSWORD '' -redis-di set-secret SOURCE_DB_CACERT /path/to/prod-ca-2021.crt +redis-di set-secret USERNAME --db supabase rdi_replication +redis-di set-secret PASSWORD --db supabase '' +redis-di set-secret CACERT --db supabase /path/to/prod-ca-2021.crt ``` RDI verifies that the direct endpoint hostname matches the certificate. @@ -187,8 +188,8 @@ sources: host: db..supabase.co port: 5432 database: postgres - user: ${SOURCE_DB_USERNAME} - password: ${SOURCE_DB_PASSWORD} + user: ${SUPABASE_DB_USERNAME} + password: ${SUPABASE_DB_PASSWORD} schemas: - public tables: From c4855e6c5c486d4340f5721fbb630ff09d6462ca Mon Sep 17 00:00:00 2001 From: Nicolaus Weidner Date: Fri, 21 Aug 2026 19:22:59 +0200 Subject: [PATCH 2/8] RDSC-5610: Add page for multi-source, adapt formulations on some main pages --- .../architecture/_index.md | 28 +-- .../data-pipelines/_index.md | 10 +- .../data-pipelines/multiple-sources.md | 226 ++++++++++++++++++ 3 files changed, 247 insertions(+), 17 deletions(-) create mode 100644 content/integrate/redis-data-integration/data-pipelines/multiple-sources.md diff --git a/content/integrate/redis-data-integration/architecture/_index.md b/content/integrate/redis-data-integration/architecture/_index.md index f9d09f0453..1bc7867f2a 100644 --- a/content/integrate/redis-data-integration/architecture/_index.md +++ b/content/integrate/redis-data-integration/architecture/_index.md @@ -20,13 +20,13 @@ weight: 30 ## Overview -RDI implements a [change data capture](https://en.wikipedia.org/wiki/Change_data_capture) (CDC) pattern that tracks changes to the data in a -non-Redis *source* database and makes corresponding changes to a Redis +RDI implements a [change data capture](https://en.wikipedia.org/wiki/Change_data_capture) (CDC) pattern that tracks changes to the data in one or +more non-Redis *source* databases and makes corresponding changes to a Redis *target* database. You can use the target as a cache to improve performance because it will typically handle read queries much faster than the source. To use RDI, you define a *dataset* that specifies which data items -you want to capture from the source and how you want to +you want to capture from each source and how you want to represent them in the target. For example, if the source is a relational database then you specify which table columns you want to capture but you don't need to store them in an equivalent table @@ -35,15 +35,15 @@ representation is most suitable for your app. To convert from the source to the target representation, RDI applies *transformations* to the data after capture. -RDI synchronizes the dataset between the source and target using +RDI synchronizes the dataset between the sources and the target using a *data pipeline* that implements several processing steps in sequence: -1. A *CDC collector* captures changes to the source database. RDI - currently uses an open source collector called +1. A *CDC collector* per source captures changes to the source + databases. RDI currently uses an open source collector called [Debezium](https://debezium.io/) for this step. -1. The collector records the captured changes using +1. Each collector records the captured changes using [Redis streams]({{< relref "/develop/data-types/streams" >}}) in the RDI database. @@ -60,12 +60,12 @@ its state and configuration data and also the change data streams in a Redis dat {{< image filename="images/rdi/ingest/ingest-dataflow.webp" >}} When you first start RDI, the target database is empty and so all -of the data in the source database is essentially "change" data. +of the data in the source databases is essentially "change" data. RDI collects this data in a phase called *initial cache loading*, which can take minutes or hours to finish, depending on the size of the source data. Once the initial cache loading is complete, there is a *snapshot* dataset in the target that will gradually -change when new data gets captured from the source. At this point, +change when new data gets captured from the sources. At this point, RDI automatically enters a second phase called *change streaming*, where changes in the data are captured as they happen. Changes are usually added to the target within a few seconds after capture. @@ -83,7 +83,7 @@ overall state). ## Checkpointing RDI uses Redis streams to store the sequence of change events -captured from the source. The events are then retrieved in order +captured from the sources. The events are then retrieved in order from the streams, processed, and written to the target. The stream processor uses a *checkpoint* mechanism to keep track of the last event in the sequence that it has successfully processed and stored. If the processor fails @@ -96,11 +96,11 @@ face of failures. Sometimes, data records can get added to the streams faster than RDI can process them. This can happen if the target is slowed or disconnected -or simply if the source quickly generates a lot of change data. +or simply if a source quickly generates a lot of change data. If this continues, then the streams will eventually occupy all the available memory. When RDI detects this situation, it applies a *backpressure* mechanism to slow or stop the flow of incoming data. -Change data is held at the source until RDI clears the backlog and has +Change data is held at the sources until RDI clears the backlog and has enough free memory to resume streaming. {{}}The Debezium log sometimes reports that RDI has run out @@ -128,7 +128,7 @@ It includes: and exports them as [Prometheus](https://prometheus.io/) metrics. The *data plane* contains the processes that actually move the data. -It includes the *CDC collector* and the *stream processor* that implement +It includes a *CDC collector* for each source and the *stream processor*, which implement the two phases of the pipeline lifecycle (initial cache loading and change streaming). The *management plane* provides tools that let you interact @@ -164,7 +164,7 @@ deploy RDI. ### RDI on your own VMs -For this deployment, you must provide two VMs. The collector and stream processor +For this deployment, you must provide two VMs. The collectors and stream processor are active on one VM, while on the other they are in standby to provide high availability. The two operators running on both VMs use a leader election algorithm to decide which VM is the active one (the "leader"). diff --git a/content/integrate/redis-data-integration/data-pipelines/_index.md b/content/integrate/redis-data-integration/data-pipelines/_index.md index 62f7f255b6..8644412b71 100644 --- a/content/integrate/redis-data-integration/data-pipelines/_index.md +++ b/content/integrate/redis-data-integration/data-pipelines/_index.md @@ -28,9 +28,9 @@ deploy them. ## How a pipeline works -An RDI pipeline captures change data records from the source database, and transforms them -into Redis data structures. It writes each of these new structures to a Redis target -database under its own key. +An RDI pipeline captures change data records from one or more source databases, and +transforms them into Redis data structures. It writes each of these new structures to a +Redis target database under its own key. By default, RDI transforms the source data into [hashes]({{< relref "/develop/data-types/hashes" >}}) or @@ -94,6 +94,10 @@ to the snapshot phase. When this is complete, the pipeline continues with CDC as Follow the steps described in the sections below to prepare and run an RDI pipeline. +The following example uses a single source pipeline. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) +for details on how to capture multiple sources in one pipeline. + ### 1. Prepare the source database Before using the pipeline you must first prepare your source database to use diff --git a/content/integrate/redis-data-integration/data-pipelines/multiple-sources.md b/content/integrate/redis-data-integration/data-pipelines/multiple-sources.md new file mode 100644 index 0000000000..d970e7f151 --- /dev/null +++ b/content/integrate/redis-data-integration/data-pipelines/multiple-sources.md @@ -0,0 +1,226 @@ +--- +Title: Multiple sources in one pipeline +alwaysopen: false +categories: +- docs +- integrate +- rs +- rdi +description: Learn how to ingest from several source databases into one Redis target. +group: di +linkTitle: Multiple sources +summary: Redis Data Integration keeps Redis in sync with the primary database in near + real time. +type: integration +weight: 4 +--- + +One RDI pipeline can capture changes from several source databases and write them all to +the same Redis target. Each source gets its own collector, its own credentials, and its +own set of Redis streams, so the sources stay independent. The sources can be of different +database types. + +{{< note >}}Multiple sources require RDI API v2. RDI API v1 supports only single-source +pipelines. See the +[RDI API migration guide]({{< relref "/integrate/redis-data-integration/reference/api-migration" >}}).{{< /note >}} + +## Name your sources + +Each source is an entry in the `sources` section of +[`config.yaml`]({{< relref "/integrate/redis-data-integration/data-pipelines/pipeline-config" >}}), +with the source name as key: + +```yaml +sources: + mysql: # this source is named 'mysql' + type: cdc +``` + +A source name must: + +- Start with a lowercase letter. +- Contain only lowercase letters, digits, and dashes. +- End with a letter or a digit. +- Be at most 22 characters long. + +The names `rdi` and `target` are reserved and cannot be used for sources. + +RDI derives the names of resources associated with the source from its name. See the +following section for additional details. + +Each source also accepts an optional `name` property, which is a free-text display name +of up to 100 characters. Unlike the source name, it is not used as an identifier. + +## What a source name determines + +The source name determines the prefix of the environment variables that hold the +source's credentials. RDI builds the prefix by upper-casing the name, replacing each +dash with an underscore, and appending `_DB`: + +| Source name | Environment variable prefix | Credential references in `config.yaml` | +| :-- | :-- | :-- | +| `mysql` | `MYSQL_DB` | `${MYSQL_DB_USERNAME}`, `${MYSQL_DB_PASSWORD}` | +| `orders-eu` | `ORDERS_EU_DB` | `${ORDERS_EU_DB_USERNAME}`, `${ORDERS_EU_DB_PASSWORD}` | + +Such references must be used in a source's `connection` section to reference credentials +saved as pipeline secrets. + +If you manage the RDI cluster yourself, you also see the source name in the names of the +resources RDI creates for it. For a source named `mysql`, the Kubernetes secret is +`mysql-db`, the collector deployment is `collector-mysql`, and the Redis streams are +`data:{rdi}:mysql..`. + +## Configure several sources + +Add one entry per source. The following example captures from a MySQL database and a +PostgreSQL database, each with its own credentials: + +```yaml +sources: + mysql: + type: cdc + connection: + type: mysql + host: + port: 3306 + user: ${MYSQL_DB_USERNAME} + password: ${MYSQL_DB_PASSWORD} + databases: + - inventory + tables: + inventory.customers: {} + inventory.orders: {} + postgresql: + type: cdc + connection: + type: postgresql + host: + port: 5432 + database: billing + user: ${POSTGRESQL_DB_USERNAME} + password: ${POSTGRESQL_DB_PASSWORD} + schemas: + - public + tables: + public.customers: {} +targets: + target: + connection: + type: redis + host: + port: + password: ${TARGET_DB_PASSWORD} +``` + +## Set secrets for each source + +Set a source's credentials with the source name in the `--db` option: + +```bash +redis-di set-secret USERNAME --db mysql +redis-di set-secret PASSWORD --db mysql +redis-di set-secret USERNAME --db postgresql +redis-di set-secret PASSWORD --db postgresql +``` + +The secret keys used as CLI arguments are the same for every source: `USERNAME`, `PASSWORD`, and, for +[Transport Layer Security (TLS)]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets" >}}) +connections, `CACERT`, `CERT`, `KEY`, and `KEY_PASSWORD`. Use `--db target` for the +target database: `redis-di set-secret PASSWORD --db target `. + +The secrets can then be referenced in the corresponding source configuration as `${MYSQL_DB_USERNAME}`, +`${MYSQL_DB_PASSWORD}`, `${POSTGRESQL_DB_USERNAME}`, `${POSTGRESQL_DB_PASSWORD}`, `${TARGET_DB_PASSWORD}`, +as in the previous example. + +See [Set secrets]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets" >}}) +for the full secret reference. + +## Route jobs to a source + +A [job]({{< relref "/integrate/redis-data-integration/data-pipelines/transform-examples" >}}) +selects the source it processes by setting `server_name` to the source name: + +```yaml +source: + server_name: mysql + db: inventory + table: customers +``` + +When a pipeline has more than one source, every job must set `server_name`. + +In a pipeline with a single source, `server_name` is optional. Omitting it means the +job does not filter by source. + +## Add or remove a source + +To add a source, set its secrets first, then add it to `config.yaml` and deploy. Adding a +source does not interrupt other sources that are already running. + +To remove a source, delete its entry from `config.yaml` and deploy. RDI removes the +source's collector. Its secrets, streams, and dead-letter queue entries remain and have +to be cleaned up manually. + +Note that renaming a source is not supported; renaming a source in `config.yaml` is equivalent +to removing the source and adding a new source with the new name. This implies in particular: + +- The source's secrets have to be created under the new name and `${...}` references in + its `connection` section updated. +- `server_name` has to be updated for every job that reads from the source. +- The source starts with a new + [initial snapshot]({{< relref "/integrate/redis-data-integration/architecture" >}}). + +## Start, stop, and reset a single source + +Pass `--source` to act on a single source instead of the whole pipeline: + +```bash +redis-di stop --source mysql +redis-di start --source mysql +redis-di reset --source mysql +``` + +Two behaviors are worth knowing: + +- A single source runs only while its pipeline runs. Starting one source does not start a + stopped pipeline. +- Stopping one source leaves the others running. Similarly, when one source fails, the other + sources keep capturing changes. + +A source of type `external` has no collector, so you cannot start or stop it. + +## Monitor each source + +Use [`redis-di describe`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-describe" >}}) +to see the state of every source at once. + +Its `Sources` section lists each source with its sync mode and whether it is connected, and +its `Components` section lists one collector per source. Errors are reported against the +component they came from. See the reference page for the command for more details. + +Each source's collector has its own metric collection, named after the collector, such as +`collector-mysql_metrics`. In Prometheus, the stream processor's `rdi_incoming_entries` and +`rdi_stream_event_latency_ms` metrics carry a `data_source` label that includes the source +name, so you can break both of them down per source. See +[Stream processor metrics]({{< relref "/integrate/redis-data-integration/observability#stream-processor-metrics" >}}) +and, for the per-source collector endpoints, +[Accessing the metrics]({{< relref "/integrate/redis-data-integration/observability#accessing-the-metrics" >}}). + +Dead-letter queue tables are reported as `..
`, so rejected records +are attributed to their source. See +[Rejected records]({{< relref "/integrate/redis-data-integration/data-pipelines/rejected-records" >}}). + +## Existing sources are unaffected by the changes + +Before RDI supported several sources, every source-scoped resource was named using `source` +in place of the actual source name in `config.yaml`. Pipeline sources that existed +before upgrading to RDI 2.0.0 keep using those names to ensure continuity: + +- The `source-db` and `source-db-ssl` secrets, and the `SOURCE_DB_*` environment + variables, so its `connection` keeps referencing `${SOURCE_DB_USERNAME}` and + `${SOURCE_DB_PASSWORD}` regardless of its name. +- The `collector-source` collector. +- Its existing streams, `data:{rdi}:.
`, and its captured position. +- `rdi` as its `server_name`. + +See [Upgrading RDI]({{< relref "/integrate/redis-data-integration/installation/upgrade" >}}). From 747a6352efb63a93abbc6ce2afba39fdd0e7f8c9 Mon Sep 17 00:00:00 2001 From: Nicolaus Weidner Date: Fri, 21 Aug 2026 19:38:31 +0200 Subject: [PATCH 3/8] RDSC-5610: Update deploy page --- .../data-pipelines/deploy.md | 166 ++++++++++-------- 1 file changed, 97 insertions(+), 69 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/deploy.md b/content/integrate/redis-data-integration/data-pipelines/deploy.md index d126291b45..4cfd2d17e3 100644 --- a/content/integrate/redis-data-integration/data-pipelines/deploy.md +++ b/content/integrate/redis-data-integration/data-pipelines/deploy.md @@ -21,33 +21,38 @@ The sections below explain how to deploy a pipeline after you have created the r ## Set secrets -Before you deploy your pipeline, you must set the authentication secrets for the -source and target databases. Each secret has a name that you pass to the +Before you deploy your pipeline, you must set the authentication secrets for the source +and target databases. Every secret belongs to one database: a source, identified by its +name in `config.yaml`, or the target. You name that database with the `--db` option of the [`redis-di set-secret`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-set-secret" >}}) -command to set the secret value. -You can then refer to these secrets in the `config.yaml` file using the syntax "`${SECRET_NAME}`" -(the sample -[config.yaml file]({{< relref "/integrate/redis-data-integration/data-pipelines/pipeline-config#example" >}}) -shows these secrets in use). +command. -The table below lists all valid secret names. Note that the -username and password are required for the source and target, but the other -secrets are only relevant for TLS/mTLS connections. +The table below lists the available secret keys. The username and password are required, +while the other keys are only relevant for TLS/mTLS connections. -| Secret name | Description | +| Secret key | Description | | :-- | :-- | -| `SOURCE_DB_USERNAME` | Username for the source database | -| `SOURCE_DB_PASSWORD` | Password for the source database | -| `SOURCE_DB_CACERT` | (For TLS only) Source database CA certificate | -| `SOURCE_DB_CERT` | (For mTLS only) Source database client certificate | -| `SOURCE_DB_KEY` | (For mTLS only) Source database private key | -| `SOURCE_DB_KEY_PASSWORD` | (For mTLS only) Source database private key password | -| `TARGET_DB_USERNAME` | Username for the target database | -| `TARGET_DB_PASSWORD` | Password for the target database | -| `TARGET_DB_CACERT` | (For TLS only) Target database CA certificate | -| `TARGET_DB_CERT` | (For mTLS only) Target database client certificate | -| `TARGET_DB_KEY` | (For mTLS only) Target database private key | -| `TARGET_DB_KEY_PASSWORD` | (For mTLS only) Target database private key password | +| `USERNAME` | Username for the database | +| `PASSWORD` | Password for the database | +| `CACERT` | (For TLS only) CA certificate | +| `CERT` | (For mTLS only) Client certificate | +| `KEY` | (For mTLS only) Private key | +| `KEY_PASSWORD` | (For mTLS only) Private key password | + +A secret can be referenced in `config.yaml` as an environment variable that is derived from +the secret key and the database: The variable name is the database name in uppercase, with +each dash replaced by an underscore, followed by `_DB_` and the key. So setting `PASSWORD` +with `--db mysql` produces `MYSQL_DB_PASSWORD`, which the source references as +`${MYSQL_DB_PASSWORD}`, and `--db target` produces `TARGET_DB_PASSWORD`. The sample +[config.yaml file]({{< relref "/integrate/redis-data-integration/data-pipelines/pipeline-config#example" >}}) +shows these references in use, and +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) +covers the source naming rules. + +{{< note >}}A pipeline with a single source also accepts the older `SOURCE_DB_*` and +`TARGET_DB_*` keys, used without `--db`. RDI stores a `SOURCE_DB_*` key against the +pipeline's own source, so the `config.yaml` reference is still the one derived from the +source name. A pipeline with more than one source rejects these keys.{{< /note >}} {{< note >}} {{< embed-md "rdi-tls-secrets.md" >}} @@ -58,38 +63,39 @@ secrets are only relevant for TLS/mTLS connections. Use [`redis-di set-secret`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-set-secret" >}}) to set secrets for any installation type (VM, Kubernetes, or Redis Cloud). -The specific command lines for source secrets are as follows: +The command lines for a source named `mysql` are as follows. Repeat them with each +source's own name for a pipeline that has several sources: ```bash # For username and password -redis-di set-secret SOURCE_DB_USERNAME yourUsername -redis-di set-secret SOURCE_DB_PASSWORD yourPassword +redis-di set-secret USERNAME --db mysql yourUsername +redis-di set-secret PASSWORD --db mysql yourPassword # With source TLS, in addition to the above -redis-di set-secret SOURCE_DB_CACERT /path/to/myca.crt +redis-di set-secret CACERT --db mysql /path/to/myca.crt # With source mTLS, in addition to the above -redis-di set-secret SOURCE_DB_CERT /path/to/myclient.crt -redis-di set-secret SOURCE_DB_KEY /path/to/myclient.key -# Use this only if SOURCE_DB_KEY is password-protected -redis-di set-secret SOURCE_DB_KEY_PASSWORD yourKeyPassword +redis-di set-secret CERT --db mysql /path/to/myclient.crt +redis-di set-secret KEY --db mysql /path/to/myclient.key +# Use this only if the private key is password-protected +redis-di set-secret KEY_PASSWORD --db mysql yourKeyPassword ``` The corresponding command lines for target secrets are: ```bash # For username and password -redis-di set-secret TARGET_DB_USERNAME yourUsername -redis-di set-secret TARGET_DB_PASSWORD yourPassword +redis-di set-secret USERNAME --db target yourUsername +redis-di set-secret PASSWORD --db target yourPassword # With target TLS, in addition to the above -redis-di set-secret TARGET_DB_CACERT /path/to/myca.crt +redis-di set-secret CACERT --db target /path/to/myca.crt # With target mTLS, in addition to the above -redis-di set-secret TARGET_DB_CERT /path/to/myclient.crt -redis-di set-secret TARGET_DB_KEY /path/to/myclient.key -# Use this only if TARGET_DB_KEY is password-protected -redis-di set-secret TARGET_DB_KEY_PASSWORD yourKeyPassword +redis-di set-secret CERT --db target /path/to/myclient.crt +redis-di set-secret KEY --db target /path/to/myclient.key +# Use this only if the private key is password-protected +redis-di set-secret KEY_PASSWORD --db target yourKeyPassword ``` By default, `set-secret` waits for the pipeline to apply the change before returning. When you set @@ -103,14 +109,17 @@ never returns secret values, these commands show only the secret keys and whethe the stored values. ```bash -# List all the secrets of a pipeline and whether each one is set +# List all the secrets of a pipeline, with the database each one belongs to redis-di list-secrets +# List only the secrets of one database +redis-di list-secrets --db mysql + # Show a single secret and whether it is set -redis-di describe-secret SOURCE_DB_PASSWORD +redis-di describe-secret PASSWORD --db mysql # Delete a secret (prompts for confirmation unless you add --force) -redis-di delete-secret SOURCE_DB_CACERT +redis-di delete-secret CACERT --db mysql ``` See the reference pages for @@ -135,9 +144,12 @@ kubectl create secret generic \ --from-literal== ``` -Where `` is either `source-db` for source secrets or `target-db` for target secrets. +Where `` is `-db` for the secrets of a source, or `target-db` for target secrets. +The examples below use a source named `mysql`, so its secret is `mysql-db`. -If you use TLS or mTLS for either the source or target databases, you also need to create the `source-db-ssl` and/or `target-db-ssl` K8s secrets that contain the certificates used to establish secure connections. The general pattern of the commands is: +If you use TLS or mTLS for either the source or target databases, you also need to create the +`-db-ssl` and/or `target-db-ssl` K8s secrets that contain the certificates used +to establish secure connections. The general pattern of the commands is: ```bash kubectl create secret generic -ssl \ @@ -149,36 +161,36 @@ The specific command lines for source secrets are as follows: ```bash # Without source TLS -# Create or update source-db secret -kubectl create secret generic source-db --namespace=rdi \ ---from-literal=SOURCE_DB_USERNAME=yourUsername \ ---from-literal=SOURCE_DB_PASSWORD=yourPassword \ +# Create or update mysql-db secret +kubectl create secret generic mysql-db --namespace=rdi \ +--from-literal=MYSQL_DB_USERNAME=yourUsername \ +--from-literal=MYSQL_DB_PASSWORD=yourPassword \ --save-config --dry-run=client -o yaml | kubectl apply -f - # With source TLS -# Create of update source-db secret -kubectl create secret generic source-db --namespace=rdi \ ---from-literal=SOURCE_DB_USERNAME=yourUsername \ ---from-literal=SOURCE_DB_PASSWORD=yourPassword \ ---from-literal=SOURCE_DB_CACERT=/etc/certificates/source_db/ca.crt \ +# Create of update mysql-db secret +kubectl create secret generic mysql-db --namespace=rdi \ +--from-literal=MYSQL_DB_USERNAME=yourUsername \ +--from-literal=MYSQL_DB_PASSWORD=yourPassword \ +--from-literal=MYSQL_DB_CACERT=/etc/certificates/mysql_db/ca.crt \ --save-config --dry-run=client -o yaml | kubectl apply -f - -# Create or update source-db-ssl secret -kubectl create secret generic source-db-ssl --namespace=rdi \ +# Create or update mysql-db-ssl secret +kubectl create secret generic mysql-db-ssl --namespace=rdi \ --from-file=ca.crt=/path/to/myca.crt \ --save-config --dry-run=client -o yaml | kubectl apply -f - # With source mTLS -# Create or update source-db secret -kubectl create secret generic source-db --namespace=rdi \ ---from-literal=SOURCE_DB_USERNAME=yourUsername \ ---from-literal=SOURCE_DB_PASSWORD=yourPassword \ ---from-literal=SOURCE_DB_CACERT=/etc/certificates/source_db/ca.crt \ ---from-literal=SOURCE_DB_CERT=/etc/certificates/source_db/client.crt \ ---from-literal=SOURCE_DB_KEY=/etc/certificates/source_db/client.key \ ---from-literal=SOURCE_DB_KEY_PASSWORD=yourKeyPassword \ # add this only if SOURCE_DB_KEY is password-protected +# Create or update mysql-db secret +kubectl create secret generic mysql-db --namespace=rdi \ +--from-literal=MYSQL_DB_USERNAME=yourUsername \ +--from-literal=MYSQL_DB_PASSWORD=yourPassword \ +--from-literal=MYSQL_DB_CACERT=/etc/certificates/mysql_db/ca.crt \ +--from-literal=MYSQL_DB_CERT=/etc/certificates/mysql_db/client.crt \ +--from-literal=MYSQL_DB_KEY=/etc/certificates/mysql_db/client.key \ +--from-literal=MYSQL_DB_KEY_PASSWORD=yourKeyPassword \ # add this only if the private key is password-protected --save-config --dry-run=client -o yaml | kubectl apply -f - -# Create or update source-db-ssl secret -kubectl create secret generic source-db-ssl --namespace=rdi \ +# Create or update mysql-db-ssl secret +kubectl create secret generic mysql-db-ssl --namespace=rdi \ --from-file=ca.crt=/path/to/myca.crt \ --from-file=client.crt=/path/to/myclient.crt \ --from-file=client.key=/path/to/myclient.key \ @@ -225,7 +237,7 @@ kubectl create secret generic target-db-ssl --namespace=rdi \ --save-config --dry-run=client -o yaml | kubectl apply -f - ``` -Note that the certificate paths contained in the secrets `SOURCE_DB_CACERT`, `SOURCE_DB_CERT`, and `SOURCE_DB_KEY` (for the source database) and `TARGET_DB_CACERT`, `TARGET_DB_CERT`, and `TARGET_DB_KEY` (for the target database) are internal to RDI, so you *must* use the values shown in the example above. You should only change the certificate paths when you create the `source-db-ssl` and `target-db-ssl` secrets. +Note that the certificate paths contained in the `CACERT`, `CERT`, and `KEY` secrets are internal to RDI, so you *must* use the values shown in the example above. Each source has its own certificate directory, named after the source, so a source named `mysql` uses `/etc/certificates/mysql_db/`. You should only change the certificate paths when you create the `-db-ssl` and `target-db-ssl` secrets. Secrets that you create directly with `kubectl` must also be labeled so that the RDI operator discovers them as pipeline secrets. Each secret needs the following labels, where the @@ -240,7 +252,7 @@ discovers them as pipeline secrets. Each secret needs the following labels, wher Apply the labels to each secret with [`kubectl label`](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_label/): ```bash -kubectl label secret source-db --namespace=rdi --overwrite \ +kubectl label secret mysql-db --namespace=rdi --overwrite \ app.kubernetes.io/name=pipeline \ app.kubernetes.io/instance=default \ product=rdi @@ -250,7 +262,7 @@ kubectl label secret target-db --namespace=rdi --overwrite \ product=rdi # With source TLS or mTLS -kubectl label secret source-db-ssl --namespace=rdi --overwrite \ +kubectl label secret mysql-db-ssl --namespace=rdi --overwrite \ app.kubernetes.io/name=pipeline \ app.kubernetes.io/instance=default \ product=rdi @@ -331,6 +343,16 @@ redis-di stop redis-di start ``` +To act on a single source instead of the whole pipeline, add `--source`: + +```bash +redis-di stop --source mysql +redis-di start --source mysql +``` + +Note that a source can only run if its parent pipeline is running. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}). + ## Reset a pipeline Use [`redis-di reset`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-reset" >}}) @@ -343,6 +365,12 @@ drifted out of sync. redis-di reset ``` +Add `--source` to reset a single source and leave the others untouched: + +```bash +redis-di reset --source mysql +``` + ## Undeploy a pipeline To remove a pipeline, use the @@ -377,8 +405,8 @@ To avoid this, set all the related secrets, or at least all of them except the l become healthy: ```bash -redis-di set-secret SOURCE_DB_USERNAME newUsername --wait=false -redis-di set-secret SOURCE_DB_PASSWORD newPassword +redis-di set-secret USERNAME --db mysql newUsername --wait=false +redis-di set-secret PASSWORD --db mysql newPassword ``` The same applies to any set of changes that are only valid together. \ No newline at end of file From b4a76da5d04e41c1f7ab9525eb10a3d101869673 Mon Sep 17 00:00:00 2001 From: Nicolaus Weidner Date: Fri, 21 Aug 2026 19:40:11 +0200 Subject: [PATCH 4/8] RDSC-5610: Update server_name description --- .../data-pipelines/transform-examples/_index.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/_index.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/_index.md index 3b0b2cfa35..7053d21c06 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/_index.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/_index.md @@ -58,7 +58,11 @@ The main sections of these files are: - `source`: This is a mandatory section that specifies the data items that you want to use. You can add the following properties here: - - `server_name`: Logical server name (optional). + - `server_name`: The name of the source whose records this job processes, as it appears + in the `sources` section of + [config.yaml]({{< relref "/integrate/redis-data-integration/data-pipelines/pipeline-config" >}}). + Required when the pipeline has more than one source. Optional in a pipeline with a single source. See + [Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}). - `db`: Database name (optional). This refers to a database name you supplied in [config.yaml]({{< relref "/integrate/redis-data-integration/data-pipelines/pipeline-config" >}}). - `schema`: Database schema (optional). This refers to a schema name you supplied in @@ -126,7 +130,7 @@ the default logic. ```yaml name: Rename field example source: - server_name: redislabs + server_name: sqlserver schema: dbo table: emp transform: From 1b259199987a316755ed0e4706357732c3d2e320 Mon Sep 17 00:00:00 2001 From: Nicolaus Weidner Date: Fri, 21 Aug 2026 19:51:19 +0200 Subject: [PATCH 5/8] RDSC-5610: More changes for multi-source --- content/embeds/rdi-tls-secrets.md | 2 +- content/embeds/rdi-when-to-use.md | 2 +- .../data-pipelines/rejected-records.md | 21 +++++++++---------- .../integrate/redis-data-integration/faq.md | 6 ++++++ .../installation/ha-test.md | 6 ++++-- .../installation/install-k8s.md | 10 +++++---- .../installation/install-vm.md | 14 +++++++------ .../installation/upgrade.md | 12 +++++++++-- .../redis-data-integration/observability.md | 6 ++++-- .../redis-data-integration/troubleshooting.md | 2 +- 10 files changed, 51 insertions(+), 30 deletions(-) diff --git a/content/embeds/rdi-tls-secrets.md b/content/embeds/rdi-tls-secrets.md index 147e76b516..a39b7f113e 100644 --- a/content/embeds/rdi-tls-secrets.md +++ b/content/embeds/rdi-tls-secrets.md @@ -1,4 +1,4 @@ -When creating secrets for TLS or mTLS, ensure that all certificates and keys are in `PEM` format. The only exception to this is that for PostgreSQL, the private key `SOURCE_DB_KEY` secret must be in `DER` format. If you have a key in `PEM` format, you must convert it to `DER` before creating the `SOURCE_DB_KEY` secret using the command: +When creating secrets for TLS or mTLS, ensure that all certificates and keys are in `PEM` format. The only exception to this is that for PostgreSQL, the source's private key secret must be in `DER` format. If you have a key in `PEM` format, you must convert it to `DER` before creating the secret using the command: ```bash openssl pkcs8 -topk8 -inform PEM -outform DER \ diff --git a/content/embeds/rdi-when-to-use.md b/content/embeds/rdi-when-to-use.md index 3251321552..7ef4734d35 100644 --- a/content/embeds/rdi-when-to-use.md +++ b/content/embeds/rdi-when-to-use.md @@ -3,7 +3,7 @@ RDI is a good fit when: - You want your app/micro-services to read from Redis to scale reads at speed. -- You want to transfer data to Redis from a *single* source database. +- You want to transfer data to Redis from one or more source databases. - You must use a slow database as the system of record for the app. - The app must always *write* its data to the slow database. - Your app can tolerate *eventual* consistency of data in the Redis cache. diff --git a/content/integrate/redis-data-integration/data-pipelines/rejected-records.md b/content/integrate/redis-data-integration/data-pipelines/rejected-records.md index 73a8c50b92..be992a1007 100644 --- a/content/integrate/redis-data-integration/data-pipelines/rejected-records.md +++ b/content/integrate/redis-data-integration/data-pipelines/rejected-records.md @@ -41,25 +41,24 @@ RDI stores rejected records in the RDI database as capped Redis streams. Each DL stream corresponds to a source table and tracks the records rejected for that table. -DLQ stream names use the `dlq:` prefix followed by the source data stream name. -In current RDI versions, the stream name is typically: +DLQ stream names use the `dlq:` prefix followed by the source data stream name, which +includes the name of the +[source]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) +the records came from: ```text -dlq:data:{rdi}:.
+dlq:data:{rdi}:..
``` -For example, rejected records for the `public.users` table are stored in: +For example, rejected records for the `public.users` table of a source named `postgresql` +are stored in: ```text -dlq:data:{rdi}:public.users +dlq:data:{rdi}:postgresql.public.users ``` -For sources that include the source name in the stream qualifier, the final part -can contain three components: - -```text -dlq:data:{rdi}:..
-``` +The dead-letter queue endpoints and the CLI report table names in that same qualified form, +so a rejected record is always attributed to the source it came from. Some RDI versions or configurations can use a hash-tagged variant such as `dlq:{data:rdi:.
}`. To find all DLQ streams in the diff --git a/content/integrate/redis-data-integration/faq.md b/content/integrate/redis-data-integration/faq.md index a9d30b57b8..995f85658d 100644 --- a/content/integrate/redis-data-integration/faq.md +++ b/content/integrate/redis-data-integration/faq.md @@ -76,6 +76,12 @@ deployment, use separate Kubernetes clusters. See [Install on Kubernetes]({{< relref "/integrate/redis-data-integration/installation/install-k8s" >}}) for installation details. +## Can one pipeline capture from several source databases? + +Yes. Add one entry per source to the `sources` section of `config.yaml`. Each source +independently captures change records from its source database. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}). + ## Can RDI automatically track changes to the source database schema? If you don't configure RDI to capture a specific set of tables in the schema then it will diff --git a/content/integrate/redis-data-integration/installation/ha-test.md b/content/integrate/redis-data-integration/installation/ha-test.md index 7f89c08514..35ace592d6 100644 --- a/content/integrate/redis-data-integration/installation/ha-test.md +++ b/content/integrate/redis-data-integration/installation/ha-test.md @@ -24,7 +24,7 @@ kubectl -n rdi get pods # Example output: NAME READY STATUS RESTARTS AGE collector-api-577d95bfd8-5wbg6 1/1 Running 0 12m -collector-source-95f45bcf7-vwn5l 1/1 Running 0 12m +collector-mysql-95f45bcf7-vwn5l 1/1 Running 0 12m fluentd-zq2lc 1/1 Running 0 72m logrotate-29530445-j729x 0/1 Completed 0 14m logrotate-29530450-dprr2 0/1 Completed 0 9m40s @@ -36,7 +36,9 @@ rdi-operator-7f7f6c7dfd-5qmjd 1/1 Running 0 71m rdi-reloader-77df5f7854-lwmvz 1/1 Running 0 71m ``` -2. Identify the leader node - this is the one that has a running `collector-source` pod. +2. Identify the leader node - this is the one that has a running collector pod. A + collector is named after its source, so the example above shows `collector-mysql` for a + source named `mysql`. ## Performing the HA Failover Testing diff --git a/content/integrate/redis-data-integration/installation/install-k8s.md b/content/integrate/redis-data-integration/installation/install-k8s.md index 55b0ebfa6f..10cfdfcb2e 100644 --- a/content/integrate/redis-data-integration/installation/install-k8s.md +++ b/content/integrate/redis-data-integration/installation/install-k8s.md @@ -423,15 +423,17 @@ Specifically, ensure that one or both of the following Helm chart values is set: - `controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz` - `controller.service.externalTrafficPolicy=Local` -## Prepare your source database +## Prepare your source databases -Before deploying a pipeline, you must configure your source database to enable CDC. See the +Before deploying a pipeline, you must configure each source database to enable CDC. See the [Prepare source databases]({{< relref "/integrate/redis-data-integration/data-pipelines/prepare-dbs" >}}) -section to learn how to do this. +section to learn how to do this. A pipeline can capture from more than one source database, +and each one has to be prepared. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}). ## Deploy a pipeline -When the Helm installation is complete and you have prepared the source database for CDC, +When the Helm installation is complete and you have prepared your source databases for CDC, you are ready to start using RDI. See the guides on how to [configure]({{< relref "/integrate/redis-data-integration/data-pipelines" >}}) and [deploy]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy" >}}) diff --git a/content/integrate/redis-data-integration/installation/install-vm.md b/content/integrate/redis-data-integration/installation/install-vm.md index 63700922fa..6409bbf772 100644 --- a/content/integrate/redis-data-integration/installation/install-vm.md +++ b/content/integrate/redis-data-integration/installation/install-vm.md @@ -17,7 +17,7 @@ weight: 10 --- This guide explains how to install Redis Data Integration (RDI) on one or more VMs and integrate it with -your source database. You can also +your source databases. You can also [Install RDI on Kubernetes]({{< relref "/integrate/redis-data-integration/installation/install-k8s" >}}). {{< note >}}We recommend you always use the latest version, which is RDI v{{< rdi-version >}}. @@ -271,15 +271,17 @@ and the RDI pipeline will be active on that VM. You may find it useful to trigger a failover deliberately to check that RDI is correctly configured to handle it. See [Test HA failover]({{< relref "/integrate/redis-data-integration/installation/ha-test" >}}) to learn how to do this. -## Prepare your source database +## Prepare your source databases -Before deploying a pipeline, you must configure your source database to enable CDC. See the +Before deploying a pipeline, you must configure each source database to enable CDC. See the [Prepare source databases]({{< relref "/integrate/redis-data-integration/data-pipelines/prepare-dbs" >}}) -section to learn how to do this. +section to learn how to do this. A pipeline can capture from more than one source database, +and each one has to be prepared. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}). ## Deploy a pipeline -When the installation is complete, and you have prepared the source database for CDC, +When the installation is complete, and you have prepared your source databases for CDC, you are ready to start using RDI. See the guides on how to [configure]({{< relref "/integrate/redis-data-integration/data-pipelines" >}}) and [deploy]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy" >}}) @@ -331,4 +333,4 @@ This will uninstall RDI and its dependencies, are you sure? [y, N] ``` If you type anything other than "y" here, the script will abort without making any changes -to RDI or your source database. +to RDI or your source databases. diff --git a/content/integrate/redis-data-integration/installation/upgrade.md b/content/integrate/redis-data-integration/installation/upgrade.md index da873958c0..b4d9cd8a36 100644 --- a/content/integrate/redis-data-integration/installation/upgrade.md +++ b/content/integrate/redis-data-integration/installation/upgrade.md @@ -74,7 +74,7 @@ run the CLI binary of the previous RDI version, which still provided the `redis- sudo redis-di upgrade --rdi-host --rdi-port ``` -{{< note >}}If the `collector-source` or the `processor` pods are not in the `Running` state after +{{< note >}}If the collector or the `processor` pods are not in the `Running` state after the upgrade, you must run `redis-di deploy` and check again that they are both in the `Running` state. {{< /note >}} @@ -174,6 +174,14 @@ described in [Uninstall RDI]({{< relref "/integrate/redis-data-integration/insta and then install the old version. {{< /note >}} +## Source names after the upgrade + +A source that existed before upgrading to RDI 2.0.0 keeps using the resource names it +was using before the upgrade, so it will keep running without changes. + +For more details on how to handle sources created after upgrading, see +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources#upgraded-pipelines-keep-their-existing-names" >}}). + ## Enabling the Flink processor The @@ -203,7 +211,7 @@ The upgrade process replaces the current RDI components with their new versions: - Firstly, the control plane components are replaced. At this point, the pipeline is still active but monitoring will be disconnected. - Secondly, the pipeline data plane components are replaced. - If a pipeline is active while upgrading, the `collector-source` and `processor` + If a pipeline is active while upgrading, the collector and `processor` pods will be restarted. The pipeline will pause for up to two minutes but it will catch up very quickly after restarting. The pipeline data and state are both stored in Redis, so data will not diff --git a/content/integrate/redis-data-integration/observability.md b/content/integrate/redis-data-integration/observability.md index eabcb2ab0d..07bbb883f5 100644 --- a/content/integrate/redis-data-integration/observability.md +++ b/content/integrate/redis-data-integration/observability.md @@ -51,7 +51,9 @@ The way you access the metrics endpoints depends on whether you are using a VM i ### VM Installation For VM installations, the metrics are available by default on the following endpoints: -- Collector metrics: `https:///collector-source/metrics` +- Collector metrics: `https:///collector-/metrics`, where `` is + the source name from `config.yaml`. Each source has its own collector, so a pipeline with + several sources has one endpoint per source. - Stream processor metrics: `https:///processor/metrics` - Operator metrics: `https:///operator/metrics` @@ -223,7 +225,7 @@ RDI reports with their descriptions. - Where the metric name has the `rdi_` prefix, this will be replaced by the Kubernetes namespace name if you supplied a custom name during installation. The prefix is always `rdi_` for VM installations. - Metrics with the `_created` suffix are automatically generated by Prometheus for counters and gauges to track when they were first created. -- The `rdi_incoming_entries` metric provides a detailed breakdown for each data source by operation type. +- The `rdi_incoming_entries` metric provides a detailed breakdown for each data source by operation type. The `data_source` label is the stream's table name, qualified with the source name for each source that captures under its own name, so you can break the metric down per source. - The `rdi_stream_event_latency_ms` metric helps monitor data freshness and processing delays. - The processor performance metrics are divided into two categories: - **Total metrics**: Accumulate values across all processed batches for historical analysis diff --git a/content/integrate/redis-data-integration/troubleshooting.md b/content/integrate/redis-data-integration/troubleshooting.md index 306d93f153..b393e1535b 100644 --- a/content/integrate/redis-data-integration/troubleshooting.md +++ b/content/integrate/redis-data-integration/troubleshooting.md @@ -42,7 +42,7 @@ By default, RDI records the following logs in the host VM file system at | :-- | :-- | | `rdi_collector-collector-initializer.log` | Initializing the collector. | | `rdi_collector-debezium-ssl-init.log` | Establishing the connector SSL connections to the source and RDI database (if you are using SSL). | -| `rdi_collector-collector-source.log` | Collector [change data capture (CDC)]({{< relref "/integrate/redis-data-integration/architecture" >}}) operations. | +| `rdi_collector-collector-.log` | Collector [change data capture (CDC)]({{< relref "/integrate/redis-data-integration/architecture" >}}) operations. There is one log per source, named after the source. | | `rdi_rdi-rdi-operator.log` | Main [RDI control plane]({{< relref "/integrate/redis-data-integration/architecture#how-rdi-is-deployed" >}}) component. | | `rdi_processor-processor.log` | RDI stream processing. | From a79d1f6bc8adbb9235a6367745263c5fef7da213 Mon Sep 17 00:00:00 2001 From: Nicolaus Weidner Date: Fri, 21 Aug 2026 20:05:14 +0200 Subject: [PATCH 6/8] RDSC-5610: Update CLI docs for multi-source --- .../reference/cli/redis-di-delete-secret.md | 3 +- .../reference/cli/redis-di-describe-secret.md | 3 +- .../reference/cli/redis-di-describe.md | 47 +++++++++++-------- .../reference/cli/redis-di-get-dlq.md | 6 ++- .../reference/cli/redis-di-get-secret.md | 3 +- .../cli/redis-di-list-dlq-records.md | 8 +++- .../reference/cli/redis-di-list-secrets.md | 2 + .../reference/cli/redis-di-reset.md | 4 ++ .../reference/cli/redis-di-set-secret.md | 17 ++++--- .../reference/cli/redis-di-start.md | 4 ++ .../reference/cli/redis-di-stop.md | 4 ++ 11 files changed, 70 insertions(+), 31 deletions(-) diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-delete-secret.md b/content/integrate/redis-data-integration/reference/cli/redis-di-delete-secret.md index d55224832c..e82e81f785 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-delete-secret.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-delete-secret.md @@ -22,6 +22,7 @@ redis-di delete-secret [flags] | Option | Description | | :----------------- | :-------------------------------------------------------------------------------- | | `-p`, `--pipeline` | Pipeline to target (default `default`). | +| `--db` | Database the secret belongs to: a source name, or `target`. | | `--force` | Skip the confirmation prompt. | | `--wait` | Wait for the pipeline to reach the expected state (default `true`). | | `--timeout` | Maximum time to wait for the pipeline to reach the expected state (default `2m`). | @@ -32,5 +33,5 @@ This command also accepts the ## Example ```bash -redis-di delete-secret SOURCE_DB_CACERT --force +redis-di delete-secret CACERT --db mysql --force ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-describe-secret.md b/content/integrate/redis-data-integration/reference/cli/redis-di-describe-secret.md index 3ca2e2c7a2..194376cabd 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-describe-secret.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-describe-secret.md @@ -22,6 +22,7 @@ redis-di describe-secret [flags] | Option | Description | | :----------------- | :-------------------------------------- | | `-p`, `--pipeline` | Pipeline to target (default `default`). | +| `--db` | Database the secret belongs to: a source name, or `target`. | This command also accepts the [global options]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di#global-options" >}}). @@ -29,5 +30,5 @@ This command also accepts the ## Example ```bash -redis-di describe-secret TARGET_DB_PASSWORD +redis-di describe-secret PASSWORD --db target ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-describe.md b/content/integrate/redis-data-integration/reference/cli/redis-di-describe.md index 0ff5a63243..cda8b4b0cc 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-describe.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-describe.md @@ -42,7 +42,8 @@ To watch the status update live, pair the command with `watch`: watch -n 1 redis-di describe ``` -The output has a section for each part of the pipeline, for example: +The output has a section for each part of the pipeline. The example below shows a pipeline with +two sources, `mysql` and `postgresql`: ``` Name: default @@ -51,30 +52,38 @@ Status: started Current: yes Sources: - Name Type Db Type Connection Sync Mode Connected - ---- ---- ------- ---------- --------- --------- - mysql cdc mysql ${HOST_IP}:13000 streaming yes + Name Type Db Type Connection Sync Mode Connected + ---- ---- ------- ---------- --------- --------- + mysql cdc mysql :3306 streaming yes + postgresql cdc postgresql :5432 streaming yes Targets: - Name Db Type Connection Connected - ---- ------- ---------- --------- - target redis ${HOST_IP}:12000 yes + Name Db Type Connection Connected + ---- ------- ---------- --------- + target redis :12000 yes + +Processor: + Type: classic Jobs: - Name Source Transformations Outputs Connections - ---- ------ --------------- ------- ----------- - address_job inventory.addresses 1 1 target - customers_hash_job inventory.customers 0 1 target + Name Server Name Db / Schema Table Transformations Outputs Connections + ---- ----------- ----------- ----- --------------- ------- ----------- + billing_job postgresql public customers 1 1 target + customers_hash_job mysql inventory customers 0 1 target + orders_job mysql inventory orders 1 1 target Components: - Name Type Version Status - ---- ---- ------- ------ - collector-api collector-api 0.0.0 started - collector-source debezium-collector ... started - processor processor 0.0.0 started + Name Type Version Status + ---- ---- ------- ------ + collector-api collector-api started + collector-mysql debezium-collector started + collector-postgresql debezium-collector started + processor stream-processor started Statistics: - Name Total Pending Inserted Updated Deleted Filtered Rejected Deduplicated Last Arrival - ---- ----- ------- -------- ------- ------- -------- -------- ------------ ------------ - {rdi}:inventory.customers 4 0 4 0 0 0 0 0 2026-06-18T13:42:44Z + Name Total Pending Inserted Updated Deleted Filtered Rejected Deduplicated Last Arrival + ---- ----- ------- -------- ------- ------- -------- -------- ------------ ------------ + mysql.inventory.customers 4 0 4 0 0 0 0 0 2026-06-18T13:42:44Z + mysql.inventory.orders 12 0 12 0 0 0 0 0 2026-06-18T13:42:51Z + postgresql.public.customers 7 0 7 0 0 0 0 0 2026-06-18T13:42:49Z ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-get-dlq.md b/content/integrate/redis-data-integration/reference/cli/redis-di-get-dlq.md index 994ac42c9d..f782d2b593 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-get-dlq.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-get-dlq.md @@ -11,6 +11,10 @@ aliases: Gets a single dead-letter queue (DLQ) of a pipeline and prints it in the compact `list-dlqs` table format. +The name is the DLQ's source-qualified table name, `..
`, as +[`list-dlqs`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-list-dlqs" >}}) +reports it. + ## Usage ``` @@ -30,5 +34,5 @@ This command also accepts the ## Example ```bash -redis-di get-dlq inventory.customers +redis-di get-dlq mysql.inventory.customers ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-get-secret.md b/content/integrate/redis-data-integration/reference/cli/redis-di-get-secret.md index 0b94f16073..3b5fd8c75e 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-get-secret.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-get-secret.md @@ -23,6 +23,7 @@ redis-di get-secret [flags] | Option | Description | | :----------------- | :--------------------------------------------------- | | `-p`, `--pipeline` | Pipeline to target (default `default`). | +| `--db` | Database the secret belongs to: a source name, or `target`. | | `-o`, `--output` | Output format: `table` (default), `json`, or `yaml`. | This command also accepts the @@ -31,5 +32,5 @@ This command also accepts the ## Example ```bash -redis-di get-secret SOURCE_DB_USERNAME +redis-di get-secret USERNAME --db mysql ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-list-dlq-records.md b/content/integrate/redis-data-integration/reference/cli/redis-di-list-dlq-records.md index 49b1744364..23d1668be3 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-list-dlq-records.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-list-dlq-records.md @@ -16,6 +16,10 @@ and paging with `--limit`, `--offset`, and `--sort-order`. The operation code is Use [`list-dlqs`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-list-dlqs" >}}) to see all the pipeline's dead-letter queues and their record counts. +The DLQ name is its source-qualified table name, `..
`, as +[`list-dlqs`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-list-dlqs" >}}) +reports it. + ## Usage ``` @@ -48,8 +52,8 @@ This command also accepts the ```bash # Newest 20 rejected records of a queue -redis-di list-dlq-records inventory.customers +redis-di list-dlq-records mysql.inventory.customers # Oldest 100 records, as JSON -redis-di list-dlq-records inventory.customers --limit 100 --sort-order asc -o json +redis-di list-dlq-records mysql.inventory.customers --limit 100 --sort-order asc -o json ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-list-secrets.md b/content/integrate/redis-data-integration/reference/cli/redis-di-list-secrets.md index 14c64f63c5..00c2e5db80 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-list-secrets.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-list-secrets.md @@ -22,6 +22,7 @@ redis-di list-secrets [flags] | Option | Description | | :----------------- | :--------------------------------------------------- | | `-p`, `--pipeline` | Pipeline to target (default `default`). | +| `--db` | List only the secrets of this database: a source name, or `target`. | | `-o`, `--output` | Output format: `table` (default), `json`, or `yaml`. | This command also accepts the @@ -31,5 +32,6 @@ This command also accepts the ```bash redis-di list-secrets +redis-di list-secrets --db mysql redis-di list-secrets -p my-pipeline ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-reset.md b/content/integrate/redis-data-integration/reference/cli/redis-di-reset.md index 14d627c0a4..d8554a168f 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-reset.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-reset.md @@ -25,6 +25,7 @@ The pipeline name is an optional argument that defaults to `default`. | Option | Description | | :---------- | :-------------------------------------------------------------------------------- | +| `--source` | Target only this source instead of the whole pipeline. | | `--wait` | Wait for the pipeline to reach the expected state (default `true`). | | `--timeout` | Maximum time to wait for the pipeline to reach the expected state (default `2m`). | @@ -35,4 +36,7 @@ This command also accepts the ```bash redis-di reset + +# Reset only source mysql +redis-di reset --source mysql ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-set-secret.md b/content/integrate/redis-data-integration/reference/cli/redis-di-set-secret.md index f82e5978db..1bb2e01988 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-set-secret.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-set-secret.md @@ -10,10 +10,11 @@ aliases: --- Creates or updates a secret of a pipeline. Secrets hold the credentials and certificates that the -pipeline uses to connect to the source and target databases (see +pipeline uses to connect to its source and target databases (see [Set secrets]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets" >}}) -for the list of secret names). You can then refer to a secret in the `config.yaml` file with the -syntax `${SECRET_NAME}`. +for the secret keys). Every secret belongs to one database, which you name with `--db`: a source, or +`target`. RDI combines the key and the database into the environment variable that `config.yaml` +references, so `PASSWORD` with `--db mysql` becomes `${MYSQL_DB_PASSWORD}`. The secret value comes from the `[value]` argument, the `--file` option, or the `--literal` option. If you provide none of these on an interactive terminal, the command prompts for the value without @@ -30,6 +31,7 @@ redis-di set-secret [value] [flags] | Option | Description | | :----------------- | :-------------------------------------------------------------------------------- | | `-p`, `--pipeline` | Pipeline to target (default `default`). | +| `--db` | Database the secret belongs to: a source name, or `target`. | | `--file` | Read the secret value from the file at this path. | | `--literal` | Use this literal string as the secret value. | | `--wait` | Wait for the pipeline to reach the expected state (default `true`). | @@ -42,11 +44,14 @@ This command also accepts the ```bash # Value from an argument -redis-di set-secret SOURCE_DB_USERNAME myuser +redis-di set-secret USERNAME --db mysql myuser # Value from a file (for example, a certificate) -redis-di set-secret SOURCE_DB_CACERT --file /path/to/myca.crt +redis-di set-secret CACERT --db mysql --file /path/to/myca.crt # Value read from an interactive prompt -redis-di set-secret SOURCE_DB_PASSWORD +redis-di set-secret PASSWORD --db mysql + +# Target database credentials +redis-di set-secret PASSWORD --db target mypassword ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-start.md b/content/integrate/redis-data-integration/reference/cli/redis-di-start.md index 23a5b9b124..cae732fd08 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-start.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-start.md @@ -24,6 +24,7 @@ The pipeline name is an optional argument that defaults to `default`. | Option | Description | | :---------- | :-------------------------------------------------------------------------------- | +| `--source` | Target only this source instead of the whole pipeline. | | `--wait` | Wait for the pipeline to reach the expected state (default `true`). | | `--timeout` | Maximum time to wait for the pipeline to reach the expected state (default `2m`). | @@ -35,4 +36,7 @@ This command also accepts the ```bash redis-di start redis-di start my-pipeline --wait=false + +# Start only source mysql. Does not start the pipeline if it's currently stopped. +redis-di start --source mysql ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-stop.md b/content/integrate/redis-data-integration/reference/cli/redis-di-stop.md index 17b8dd5877..86bfdd8036 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-stop.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-stop.md @@ -24,6 +24,7 @@ The pipeline name is an optional argument that defaults to `default`. | Option | Description | | :---------- | :-------------------------------------------------------------------------------- | +| `--source` | Target only this source instead of the whole pipeline. | | `--wait` | Wait for the pipeline to reach the expected state (default `true`). | | `--timeout` | Maximum time to wait for the pipeline to reach the expected state (default `2m`). | @@ -34,4 +35,7 @@ This command also accepts the ```bash redis-di stop + +# Stop only source mysql +redis-di stop --source mysql ``` From d663c22adc79f6d2bc10f37530ad362cda26ca99 Mon Sep 17 00:00:00 2001 From: Nicolaus Weidner Date: Fri, 21 Aug 2026 21:57:15 +0200 Subject: [PATCH 7/8] RDSC-5610: More updates --- .../data-pipelines/deploy.md | 8 +- .../data-pipelines/multiple-sources.md | 85 +++++++++---------- .../data-pipelines/prepare-dbs/mongodb.md | 29 ++++--- .../data-pipelines/prepare-dbs/spanner.md | 14 ++- .../data-pipelines/rejected-records.md | 8 +- .../transform-examples/_index.md | 7 ++ .../redis-data-integration/observability.md | 2 +- content/operate/rc/rdi/define.md | 2 +- content/operate/rc/rdi/quick-start.md | 2 +- content/operate/rc/rdi/setup.md | 2 +- 10 files changed, 86 insertions(+), 73 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/deploy.md b/content/integrate/redis-data-integration/data-pipelines/deploy.md index 4cfd2d17e3..4b7a6e369e 100644 --- a/content/integrate/redis-data-integration/data-pipelines/deploy.md +++ b/content/integrate/redis-data-integration/data-pipelines/deploy.md @@ -49,10 +49,10 @@ shows these references in use, and [Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) covers the source naming rules. -{{< note >}}A pipeline with a single source also accepts the older `SOURCE_DB_*` and -`TARGET_DB_*` keys, used without `--db`. RDI stores a `SOURCE_DB_*` key against the -pipeline's own source, so the `config.yaml` reference is still the one derived from the -source name. A pipeline with more than one source rejects these keys.{{< /note >}} +{{< note >}}The older scope-prefixed keys like `SOURCE_DB_PASSWORD` or `TARGET_DB_PASSWORD` +are still accepted for pipelines that were created before the upgrade and that have a single +source. They are used without specifying `--db`. These keys are deprecated and will be +removed in a future release, so all usage should migrate to the new keys.{{< /note >}} {{< note >}} {{< embed-md "rdi-tls-secrets.md" >}} diff --git a/content/integrate/redis-data-integration/data-pipelines/multiple-sources.md b/content/integrate/redis-data-integration/data-pipelines/multiple-sources.md index d970e7f151..0c5082374d 100644 --- a/content/integrate/redis-data-integration/data-pipelines/multiple-sources.md +++ b/content/integrate/redis-data-integration/data-pipelines/multiple-sources.md @@ -15,10 +15,10 @@ type: integration weight: 4 --- -One RDI pipeline can capture changes from several source databases and write them all to -the same Redis target. Each source gets its own collector, its own credentials, and its -own set of Redis streams, so the sources stay independent. The sources can be of different -database types. +One RDI pipeline can capture changes from several source databases, which can be of +different database types, and write them all to the same Redis target. Each source has its +own collector, its own credentials, and its own set of Redis streams, so the sources stay +independent. {{< note >}}Multiple sources require RDI API v2. RDI API v1 supports only single-source pipelines. See the @@ -45,31 +45,19 @@ A source name must: The names `rdi` and `target` are reserved and cannot be used for sources. -RDI derives the names of resources associated with the source from its name. See the -following section for additional details. +The source name determines the environment variables that carry the source's credentials, so +a source named `mysql` references `${MYSQL_DB_USERNAME}` and `${MYSQL_DB_PASSWORD}` in its +`connection` section. See +[Set secrets]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets" >}}) +for how RDI derives those names and for the full list of secret keys. + +The source name also appears in the resources RDI creates for the source. For a source named +`mysql`, the collector deployment is `collector-mysql` and the Redis streams are +`data:{rdi}:mysql..
`. Each source also accepts an optional `name` property, which is a free-text display name of up to 100 characters. Unlike the source name, it is not used as an identifier. -## What a source name determines - -The source name determines the prefix of the environment variables that hold the -source's credentials. RDI builds the prefix by upper-casing the name, replacing each -dash with an underscore, and appending `_DB`: - -| Source name | Environment variable prefix | Credential references in `config.yaml` | -| :-- | :-- | :-- | -| `mysql` | `MYSQL_DB` | `${MYSQL_DB_USERNAME}`, `${MYSQL_DB_PASSWORD}` | -| `orders-eu` | `ORDERS_EU_DB` | `${ORDERS_EU_DB_USERNAME}`, `${ORDERS_EU_DB_PASSWORD}` | - -Such references must be used in a source's `connection` section to reference credentials -saved as pipeline secrets. - -If you manage the RDI cluster yourself, you also see the source name in the names of the -resources RDI creates for it. For a source named `mysql`, the Kubernetes secret is -`mysql-db`, the collector deployment is `collector-mysql`, and the Redis streams are -`data:{rdi}:mysql..
`. - ## Configure several sources Add one entry per source. The following example captures from a MySQL database and a @@ -128,9 +116,9 @@ The secret keys used as CLI arguments are the same for every source: `USERNAME`, connections, `CACERT`, `CERT`, `KEY`, and `KEY_PASSWORD`. Use `--db target` for the target database: `redis-di set-secret PASSWORD --db target `. -The secrets can then be referenced in the corresponding source configuration as `${MYSQL_DB_USERNAME}`, -`${MYSQL_DB_PASSWORD}`, `${POSTGRESQL_DB_USERNAME}`, `${POSTGRESQL_DB_PASSWORD}`, `${TARGET_DB_PASSWORD}`, -as in the previous example. +Each source's `connection` section then references its own secrets: `${MYSQL_DB_USERNAME}` and +`${MYSQL_DB_PASSWORD}` for `mysql`, `${POSTGRESQL_DB_USERNAME}` and `${POSTGRESQL_DB_PASSWORD}` +for `postgresql`, and `${TARGET_DB_PASSWORD}` for the target. See [Set secrets]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets" >}}) for the full secret reference. @@ -152,14 +140,19 @@ When a pipeline has more than one source, every job must set `server_name`. In a pipeline with a single source, `server_name` is optional. Omitting it means the job does not filter by source. +`server_name` also accepts a list of source names, and an entry prefixed with `regex:` is +matched as a regular expression, so one job can serve several sources. See +[Job files]({{< relref "/integrate/redis-data-integration/data-pipelines/transform-examples" >}}). + ## Add or remove a source To add a source, set its secrets first, then add it to `config.yaml` and deploy. Adding a source does not interrupt other sources that are already running. To remove a source, delete its entry from `config.yaml` and deploy. RDI removes the -source's collector. Its secrets, streams, and dead-letter queue entries remain and have -to be cleaned up manually. +source's collector. The source's secrets are not deleted, so remove them yourself with +[`redis-di delete-secret`]({{< relref "/integrate/redis-data-integration/reference/cli/redis-di-delete-secret" >}}) +if you no longer need them. Note that renaming a source is not supported; renaming a source in `config.yaml` is equivalent to removing the source and adding a new source with the new name. This implies in particular: @@ -180,14 +173,12 @@ redis-di start --source mysql redis-di reset --source mysql ``` -Two behaviors are worth knowing: +A source runs only while its pipeline runs, so starting one source does not start a stopped +pipeline. Stopping one source leaves the others running, and when one source fails, the other +sources keep capturing changes. -- A single source runs only while its pipeline runs. Starting one source does not start a - stopped pipeline. -- Stopping one source leaves the others running. Similarly, when one source fails, the other - sources keep capturing changes. - -A source of type `external` has no collector, so you cannot start or stop it. +A source of type `external` is the exception: RDI creates no collector for it, so you cannot +start or stop it. ## Monitor each source @@ -200,8 +191,9 @@ component they came from. See the reference page for the command for more detail Each source's collector has its own metric collection, named after the collector, such as `collector-mysql_metrics`. In Prometheus, the stream processor's `rdi_incoming_entries` and -`rdi_stream_event_latency_ms` metrics carry a `data_source` label that includes the source -name, so you can break both of them down per source. See +`rdi_stream_event_latency_ms` metrics carry a `data_source` label that identifies the stream +the value belongs to, including the source name, so you can break both of them down per +source. See [Stream processor metrics]({{< relref "/integrate/redis-data-integration/observability#stream-processor-metrics" >}}) and, for the per-source collector endpoints, [Accessing the metrics]({{< relref "/integrate/redis-data-integration/observability#accessing-the-metrics" >}}). @@ -210,17 +202,20 @@ Dead-letter queue tables are reported as `..
`, so rejecte are attributed to their source. See [Rejected records]({{< relref "/integrate/redis-data-integration/data-pipelines/rejected-records" >}}). -## Existing sources are unaffected by the changes +## Upgraded pipelines keep their existing names Before RDI supported several sources, every source-scoped resource was named using `source` -in place of the actual source name in `config.yaml`. Pipeline sources that existed -before upgrading to RDI 2.0.0 keep using those names to ensure continuity: +in place of the actual source name in `config.yaml`. A source that existed before you +upgraded to RDI 2.0.0 keeps those names, whatever it is called in `config.yaml`, so that it +keeps running unchanged: -- The `source-db` and `source-db-ssl` secrets, and the `SOURCE_DB_*` environment - variables, so its `connection` keeps referencing `${SOURCE_DB_USERNAME}` and - `${SOURCE_DB_PASSWORD}` regardless of its name. +- The `source-db` and `source-db-ssl` secrets and the `SOURCE_DB_*` environment variables, + so its `connection` section keeps referencing `${SOURCE_DB_USERNAME}` and + `${SOURCE_DB_PASSWORD}`. - The `collector-source` collector. - Its existing streams, `data:{rdi}:.
`, and its captured position. - `rdi` as its `server_name`. +Sources you add after the upgrade use their own names, as described on this page. + See [Upgrading RDI]({{< relref "/integrate/redis-data-integration/installation/upgrade" >}}). diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md index a49be68c82..17c678c255 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md @@ -34,7 +34,7 @@ The following table summarizes the considerations to prepare a MongoDB database | Oplog | Sufficient size for snapshot and streaming | | Pre/Post Images | Enable on collections **only if using a custom key** | | Connection String | Must include all hosts, replicaSet (if applicable), authSource, credentials | -| MongoDB Atlas | **[SSL required](https://debezium.io/documentation/reference/stable/connectors/mongodb.html#mongodb-property-mongodb-ssl-enabled)**, provide root CA as `SOURCE_DB_CACERT` secret in RDI | +| MongoDB Atlas | **[SSL required](https://debezium.io/documentation/reference/stable/connectors/mongodb.html#mongodb-property-mongodb-ssl-enabled)**, provide root CA as the source's `CACERT` secret in RDI | | MongoDB mTLS | X.509 authentication requires source TLS secrets and MongoDB SSL properties | | Network | RDI Collector must reach all MongoDB nodes on required ports | @@ -90,13 +90,18 @@ db.createUser({ The RDI Collector requires a MongoDB connection string that includes all relevant hosts and authentication details. +The credential references come from the source name, which is `mongodb` in the examples on +this page. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) +for the source naming rules. + Example (Replica Set): ``` -mongodb://${SOURCE_DB_USERNAME}:${SOURCE_DB_PASSWORD}@host1:27017,host2:27017,host3:27017/?replicaSet=rs0&authSource=admin +mongodb://${MONGODB_DB_USERNAME}:${MONGODB_DB_PASSWORD}@host1:27017,host2:27017,host3:27017/?replicaSet=rs0&authSource=admin ``` Example (Sharded Cluster): ``` -mongodb://${SOURCE_DB_USERNAME}:${SOURCE_DB_PASSWORD}@host:30000 +mongodb://${MONGODB_DB_USERNAME}:${MONGODB_DB_PASSWORD}@host:30000 ``` - For Atlas, adjust the connection string accordingly (see example below). - Set `replicaSet` and `authSource` as appropriate for your deployment. @@ -119,22 +124,23 @@ db.runCommand({ ## 5. MongoDB Atlas specific requirements MongoDB Atlas only supports secure connections via SSL. -The root CA certificate for MongoDB Atlas must be added as a SOURCE_DB_CACERT secret in RDI. +The root CA certificate for MongoDB Atlas must be added as the source's `CACERT` secret in RDI. - Download the MongoDB Atlas root CA certificate. -- In RDI, add this certificate as a secret named SOURCE_DB_CACERT. +- In RDI, add this certificate with + `redis-di set-secret CACERT --db mongodb /path/to/atlas-ca.crt`. - Ensure that the `mongodb.ssl.enabled: true` setting is present in your RDI configuration. Example connection string for Atlas: ``` -mongodb+srv://${SOURCE_DB_USERNAME}:${SOURCE_DB_PASSWORD}@cluster0.mongodb.net/?authSource=admin +mongodb+srv://${MONGODB_DB_USERNAME}:${MONGODB_DB_PASSWORD}@cluster0.mongodb.net/?authSource=admin ``` ## 6. Self-hosted MongoDB mTLS and X.509 authentication -For self-hosted MongoDB deployments that require TLS, set the source CA certificate -as the `SOURCE_DB_CACERT` secret. For X.509 client certificate authentication, also -set the `SOURCE_DB_CERT` and `SOURCE_DB_KEY` secrets. See +For self-hosted MongoDB deployments that require TLS, set the source's `CACERT` secret to +the source CA certificate. For X.509 client certificate authentication, also set the +source's `CERT` and `KEY` secrets. See [Set secrets]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets" >}}) for the full list of source database TLS and mTLS secrets. @@ -145,11 +151,12 @@ properties in the source `advanced.source` section: advanced: source: mongodb.ssl.enabled: true - mongodb.ssl.keystore: /debezium/certs/source_db_keystore + mongodb.ssl.keystore: /debezium/certs/mongodb_db_keystore mongodb.ssl.keystore.password: debezium ``` -The RDI Collector builds `/debezium/certs/source_db_keystore` from the source +The keystore is named after the source, so a source named `mongodb` uses +`/debezium/certs/mongodb_db_keystore`. The RDI Collector builds it from the source database client certificate and private key secrets. Debezium requires the `mongodb.ssl.keystore` and `mongodb.ssl.keystore.password` properties to present the client certificate to MongoDB. diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md index 1e5b16966d..cdcb50c00f 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md @@ -150,12 +150,18 @@ database. Instructions for setting up the target database secrets are available [RDI deployment guide]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets-for-k8shelm-deployment-using-kubectl-command" >}}). **Optional**: If you prefer to use a service account credentials file instead of Workload Identity -authentication, you'll need to create a Spanner-specific secret named `source-db-credentials`. +authentication, you'll need to create a Spanner-specific secret named +`-db-credentials`, where `` is the name of the source in `config.yaml`. +The source configured in the next step is named `spanner`, so its secret is +`spanner-db-credentials`. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) +for the source naming rules. + This secret should contain the service account key file generated during the Spanner setup phase. Use the command below to create it: ```bash -kubectl create secret generic source-db-credentials --namespace=rdi \ +kubectl create secret generic spanner-db-credentials --namespace=rdi \ --from-file=gcp-service-account.json=~/spanner-reader-account.json \ --save-config --dry-run=client -o yaml | kubectl apply -f - ``` @@ -164,7 +170,7 @@ Be sure to adjust the file path (`~/spanner-reader-account.json`) if your servic stored elsewhere. {{< note >}} -If you create the `source-db-credentials` secret, you must also set `use_credentials_file: true` +If you create the `spanner-db-credentials` secret, you must also set `use_credentials_file: true` in your RDI configuration to use the credentials file instead of Workload Identity authentication. {{< /note >}} @@ -175,7 +181,7 @@ When configuring your RDI pipeline for Spanner, use the following example config ```yaml sources: - source: + spanner: type: flink connection: type: spanner diff --git a/content/integrate/redis-data-integration/data-pipelines/rejected-records.md b/content/integrate/redis-data-integration/data-pipelines/rejected-records.md index be992a1007..fb3cc0863d 100644 --- a/content/integrate/redis-data-integration/data-pipelines/rejected-records.md +++ b/content/integrate/redis-data-integration/data-pipelines/rejected-records.md @@ -41,10 +41,7 @@ RDI stores rejected records in the RDI database as capped Redis streams. Each DL stream corresponds to a source table and tracks the records rejected for that table. -DLQ stream names use the `dlq:` prefix followed by the source data stream name, which -includes the name of the -[source]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) -the records came from: +DLQ stream names use the `dlq:` prefix followed by the source data stream name: ```text dlq:data:{rdi}:..
@@ -58,7 +55,8 @@ dlq:data:{rdi}:postgresql.public.users ``` The dead-letter queue endpoints and the CLI report table names in that same qualified form, -so a rejected record is always attributed to the source it came from. +so a rejected record is always attributed to the source it came from. Sources that existed +before upgrading to RDI 2.0.0 do not have the source segment in the key. Some RDI versions or configurations can use a hash-tagged variant such as `dlq:{data:rdi:.
}`. To find all DLQ streams in the diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/_index.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/_index.md index 7053d21c06..2bdb685e2c 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/_index.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/_index.md @@ -76,6 +76,13 @@ The main sections of these files are: and is set to `true` by default. Set it to `false` if you need to use case-sensitive values for these properties. + `server_name`, `db`, `schema`, and `table` each accept either a single name or a list of names. + An entry prefixed with `regex:` is matched as a regular expression that must match the whole + value, rather than as a literal name. For example, `server_name: [mysql, postgresql]` selects + two sources, and `table: "regex:orders_[0-9]+"` selects every numbered `orders` table. A job + processes the records that match all of the properties it sets, so a single job can cover + several sources, schemas, or tables. + - `transform`: This is an optional section describing the transformation that the pipeline applies to the data before writing it to the target. The `uses` property specifies a *transformation block* that will use the parameters supplied in the `with` section. See the diff --git a/content/integrate/redis-data-integration/observability.md b/content/integrate/redis-data-integration/observability.md index 07bbb883f5..8d3bafc172 100644 --- a/content/integrate/redis-data-integration/observability.md +++ b/content/integrate/redis-data-integration/observability.md @@ -225,7 +225,7 @@ RDI reports with their descriptions. - Where the metric name has the `rdi_` prefix, this will be replaced by the Kubernetes namespace name if you supplied a custom name during installation. The prefix is always `rdi_` for VM installations. - Metrics with the `_created` suffix are automatically generated by Prometheus for counters and gauges to track when they were first created. -- The `rdi_incoming_entries` metric provides a detailed breakdown for each data source by operation type. The `data_source` label is the stream's table name, qualified with the source name for each source that captures under its own name, so you can break the metric down per source. +- The `rdi_incoming_entries` metric provides a detailed breakdown for each data source by operation type. The `data_source` label identifies the data stream the value belongs to, as the stream's key without the leading `data:` — for example, `{rdi}:mysql.inventory.customers`. The key contains the source name, so you can break the metric down per source. Sources that existed before upgrading to RDI 2.0.0 do not have the source segment in the key. - The `rdi_stream_event_latency_ms` metric helps monitor data freshness and processing delays. - The processor performance metrics are divided into two categories: - **Total metrics**: Accumulate values across all processed batches for historical analysis diff --git a/content/operate/rc/rdi/define.md b/content/operate/rc/rdi/define.md index bef1b571fa..102d2714dd 100644 --- a/content/operate/rc/rdi/define.md +++ b/content/operate/rc/rdi/define.md @@ -46,7 +46,7 @@ In the **Pipeline setup** step: - Snowflake _(Preview)_ {{}} -1. Enter a name for your source database in the **Source name** field. This is a name for the source database that will appear on Redis Cloud. +1. Enter a name for your source database in the **Source name** field. This is a display label for the source database on Redis Cloud. It does not affect the pipeline's configuration. 1. Select the target Redis Cloud database from the **Target database** list. {{}} diff --git a/content/operate/rc/rdi/quick-start.md b/content/operate/rc/rdi/quick-start.md index 298ae00591..944f26b0fb 100644 --- a/content/operate/rc/rdi/quick-start.md +++ b/content/operate/rc/rdi/quick-start.md @@ -45,7 +45,7 @@ Before you can create your first Data Integration pipeline for a Redis Cloud sub {{}} 1. Select **PostgreSQL** as the source database type. {{}} -1. Enter a name for your source database in the **Source name** field. This is a name for the source database that will appear on Redis Cloud. +1. Enter a name for your source database in the **Source name** field. This is a display label for the source database on Redis Cloud. It does not affect the pipeline's configuration. 1. Select **Continue to source** to move to the **Source configuration** step. {{}} diff --git a/content/operate/rc/rdi/setup.md b/content/operate/rc/rdi/setup.md index 88b34ac3fc..bc8048ca7e 100644 --- a/content/operate/rc/rdi/setup.md +++ b/content/operate/rc/rdi/setup.md @@ -49,7 +49,7 @@ Before you can set up your source connectivity and secrets, you need the AWS Acc - MongoDB - Snowflake {{}} -1. Enter a name for your source database in the **Source name** field. This is a name for the source database that will appear on Redis Cloud. +1. Enter a name for your source database in the **Source name** field. This is a display label for the source database on Redis Cloud. It does not affect the pipeline's configuration. 1. Select **Continue to source** to move to the **Source configuration** step. {{}} From a95f27ef09a7f2d9e641065ac80604c210cf6546 Mon Sep 17 00:00:00 2001 From: Nicolaus Weidner Date: Fri, 21 Aug 2026 22:16:28 +0200 Subject: [PATCH 8/8] RDSC-5610: More fixes --- .../redis-data-integration/data-pipelines/deploy.md | 4 ++-- .../data-pipelines/prepare-dbs/_index.md | 7 ++++++- .../redis-data-integration/installation/_index.md | 2 +- .../redis-data-integration/reference/api-migration.md | 7 ++++--- .../integrate/redis-data-integration/troubleshooting.md | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/deploy.md b/content/integrate/redis-data-integration/data-pipelines/deploy.md index 4b7a6e369e..81e05545ea 100644 --- a/content/integrate/redis-data-integration/data-pipelines/deploy.md +++ b/content/integrate/redis-data-integration/data-pipelines/deploy.md @@ -50,8 +50,8 @@ shows these references in use, and covers the source naming rules. {{< note >}}The older scope-prefixed keys like `SOURCE_DB_PASSWORD` or `TARGET_DB_PASSWORD` -are still accepted for pipelines that were created before the upgrade and that have a single -source. They are used without specifying `--db`. These keys are deprecated and will be +are still accepted, and are used without specifying `--db`. A `SOURCE_DB_*` key requires the +pipeline to have exactly one source. These keys are deprecated and will be removed in a future release, so all usage should migrate to the new keys.{{< /note >}} {{< note >}} diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/_index.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/_index.md index e1cef5a262..dd11c9529e 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/_index.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/_index.md @@ -20,7 +20,12 @@ weight: 1 Each database uses a different mechanism to track changes to its data and generally, these mechanisms are not switched on by default. RDI's Debezium collector uses these mechanisms for change data capture (CDC), -so you must prepare your source database before you can use it with RDI. +so you must prepare each source database before you can use it with RDI. + +A pipeline can capture from more than one source database, and every source needs its own +preparation. Follow the page for each source's database type. See +[Multiple sources in one pipeline]({{< relref "/integrate/redis-data-integration/data-pipelines/multiple-sources" >}}) +for how to configure several sources in one pipeline. RDI supports the following source databases: diff --git a/content/integrate/redis-data-integration/installation/_index.md b/content/integrate/redis-data-integration/installation/_index.md index c5e7bdee75..9c38973df9 100644 --- a/content/integrate/redis-data-integration/installation/_index.md +++ b/content/integrate/redis-data-integration/installation/_index.md @@ -19,6 +19,6 @@ weight: 20 The guides in this section explain the options you have for installing and upgrading RDI on your own servers. See the [Redis Cloud RDI guide]({{< relref "/operate/rc/rdi" >}}) to learn how to set up RDI for a cloud database. -Before you use RDI, you must also configure your source database to enable CDC. See the +Before you use RDI, you must also configure each of your source databases to enable CDC. See the [Prepare source databases]({{< relref "/integrate/redis-data-integration/data-pipelines/prepare-dbs" >}}) section to learn how to do this. \ No newline at end of file diff --git a/content/integrate/redis-data-integration/reference/api-migration.md b/content/integrate/redis-data-integration/reference/api-migration.md index eed803b967..ea4ef103d2 100644 --- a/content/integrate/redis-data-integration/reference/api-migration.md +++ b/content/integrate/redis-data-integration/reference/api-migration.md @@ -45,7 +45,7 @@ The API version is part of the URL. Update `/api/v1` requests to use `/api/v2` w | `PUT /api/v1/pipelines/sources` and source subresources | `PATCH /api/v2/pipelines/{name}` with `sources` in the payload | | `PUT /api/v1/pipelines/targets` and target subresources | `PATCH /api/v2/pipelines/{name}` with `targets` in the payload | | `PUT /api/v1/pipelines/processors` and `PUT /api/v1/pipelines/processors/{prop}` | `PATCH /api/v2/pipelines/{name}` with `processors` in the payload | -| Secret provider endpoints | `POST`, `PUT`, or `DELETE /api/v2/pipelines/{name}/secrets[/{key}]` | +| `PUT /api/v1/secrets` and `PUT /api/v1/secrets/{secret_name}` | `POST`, `PUT`, or `DELETE /api/v2/pipelines/{name}/secrets[/{key}]` with the `db` query parameter | | Source metadata, schemas, databases, tables, and columns endpoints | `GET /api/v2/pipelines/{name}/source-schemas/{source_name}` with the appropriate filters | | `POST /api/v1/pipelines/sources/dry-run` | `POST /api/v2/pipelines?dry_run=true` | | `POST /api/v1/pipelines/targets/dry-run` | `POST /api/v2/pipelines?dry_run=true` | @@ -118,7 +118,8 @@ Note: 2. Add the pipeline name to each v2 request. The only pipeline in 1.19.0 is always named `default`. 3. Check the pipeline response, or call `GET /api/v2/pipelines/{name}/status`, instead of polling an action ID. 4. Use `POST /api/v2/pipelines`, `PUT /api/v2/pipelines/{name}`, or `PATCH /api/v2/pipelines/{name}` to update source, target, processor, and secret-provider settings as needed. When using `PATCH`, omit the configuration sections that you do not want to change. -5. Use `GET /api/v2/pipelines/{name}/metric-collections/{collection_name}` for monitoring and `GET /api/v2/pipelines/{name}/source-schemas/{source_name}` for source metadata. -6. Test creating, updating, validating, starting, stopping, resetting, and deleting a pipeline on a non-production RDI 1.19.0 or later installation before updating production applications. +5. Change secret requests to address a secret by a database-independent key and a `db` parameter. Where a v1 request set `SOURCE_DB_PASSWORD`, a v2 request sets the `PASSWORD` key with `db` naming the source, and `TARGET_DB_PASSWORD` becomes the `PASSWORD` key with `db=target`. See [Set secrets]({{< relref "/integrate/redis-data-integration/data-pipelines/deploy#set-secrets" >}}). +6. Use `GET /api/v2/pipelines/{name}/metric-collections/{collection_name}` for monitoring and `GET /api/v2/pipelines/{name}/source-schemas/{source_name}` for source metadata. +7. Test creating, updating, validating, starting, stopping, resetting, and deleting a pipeline on a non-production RDI 1.19.0 or later installation before updating production applications. Authentication and the API base URL do not change. The migration requires updates to the endpoint paths, pipeline scoping, request models, and operation status handling. diff --git a/content/integrate/redis-data-integration/troubleshooting.md b/content/integrate/redis-data-integration/troubleshooting.md index b393e1535b..306d93f153 100644 --- a/content/integrate/redis-data-integration/troubleshooting.md +++ b/content/integrate/redis-data-integration/troubleshooting.md @@ -42,7 +42,7 @@ By default, RDI records the following logs in the host VM file system at | :-- | :-- | | `rdi_collector-collector-initializer.log` | Initializing the collector. | | `rdi_collector-debezium-ssl-init.log` | Establishing the connector SSL connections to the source and RDI database (if you are using SSL). | -| `rdi_collector-collector-.log` | Collector [change data capture (CDC)]({{< relref "/integrate/redis-data-integration/architecture" >}}) operations. There is one log per source, named after the source. | +| `rdi_collector-collector-source.log` | Collector [change data capture (CDC)]({{< relref "/integrate/redis-data-integration/architecture" >}}) operations. | | `rdi_rdi-rdi-operator.log` | Main [RDI control plane]({{< relref "/integrate/redis-data-integration/architecture#how-rdi-is-deployed" >}}) component. | | `rdi_processor-processor.log` | RDI stream processing. |