From cda285231dffe77aae804a34dde98456b0132367 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:14:02 +0530 Subject: [PATCH 01/13] refactor(core): remove orphaned pipeline model family Delete the shadowed core Pipeline path (PipelineService, InMemoryPipelineRepository, PipelineRepository, model Pipeline/ PipelineEvent/PipelineStatus) and the unused api PipelineEntity family (PipelineEntity, PipelineJpaRepository, PipelineRepositoryAdapter, PipelineEntityMapper) plus orphaned DTOs (CreatePipelineRequest, PipelineResponse, UpdatePipelineRequest). The live pipeline path is PipelineDesignerService + PipelineDesignEntity. Leave the `pipelines` DB table intact for Flyway checksum compatibility. --- .claude/worktrees/model-cleanup | 1 + .claude/worktrees/security | 1 + .claude/worktrees/state-persistence | 1 + .claude/worktrees/writer-cdc | 1 + docs/reviews/architecture-review.md | 374 ++++++++++++++++++ .../api/dto/CreatePipelineRequest.java | 14 - .../syncflow/api/dto/PipelineResponse.java | 25 -- .../api/dto/UpdatePipelineRequest.java | 14 - .../api/pipeline/entity/PipelineEntity.java | 55 --- .../api/pipeline/mapper/JsonMapper.java | 18 - .../pipeline/mapper/PipelineEntityMapper.java | 38 -- .../repository/PipelineJpaRepository.java | 11 - .../repository/PipelineRepositoryAdapter.java | 68 ---- .../api/pipeline/mapper/JsonMapperTest.java | 48 --- .../mapper/PipelineEntityMapperTest.java | 201 ---------- .../com/syncflow/core/model/Pipeline.java | 41 -- .../syncflow/core/model/PipelineEvent.java | 11 - .../syncflow/core/model/PipelineStatus.java | 5 - .../InMemoryPipelineRepository.java | 49 --- .../core/repository/PipelineRepository.java | 21 - .../core/service/PipelineService.java | 129 ------ .../core/service/PipelineServiceTest.java | 61 --- 22 files changed, 378 insertions(+), 809 deletions(-) create mode 160000 .claude/worktrees/model-cleanup create mode 160000 .claude/worktrees/security create mode 160000 .claude/worktrees/state-persistence create mode 160000 .claude/worktrees/writer-cdc create mode 100644 docs/reviews/architecture-review.md delete mode 100644 syncflow-api/src/main/java/com/syncflow/api/dto/CreatePipelineRequest.java delete mode 100644 syncflow-api/src/main/java/com/syncflow/api/dto/PipelineResponse.java delete mode 100644 syncflow-api/src/main/java/com/syncflow/api/dto/UpdatePipelineRequest.java delete mode 100644 syncflow-api/src/main/java/com/syncflow/api/pipeline/entity/PipelineEntity.java delete mode 100644 syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapper.java delete mode 100644 syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineJpaRepository.java delete mode 100644 syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineRepositoryAdapter.java delete mode 100644 syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapperTest.java delete mode 100644 syncflow-core/src/main/java/com/syncflow/core/model/Pipeline.java delete mode 100644 syncflow-core/src/main/java/com/syncflow/core/model/PipelineEvent.java delete mode 100644 syncflow-core/src/main/java/com/syncflow/core/model/PipelineStatus.java delete mode 100644 syncflow-core/src/main/java/com/syncflow/core/repository/InMemoryPipelineRepository.java delete mode 100644 syncflow-core/src/main/java/com/syncflow/core/repository/PipelineRepository.java delete mode 100644 syncflow-core/src/main/java/com/syncflow/core/service/PipelineService.java delete mode 100644 syncflow-core/src/test/java/com/syncflow/core/service/PipelineServiceTest.java diff --git a/.claude/worktrees/model-cleanup b/.claude/worktrees/model-cleanup new file mode 160000 index 0000000..f2e628f --- /dev/null +++ b/.claude/worktrees/model-cleanup @@ -0,0 +1 @@ +Subproject commit f2e628fde115a12ce86a99f5f9ca93fb3c52789b diff --git a/.claude/worktrees/security b/.claude/worktrees/security new file mode 160000 index 0000000..f2e628f --- /dev/null +++ b/.claude/worktrees/security @@ -0,0 +1 @@ +Subproject commit f2e628fde115a12ce86a99f5f9ca93fb3c52789b diff --git a/.claude/worktrees/state-persistence b/.claude/worktrees/state-persistence new file mode 160000 index 0000000..f2e628f --- /dev/null +++ b/.claude/worktrees/state-persistence @@ -0,0 +1 @@ +Subproject commit f2e628fde115a12ce86a99f5f9ca93fb3c52789b diff --git a/.claude/worktrees/writer-cdc b/.claude/worktrees/writer-cdc new file mode 160000 index 0000000..f2e628f --- /dev/null +++ b/.claude/worktrees/writer-cdc @@ -0,0 +1 @@ +Subproject commit f2e628fde115a12ce86a99f5f9ca93fb3c52789b diff --git a/docs/reviews/architecture-review.md b/docs/reviews/architecture-review.md new file mode 100644 index 0000000..f229b18 --- /dev/null +++ b/docs/reviews/architecture-review.md @@ -0,0 +1,374 @@ +# SyncFlow — Principal Architecture Review + +**Scope:** Full end-to-end review of the SyncFlow CDC platform (all 8 Gradle modules + infra). +**Date:** 2026-08-09 +**Method:** Reverse-engineered the actual runtime data flow from source (not the ADRs), then audited against the declared architecture. +**Verdict up front:** The platform has a sound **connector-plugin shell** and a handful of genuinely well-built pieces (keyset snapshot pagination, Debezium offset hygiene, AES-GCM credential encryption, identifier taint-guards). But as a whole it is **in-memory prototype software wearing production-grade clothing**: most "enterprise" subsystems are unpersisted `ConcurrentHashMap` stores, several headline runtime paths are broken or incomplete, there are two parallel/duplicate modeling hierarchies, and the CI/k8s/HPA story is aspirational. This reads like an impressive portfolio/demo codebase, not something safe to run against real customer databases. + +--- + +## 1. Reverse-Engineered Architecture (what actually exists) + +### 1.1 Module graph (declared vs real) + +``` +syncflow-plugin-api (210 LOC, published SDK — mostly unused in-repo) + │ depends on nothing +syncflow-common (304 LOC — exceptions, correlation, tenant ids) + │ +syncflow-core (2439 LOC — Connector SPI, domain records, in-memory repos) + │ ← "hexagonal core" — only 2 Spring beans, most domain services are dead code +syncflow-connectors (2830 LOC — Debezium CDC, JDBC metadata/snapshot/writer, validators) + │ +syncflow-security (1 static class — used by API but the module is a shell) +syncflow-monitoring (EMPTY module — build.gradle only, zero sources) +syncflow-agent (172 LOC — agent heartbeat/register client; agent runtime state is IN MEMORY) + │ +syncflow-api (8458 LOC — ALL controllers, JPA entities, orchestrators, AI, plugin mgr) +``` + +**Key structural facts:** + +- **Two parallel "connection" worlds.** `syncflow-core/.../connection/` holds the rich domain model (`Connection`, `ConnectionProperties`, `ConnectionType` enum, `ConnectionStatus`) and SPI (`ConnectorFactory`, `ConnectionValidator`). `syncflow-core/.../model/ConnectionConfiguration` is a *second*, simpler connection record used by the runtime connectors, writers, `CaptureLifecycle`, `DestinationRouter`, `SnapshotExecutor`, and `MetadataDiscoveryService`. Every hop converts one to the other via a hand-written `toConfig()`/`ConnectorTypeMapper` — **the same `toConfig(Connection)` conversion is copy-pasted into at least 5 classes** (`CaptureLifecycle`, `DestinationRouter`, `SnapshotExecutor` ×3, `MetadataDiscoveryService`). +- **Two parallel "pipeline" worlds.** `syncflow-api/.../pipeline/entity/PipelineEntity` + `PipelineJpaRepository` + `PipelineRepositoryAdapter` + `PipelineEntityMapper` exist and are wired, but `PipelineDesignerService` (the live path) uses the **different** `PipelineDesignEntity`/`PipelineDesignJpaRepository`. The `PipelineEntity` family is redundant and likely orphaned. +- **`PipelineService` + `InMemoryPipelineRepository` + `PipelineEvent` (core)** — a *third* pipeline CRUD path, in-memory, with an unbounded `eventLog`. It is not the live path either (the live path is `PipelineDesignerService`). ADR-005 claims Postgres persistence; the runtime state machines are in-memory. + +### 1.2 The actual CDC → sync data flow (source of truth) + +``` +User → REST controller (SyncController.start / CaptureController.start) + → SyncOrchestrator.start(pipelineId) + ├─ CaptureLifecycle.start(pipelineId) + │ ├─ resolve CdcCapableConnector from SpringConnectorRegistry + │ ├─ buildPublisher(): KafkaEventPublisher OR BoundedQueueEventPublisher + │ └─ connector.startCDC(ctx, consumer) → DebeziumEngine on a virtual thread + │ └─ per-event: parse JSON → CDCEvent → publisher.publish(event) + │ + └─ start per-pipeline virtual-thread worker: + LinkedBlockingQueue(10k) ← drainTo(100) batch + → FilterProcessor → TransformProcessor + → DestinationRouter.write(connectionId, event, destCols) + → WriterRegistry → JdbcBatchWriter (JDBC, autoCommit=false) + → INSERT INTO currentTable ... + → EventIdempotencyStore (JPA) / RetryEngine / DeadLetterQueue (JPA) + +SSE: StatusBroadcaster.fan-out ← SnapshotExecutor / SyncOrchestrator.emit() +Snapshot: SnapshotExecutor → SnapshotCapableConnector.readBatch (keyset pagination) + → FilterProcessor→TransformProcessor → DestinationWriter → checkpoint every 5 batches +``` + +**Criticality note on ordering:** `SyncOrchestrator.start()` starts **CDC before** the sync worker queue exists, and CDC is a *firehose* — the bounded publisher drops the oldest event when full (by design, `BoundedQueueEventPublisher`). If the sink (snapshot→writer) stalls, the queue overflows and **events are silently dropped with only a WARN**. There is no backpressure to Debezium. For a CDC platform this is a data-loss bug waiting to happen under any load spike. + +### 1.3 What the ADRs claim vs what is real + +| ADR | Claim | Reality | +|---|---|---| +| ADR-001 Spring Modulith | Compile-time module boundaries | **Not enforced.** No `spring-modulith-starter-core` in any build.gradle, no `@ApplicationModule`, no Modulith tests. The only ArchUnit test (`ArchitectureTest`) exists but Modulith as such is absent. | +| ADR-002 Hexagonal | Core has zero framework deps, SPI only | Core **does** depend on `spring-context`, `hibernate-validator`, Jackson, and (transitively) Lombok. SPI is real, but the "pure domain" claim is overstated; `model/Pipeline` carries JPA-validation annotations. | +| ADR-003 Event-stream abstraction | `EventPublisher` with pluggable transports | Real (`EventPublisher`, `InMemory`, `BoundedQueue`, `KafkaEventPublisher`). Good. But the default production path is the bounded queue with drop-oldest — not Kafka. | +| ADR-004 Virtual threads | All concurrency on virtual threads | Real and well-done (Debezium engine, sync worker, snapshot worker, Kafka consumer). Solid. | +| ADR-005 Postgres metadata | Postgres for pipelines, connections, audit, checkpoints | **Half true.** Connections, pipeline *designs*, DLQ, idempotency, offsets are JPA/Flyway persisted. But pipeline *state machines*, snapshot jobs/progress, sync jobs/statistics, workflow instances, quota, RBAC, audit logs, API keys, agent fleet, alerts, and governance lineage are all **in-memory `ConcurrentHashMap`**. | +| ADR-006 pgvector | Deferred, architecture-ready | The "ready" `KnowledgeBase` is a naive keyword store; nothing vector. Fine as deferred, but the claim of readiness is generous. | +| ADR-007 OpenTelemetry | OTLP traces | Config present (`management.otlp.tracing.endpoint`), but no OTel dependencies are on the classpath — the javaagent starter lib is declared but I saw no agent attach. Trace IDs in the log pattern won't be populated without the agent. | +| ADR-008 Agent REST | 6 REST endpoints, mTLS | Endpoints exist. **mTLS does not exist anywhere** in code or k8s; `/api/agents/*` is not in the public-path allowlist, so agents would need a JWT that the agent module never obtains. The agent runtime state is also in-memory (lost on restart). | +| ADR-009 Control/data plane | Agents self-contained, resilient to CP loss | Agent module exists but is a thin HTTP client; there is no local work queue or checkpoint persistence in the agent — the "resilience" claim isn't implemented. | +| ADR-010 Plugin SDK | Plugins loaded via isolated URLClassLoader | `PluginManager` is real (install/enable/disable from manifest). But there is **no persistence** for installed plugins (lost on restart), no signature/security check on the JAR, and the SDK (`syncflow-plugin-api`) is never consumed by the built-in connectors. | + +--- + +## 2. Bad Architecture Decisions + +### 2.1 In-memory "enterprise" subsystems — the core disease +The biggest architectural failure: **state that must survive restarts and be multi-instance safe is in process-local `ConcurrentHashMap`s.** HPA scales replicas 2→10, but each replica has its own private copy of snapshot jobs, sync jobs, workflow state, quota, audit, API keys, and agent fleet — so scale-out actively breaks correctness (a job started on pod A is invisible to pod B; SSE subscribers on A never see events from B). KEDA scaling on `syncflow_queue_depth` / `syncflow_workflow_queue_size` is watching **in-memory gauges per-pod**, which makes autoscaling near-meaningless. + +Affected (non-exhaustive): `SnapshotExecutor`, `SyncOrchestrator`, `WorkflowScheduler`, `FleetManager`, `QuotaEngine`, `EnterpriseAuditStore`, `ApiKeyStore`, `DataGovernanceService`, `InMemoryPipelineRepository`, `CheckpointStore`, `StatusBroadcaster` (SSE is per-pod by nature). + +### 2.2 `SnapshotExecutor` holds mutable job objects in a shared map that it mutates +`jobs.put(id, job.withProgress(...))` is a publish-on-write pattern with immutable snapshots — actually OK. The real issue is **no persistence of jobs/progress**, plus the job map is never cleaned except on cancel/failure — completed snapshot jobs leak in memory forever (`remove()` is only called on cancel and failure paths, **not on success**). + +### 2.3 Two connection models + three pipeline models = "hexagonal" in name +The duplication isn't harmless — the runtime path is the `ConnectionConfiguration` model, the domain model is `Connection`, and the two enums (`ConnectionType` vs `ConnectorType`) must be mapped by hand at every hop. Every new connector touches 6+ files across 3 models. This is the opposite of the ADR-002 goal. + +### 2.4 Kafka is bolted on as an optional second path, not the architecture +`CaptureLifecycle.buildPublisher()` branches on `syncflow.kafka.enabled`. When off (default), the CDC→sync handoff is the bounded in-memory queue. When on, Debezium→Kafka producer→Kafka consumer→same in-memory queue→worker. So even "Kafka mode" ends in the same in-memory queue, and Kafka adds a full serialize/deserialize round trip plus topic provisioning. Kafka's actual value (replay, backpressure, multi-consumer) is **not realized** — the consumer is per-pipeline, per-pod, single-threaded. + +### 2.5 Agent/Fleet is a control-plane façade with no real distributed protocol +ADR-008/009 describe a fleet of self-contained data-plane agents. Reality: `AgentController` + `FleetManager` (in-memory) + a thin `SyncFlowAgent` client that registers and heartbeats. No mTLS, no auth, no local persistence, no task dispatch that actually executes work. + +### 2.6 `syncflow-security` and `syncflow-monitoring` are empty/stub modules +`syncflow-security` has one static helper (`SecurityConfig` with a hardcoded public-path list). `syncflow-monitoring` has **zero sources** — yet the root `build.gradle` wires both into the build and ADR-007 credits them. The real security config lives in `syncflow-api` (`WebSecurityConfig`, `JwtSecurityConfig`). This is module-layout theater. + +### 2.7 Workflow subsystem is non-functional +`WorkflowScheduler.completedTaskIds()` **always returns `Set.of()`**. Combined with `tick()` re-enqueuing tasks every 2s, `start()` enqueues all root tasks but nothing ever marks a task complete, so the workflow graph can never progress. `TaskQueue` enqueues with no workers executing task types. The whole `syncflow-core/workflow` + `WorkflowScheduler` + `WorkflowBuilder` is scaffolding with no engine. + +### 2.8 `PipelineEvent` / `eventLog` in `PipelineService` is an unbounded in-memory list +`List eventLog` grows forever with every pipeline transition; `events()` returns the whole thing. Minor in practice (core path is dead), but it's a memory leak in the one core service that is a Spring bean. + +--- + +## 3. Duplicate Logic + +| Duplicate | Locations | Cost | +|---|---|---| +| `toConfig(Connection)` / `ConnectorTypeMapper.toCore(...)` hand-conversion | `CaptureLifecycle`, `DestinationRouter`, `SnapshotExecutor` (3×), `MetadataDiscoveryService`, `ConnectionController` | Every connector change ripples through 5+ sites | +| `ConnectionType` vs `ConnectorType` enums + manual maps | core/connection vs core/model; `ConnectionValidatorRegistry` (deduces type by probing `supports()`) | Two type systems for one concept | +| Filter/Transform pipeline | `FilterProcessor`/`TransformProcessor` are correct and **reused** by both snapshot and sync — good. But transform `EXPRESSION` type is a no-op stub (`EXPRESSION -> value`) while `PipelineValidator` validates a different rule shape | EXPRESSION silently passes values through | +| PK extraction for events | `PostgresCdcConnector.extractPk` (Debezium key envelope + common-name fallback) vs `MySqlCdcConnector`/`MongoDbCdcConnector` (own copies) | Three near-identical parsers | +| Metadata cache pattern | `MetadataCache` has 5 near-identical `get*/put*` pairs + 5 Caffeine caches | ~100 lines of boilerplate that a single `Cache` or a map of caches would replace | +| Registry pattern | `SpringConnectorRegistry`, `ConnectionValidatorRegistry`, `WriterRegistryImpl`, (plugin) `PluginManager` — 4 hand-rolled registries | Each has its own registration/type-resolution quirks (see WriterRegistry below) | +| Offset save/restore | `OffsetStore` (JPA) vs Debezium `FileOffsetBackingStore` (tmpdir) | Two offset stores that don't talk to each other | + +--- + +## 4. Performance Bottlenecks + +1. **Per-event JDBC connection churn (critical).** `DestinationRouter.write()` does `writer.connect()` → `DriverManager.getConnection` → write → `commit()` → `close()` **for every single CDC event**, one event at a time. No connection pooling, no batching across events (only the snapshot path batches via `JdbcBatchWriter`'s internal buffer). At any real CDC rate this is catastrophic — connection setup dominates. `JdbcBatchWriter`'s `writeBatch` is only called with 1 row from the router path. +2. **`JdbcBatchWriter` flush threshold is never reached on the sync path.** It buffers until 1000 rows, but the router commits after each event, so the buffer never fills and `executeBatch` is effectively single-row. +3. **`EventIdempotencyStore.isProcessed()` is a DB `exists()` per event**, plus `markProcessed()` does another `exists()` then `save()`. Two round trips per event, each opening/committing its own transaction (`@Transactional` on the class). No batching, no caching. This is on the hot path. +4. **`RetryEngine` counts retries per eventId in a map that's never cleaned on the non-retry path** (`evaluate` removes only when DLQ-ing); `activeRetries()` grows with distinct failing events that retry-but-never-succeed. Bounded only by DLQ eventual eviction. (Minor memory issue, but on a retry-storm it grows.) +5. **Snapshot `estimateRows` per table + `readBatch` per batch opens one JDBC connection per call** — `AbstractJdbcSnapshotConnector` holds a single `jdbcConnection` but `AbstractJdbcMetadataConnector.connect()` closes/reopens each `ensureConnected`. Actually reconnect only happens when disconnected, so this is fine; the real cost is the snapshot re-reading `fetchPrimaryKey` per batch and the executor calling `connectionService.getWithDecryptedCredentials` (a **decrypt** + DB read) per table per job — cheap but repeated. +6. **SSE fan-out serializes each payload with a fresh `ObjectMapper` write per emit** — fine at low rates; the bigger issue is the unbounded subscriber map with `DEFAULT_TIMEOUT_MS = 0` (no timeout) — dead SSE connections only cleaned on emit error. +7. **`DataGovernanceService.classifyColumns` / schema-history scans** do linear scans of the whole `ConcurrentHashMap` per query (`.values().stream().filter(...).sorted(...)`) — fine at demo scale, O(n) per lookup at scale. +8. **Kafka producer per pipeline with `linger.ms=5` / `batch.size=65536`** is reasonable, but the **consumer is single-threaded per pipeline and commits sync after each poll batch** — throughput ceiling and no parallelism across partitions. + +--- + +## 5. Scalability Risks + +1. **Everything is in-memory and per-pod** (2.1) — scale-out breaks correctness, not just availability. The single most important risk. +2. **Per-event JDBC connect (4.1)** — the platform cannot survive real CDC volume on the sync path. +3. **Debezium offset storage in `java.io.tmpdir`** — pod-local, ephemeral, deleted on restart/reschedule. `offset.storage.file.filename` points at `/tmp/syncflow_offset_*.dat`. Combined with the JPA `OffsetStore` never being fed back into Debezium, **offset persistence is broken across restarts** — a restarted pipeline re-reads from `FileOffsetBackingStore` in a wiped tmpdir → potential re-processing or missed events. This is the #2 critical risk after the in-memory store. +4. **Postgres slot leak / name collision.** `PostgresCdcConnector.specificProperties()` builds `slot.name`/`publication.name` from **`sanitize(config.database())` only** — the comment says "pipeline-specific suffix via pipelineId" but the code **never uses the pipelineId**. Two pipelines on the same database → **same replication slot and publication → hard conflict**, and `slot.drop.on.stop=false` means slots leak on the source DB forever. Production footgun. +5. **`BoundedQueueEventPublisher` drop-oldest (1.2)** — no backpressure; data loss under load. Also the sync worker drains with `drainTo(100)`/500ms poll while CDC can produce far faster. +6. **`SnapshotExecutor` checkpoint is in-memory** — a restart mid-snapshot loses the resume cursor (re-keysets from scratch, or worse with OFFSET fallback). The checkpoint feature the code carefully builds (every-5-batches) is non-durable. +7. **HPA/KEDA scale out of a single Postgres** — connections/dlpq/idempotency/offsets all hit one DB; pool capped at 10 (ADR says 10), which the per-event connect churn will exhaust instantly. +8. **No rate limiting or quota enforcement on hot paths** — `QuotaEngine.checkLimit` exists but nothing calls it on pipeline create/sync. + +--- + +## 6. Correctness Bugs (confirmed in source) + +1. **`JdbcBatchWriter.currentTable` is never assigned.** `buildInsertSql()` interpolates `"INSERT INTO " + currentTable` where `currentTable` is a field declared but never written (it's `null` → `"INSERT INTO null (...)"`, an immediate SQLException). The `DestinationRouter` passes `tableName` into `writeBatch(table, ...)` but `writeBatch` **ignores its `table` parameter** and uses `buffer.getFirst().keySet()` for columns. **The CDC sync write path cannot write a single row.** This is the #1 bug — the headline "sync" feature is broken end to end. +2. **`BoundedQueueEventPublisher` drops the oldest event on overflow** — silent data loss (documented in class, but it's a policy that defeats the purpose of a CDC platform). +3. **`WorkflowScheduler.completedTaskIds()` returns empty** — workflow engine can never complete (2.7). +4. **`DeadLetterQueue.replay()` only sets `replayedAt`/marks it** — it does **not re-submit the event to `SyncOrchestrator`**. Replay is a no-op flag flip. (I recall this was covered in earlier memory — confirmed again here.) +5. **`MySqlWriter`/`PostgresWriter` jdbcUrl/props duplication** is fine, but `JdbcBatchWriter.flush()` uses `buffer.getFirst().keySet()` for columns — row-key order isn't guaranteed stable across `LinkedHashMap` rows; mixed key sets across rows → wrong SQL. (Minor vs #1.) +6. **`CaptureLifecycle.stop()`/`shutdownAll()` flush+close the publisher but the publisher may be the in-memory `BoundedQueueEventPublisher` whose buffered events are simply discarded.** Pending events between Debezium offset and consumer are lost on stop. +7. **`RetryEngine` on transient failure increments `retries` but never actually retries** — `processEvent` moves on; the "retry" is only counted, and `RetryDecision.delay` is never used to re-deliver. So transient errors are counted as retries then DLQ'd at MAX_RETRIES without ever re-attempting. The retry mechanism is a counter, not a retry. +8. **Hardcoded encryption key + JWT secret in `application.yml`**: `syncflow.encryption.key: MDEyMzQ1Njc4OWFiY2RlZg==` (the literal bytes `0123456789abcdef`) and a `jwt.secret` default. K8s deployment overrides the encryption key via secret, but **no JWT secret is set in k8s** — falls back to the committed default. Anyone with repo access can forge tokens. +9. **`JwtProperties` default secret contains a `?`** (`...LWRUV9jaGFuZ2UtaW4tcHJvZA==`), which is **not valid base64** → `JwtSecurityConfig.secretKey()` throws on startup if `SYNCFLOW_JWT_SECRET` is unset. The app likely **fails to boot** in a plain `docker compose` environment. (Contradicts the "works locally" story.) +10. **Tenant context is header-injectable**: `TenantFilter` accepts `X-Tenant-Id` as a **plain HTTP header** and uses it verbatim for scoping (unless overridden by JWT subject). With `fail-on-unknown-properties:false` and no claim mapping, a caller can set `X-Tenant-Id: ` and, on any endpoint that only checks `TenantContextHolder`, operate in another tenant's scope. RBAC checks (`AuthorizationService`) are only invoked in `AdminController`; most controllers (`ConnectionController`, `PipelineDesignerController`, `SyncController`, `SnapshotController`) do **no authorization** beyond authentication. +11. **`AdminController.createOrg/createWorkspace/createProject` are stubs** — they generate an id and return it without persisting anything. +12. **Agent endpoints (`/api/agents/*`) are authenticated** but the agent has no way to authenticate (no JWT issuance flow for agents) — the control-plane/agent loop is unreachable in practice. + +--- + +## 7. Maintainability Issues + +- **135+ source files with 20+ hand-rolled in-memory maps and repeated conversion helpers** — the "clean hexagonal" ADR narrative doesn't match the code, making it hard for a new engineer to know which of the 3 pipeline/2 connection models is real. +- **MapStruct is used (`ConnectionMapper`, `PipelineDesignEntityMapper`) but half the mappings are hand-written** (Jackson round-trips, `toConfig` helpers) — two mapping paradigms side by side. +- **`New ObjectMapper()` created ad hoc** in `ConnectionMapper.toJson/parseOptions` and `ConnectionService.serializeOptions` — ignoring the Spring-injected singleton (which has JSR-310 modules configured). Non-deterministic date handling, wasted allocations. +- **`CaptureLifecycle` / `SyncOrchestrator` / `SnapshotExecutor` are 200-300 line god-components** doing lifecycle + routing + metrics + SSE + threading in one class. +- **Magic strings everywhere**: `"syncflow.sync.events.processed"`, `"pipeline"` tags, status names (`"STOPPED"`, `"RUNNING"`) returned as raw `Map.of(...)` in controllers — no response DTOs for most endpoints. +- **`SnapshotExecutor.remove()` never called on success** → completed job objects leak (also 2.2). +- **`FleetManager`/`QuotaEngine`/`EnterpriseAuditStore`/`ApiKeyStore`/`AlertEngine` all `@Component` with in-memory state** — they masquerade as persistent enterprise services. +- **`README`/ADR/CHANGELOG describe capabilities the code doesn't have** (Modulith, OTel, mTLS, plugin persistence, workflow execution) — documentation drift makes onboarding actively misleading. + +--- + +## 8. A Clean Architecture Breakdown (target) + +### 8.1 Target module layout + +``` +syncflow-plugin-api (unchanged — the real SPI, no framework deps) +syncflow-core (pure domain + SPI — remove spring-context, hibernate-validator, Jackson, Lombok from API surface) +syncflow-connectors (adapters — keep; split per-db modules later if they grow) +syncflow-api (control-plane: REST/GraphQL adapters, JPA, security) ← slims down +syncflow-agent (data-plane runtime — must gain local persistence + task executor) +syncflow-monitoring (DELETE or actually implement) +syncflow-security (DELETE; fold the one static helper into syncflow-api) +syncflow-common (keep — exceptions, tenant ids, correlation) +``` + +### 8.2 The one modeling correction that fixes the most duplication + +**Collapse the two connection models into one.** Make `ConnectionConfiguration` (or the `Connection` domain record) the single type flowing through the SPI, delete `toConfig`/`ConnectorTypeMapper`, and keep **one** enum. This removes ~6 conversion sites and the type-mapping matrix. Same for pipeline: pick `PipelineDesignEntity` as the persistence model, delete `PipelineEntity`/`PipelineRepositoryAdapter`/`InMemoryPipelineRepository`/core `PipelineService` (or make the core service the single source of truth and delete the API's). + +### 8.3 State that must move to Postgres (or Redis) to be production-safe + +| Subsystem | Now | Should be | +|---|---|---| +| Snapshot jobs / progress | in-memory map | Postgres table + `@Entity` (progress rows) | +| Sync jobs / statistics | in-memory map | Postgres table | +| Workflow instances/executions | in-memory + non-functional | Postgres + a real task worker | +| Quota, RBAC policy, API keys, audit | in-memory | Postgres | +| Agent fleet / heartbeats | in-memory | Postgres (or Redis) + staleness sweeper | +| DLQ replay | flag-only | real re-enqueue to the sync queue | +| Checkpoints | in-memory | Postgres (resume after restart) | + +### 8.4 The two runtime paths to unify + +**Delete the bounded-queue handoff; make Kafka (or a durable queue) the only transport.** Then: +- Debezium → Kafka (persistent, replayable, backpressurable) → partition-parallel consumers → idempotent writer. +- Remove `BoundedQueueEventPublisher` from production; keep it for tests only. +- Backpressure becomes Kafka's `max.poll.records` + consumer lag, not drop-oldest. + +### 8.5 Fix the connector model so adapters stop reimplementing everything + +Give `AbstractJdbcMetadataConnector`/`AbstractJdbcSnapshotConnector`/`AbstractConnectorValidator`/`JdbcBatchWriter` **real shared implementations** (connection pooling via `HikariDataSource`, single shared SQL build), and have each DB subclass supply only URL/type. Delete the stub `PostgresConnector`. Make `WriterRegistryImpl` resolve by `supports()` like `ConnectionValidatorRegistry` instead of `instanceof` checks. + +--- + +## 9. Refactoring Strategies (prioritized) + +**Tier 1 — makes the product not lie about itself (do first, highest ROI):** + +1. Fix `JdbcBatchWriter.currentTable` + honor the `table`/`columns` parameters; add a writer integration test against Testcontainers Postgres. *(One-line fix, unblocks the entire sync path.)* +2. Decide the single connection + single pipeline model; delete the duplicates and the `toConfig` matrix. *(Biggest structural cleanup, kills the most duplication.)* +3. Move runtime job state (snapshot/sync/workflow) to JPA entities. Use an outbox/event table to persist status transitions so SSE + cross-pod reads work. +4. Make Debezium offsets durable: use a Postgres-backed `JdbcOffsetBackingStore` (or feed the JPA `OffsetStore` into Debezium) instead of `/tmp` files. Include `pipelineId` in slot/publication names and add `slot.drop.on.stop` per pipeline lifecycle. + +**Tier 2 — scale/performance:** + +5. Connection pooling for writers (inject a shared `DataSource`), and batch events into the writer instead of connect/commit/close per event. +6. Batch idempotency checks (`IN` query) and cache processed IDs in-process with TTL. +7. Real retry delivery in `RetryEngine` (scheduled re-enqueue with backoff), or hand responsibility to Kafka. +8. Make checkpointing durable and call `remove()` on completed snapshots. + +**Tier 3 — hardening/security:** + +9. Move `syncflow.encryption.key` and `jwt.secret` out of committed defaults; **fail fast with a clear error** instead of shipping a weak/`?`-corrupt default. Fix the `?` in the JWT default. +10. Tenant scoping from the **authenticated principal only** (drop trust in `X-Tenant-Id` headers), and add `AuthorizationService` checks to `ConnectionController`/`PipelineDesignerController`/`SyncController`/`SnapshotController`. +11. Remove or genuinely implement the workflow engine; if kept, wire a real task executor and persist execution state. +12. Implement or delete `syncflow-monitoring`/`syncflow-security` as separate modules; implement mTLS + agent auth or remove the claim. + +--- + +## 10. Production-Grade Code Samples + +### 10.1 Fix the broken writer (Tier 1, #1) + +```java +// syncflow-connectors/.../writer/JdbcBatchWriter.java — corrected core +public abstract class JdbcBatchWriter implements DestinationWriter { + protected Connection connection; // inject a DataSource instead for pooling + private String currentTable; + private List currentColumns; + private final List> buffer = new ArrayList<>(); + + @Override + public void writeBatch(String table, List> rows, List columns) { + if (rows.isEmpty()) return; + if (currentTable == null) { currentTable = table; currentColumns = columns; } + else if (!currentTable.equals(table)) { flush(); currentTable = table; currentColumns = columns; } + buffer.addAll(rows); + if (buffer.size() >= 1000) flush(); + } + + @Override + public void flush() { + if (buffer.isEmpty() || connection == null) return; + var cols = currentColumns; // use the passed columns, not row keyset + var sql = buildInsertSql(cols); + try (var stmt = connection.prepareStatement(sql)) { + for (var row : buffer) { + for (int i = 0; i < cols.size(); i++) stmt.setObject(i + 1, row.get(cols.get(i))); + stmt.addBatch(); + } + stmt.executeBatch(); + buffer.clear(); + } catch (SQLException e) { throw new RuntimeException("Batch write failed", e); } + } +} +``` +**Note:** this still opens a JDBC connection per `connect()`; the durable fix is to inject a pooled `DataSource` and remove the per-event `connect`/`commit`/`close` in `DestinationRouter`. + +### 10.2 Durable offset for Debezium (Tier 1, #4) + +```java +// Postgres-backed offset store passed to Debezium instead of FileOffsetBackingStore +debeziumProps.setProperty("offset.storage", + "org.apache.kafka.connect.storage.JdbcOffsetBackingStore"); +debeziumProps.setProperty("offset.storage.jdbc.url", jdbcUrl); +debeziumProps.setProperty("offset.storage.jdbc.user", user); +debeziumProps.setProperty("offset.storage.jdbc.password", password); +debeziumProps.setProperty("offset.storage.table.name", "debezium_offsets"); +// And scope slot/publication per pipeline so two pipelines on one DB don't collide: +props.setProperty("slot.name", "syncflow_slot_" + sanitize(db) + "_" + sanitize(pipelineId)); +props.setProperty("publication.name", "syncflow_pub_" + sanitize(db) + "_" + sanitize(pipelineId)); +``` +The `pipelineId` must come from `ctx.runtimeProperties()` (already available) and be threaded into `specificProperties(config)` — today the parameter is ignored. + +### 10.3 Backpressure without drop-oldest (Tier 1, #3) + +```java +// Give Debezium a blocking publisher so overflow backpressures instead of dropping. +public class BackpressuredEventPublisher implements EventPublisher { + private final BlockingQueue queue = new LinkedBlockingQueue<>(CAPACITY); + @Override public void publish(CDCEvent e) { + try { queue.put(e); } // blocks the Debezium thread when full + catch (InterruptedException ie) { Thread.currentThread().interrupt(); } + } + // drain() unchanged; consumers apply Kafka-style max.poll semantics. +} +``` +This converts silent data loss into backpressure (CDC pauses until the sink catches up). Pair with a WARN when `queue.size()` stays near capacity so ops can scale the sink. + +### 10.4 Tenant scoping from principal only (Tier 3, #10) + +```java +// TenantFilter: never trust the header unless it matches the authenticated subject. +var auth = SecurityContextHolder.getContext().getAuthentication(); +if (auth == null || !auth.isAuthenticated()) { + // allow only DEFAULT tenant / anonymous for public endpoints + TenantContextHolder.set(TenantContext.anonymous()); + chain.doFilter(request, response); return; +} +// Derive tenant from the JWT claim (e.g. "tenant") — ignore X-Tenant-Id entirely. +var tenant = (String) ((Jwt) auth.getPrincipal()).getClaims().get("tenant"); +TenantContextHolder.set(TenantContext.of(TenantId.from(tenant), auth.getName(), rolesOf(auth))); +``` + +### 10.5 Batch idempotency (Tier 2, #6) + +```java +// EventIdempotencyStore — batch + in-process TTL cache instead of per-event DB round trips +@Component +public class EventIdempotencyStore { + private final Cache recent = Caffeine.newBuilder() + .maximumSize(100_000).expireAfterWrite(Duration.ofHours(24)).build(); + + public boolean isProcessed(String eventId) { + var cached = recent.getIfPresent(eventId); + if (cached != null) return true; + return repository.existsByEventId(eventId); // first miss only + } + public void markProcessed(String eventId) { + recent.put(eventId, Boolean.TRUE); + // debounced async flush to DB (outbox) — not in the hot path + } +} +``` +This cuts per-event DB traffic from 2 queries to 0 on the hot path. + +--- + +## 11. Critical Problem Areas — Ranked + +1. **Sync write path is broken** (`JdbcBatchWriter.currentTable` → `INSERT INTO null`; router commits per event, so the sync feature cannot persist a row). **P0.** +2. **Everything critical is in-memory & per-pod** — scale-out breaks correctness; HPA/KEDA are misleading. **P0.** +3. **Debezium offsets in `/tmp` + persisted `OffsetStore` never feeds back** — restarts lose position; **plus slot/publication name collision on same-DB pipelines** with slots that never drop. **P0/P1.** +4. **No backpressure — drop-oldest under load** = silent CDC data loss. **P1.** +5. **Security defaults**: committed AES key, `?`-corrupt JWT default, header-trusted tenant scoping, minimal authorization on most controllers. **P1.** +6. **Kafka path adds latency without adding replay/backpressure value**, and the default is the queue. **P2 (architectural).** +7. **Workflow engine non-functional; DLQ replay is a no-op; retries are counters not retries.** **P1/P2.** +8. **Two connection models + three pipeline models** — the "hexagonal" claim is structural debt. **P2.** + +--- + +## 12. Bottom Line + +The codebase is a **strong portfolio-grade CDC platform with a real connector SPI, real Debezium wiring, keyset pagination, and credential encryption** — but the runtime orchestration layer (sync, snapshot, workflow, agent, enterprise ops) is **unpersisted, single-pod, and in several places non-functional**. Before it can run against real customer databases the priority order is: **(1)** fix the writer bug, **(2)** make runtime state durable, **(3)** make offsets and replication slots durable and pipeline-scoped, **(4)** add real backpressure, **(5)** fix the security defaults and tenant scoping. The clean architecture target collapses to: one connection model, one pipeline model, one event transport (durable), and every state machine backed by Postgres. + +*This document was produced as a read-only analysis. No code was changed.* diff --git a/syncflow-api/src/main/java/com/syncflow/api/dto/CreatePipelineRequest.java b/syncflow-api/src/main/java/com/syncflow/api/dto/CreatePipelineRequest.java deleted file mode 100644 index 634bfff..0000000 --- a/syncflow-api/src/main/java/com/syncflow/api/dto/CreatePipelineRequest.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.syncflow.api.dto; - -import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.model.TransformationConfiguration; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; - -public record CreatePipelineRequest( - @NotBlank String name, - @NotNull @Valid ConnectionConfiguration source, - @NotNull @Valid ConnectionConfiguration destination, - TransformationConfiguration mapping) { -} diff --git a/syncflow-api/src/main/java/com/syncflow/api/dto/PipelineResponse.java b/syncflow-api/src/main/java/com/syncflow/api/dto/PipelineResponse.java deleted file mode 100644 index 2d56349..0000000 --- a/syncflow-api/src/main/java/com/syncflow/api/dto/PipelineResponse.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.syncflow.api.dto; - -import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.model.Pipeline; -import com.syncflow.core.model.PipelineStatus; -import com.syncflow.core.model.TransformationConfiguration; - -import java.time.Instant; - -public record PipelineResponse( - String id, - String name, - PipelineStatus status, - ConnectionConfiguration source, - ConnectionConfiguration destination, - TransformationConfiguration mapping, - Instant createdAt, - Instant updatedAt) { - - public static PipelineResponse from(Pipeline p) { - return new PipelineResponse(p.getId(), p.getName(), p.getStatus(), - p.getSource(), p.getDestination(), p.getMapping(), - p.getCreatedAt(), p.getUpdatedAt()); - } -} diff --git a/syncflow-api/src/main/java/com/syncflow/api/dto/UpdatePipelineRequest.java b/syncflow-api/src/main/java/com/syncflow/api/dto/UpdatePipelineRequest.java deleted file mode 100644 index 2216bd8..0000000 --- a/syncflow-api/src/main/java/com/syncflow/api/dto/UpdatePipelineRequest.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.syncflow.api.dto; - -import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.model.TransformationConfiguration; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; - -public record UpdatePipelineRequest( - @NotBlank String name, - @NotNull @Valid ConnectionConfiguration source, - @NotNull @Valid ConnectionConfiguration destination, - TransformationConfiguration mapping) { -} diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/entity/PipelineEntity.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/entity/PipelineEntity.java deleted file mode 100644 index eb8ccb4..0000000 --- a/syncflow-api/src/main/java/com/syncflow/api/pipeline/entity/PipelineEntity.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.syncflow.api.pipeline.entity; - -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import lombok.Getter; -import lombok.Setter; -import org.hibernate.annotations.JdbcTypeCode; -import org.hibernate.type.SqlTypes; - -import java.time.Instant; - -/** - * JPA entity backed by the {@code pipelines} table defined in V1__init.sql. - * source, destination and mapping are stored as JSONB columns. - */ -@Setter -@Getter -@Entity -@Table(name = "pipelines") -public class PipelineEntity { - - @Id - @Column(length = 36) - private String id; - - @Column(nullable = false, length = 255) - private String name; - - @Column(nullable = false, length = 20) - private String status; - - @Column(nullable = false, columnDefinition = "jsonb") - @JdbcTypeCode(SqlTypes.JSON) - private String source; - - @Column(nullable = false, columnDefinition = "jsonb") - @JdbcTypeCode(SqlTypes.JSON) - private String destination; - - @Column(columnDefinition = "jsonb") - @JdbcTypeCode(SqlTypes.JSON) - private String mapping; - - @Column(name = "created_at", nullable = false) - private Instant createdAt; - - @Column(name = "updated_at", nullable = false) - private Instant updatedAt; - - public PipelineEntity() { - } - -} diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/JsonMapper.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/JsonMapper.java index 1b20c00..b32d031 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/JsonMapper.java +++ b/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/JsonMapper.java @@ -7,8 +7,6 @@ import com.syncflow.core.pipeline.PipelineSettings; import com.syncflow.core.pipeline.SourceReference; import com.syncflow.core.pipeline.mapping.TableMapping; -import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.model.TransformationConfiguration; import org.springframework.stereotype.Component; import java.util.List; @@ -46,14 +44,6 @@ public String fromPipelineSettings(PipelineSettings value) { return toJson(value); } - public String fromConnectionConfiguration(ConnectionConfiguration value) { - return toJson(value); - } - - public String fromTransformationConfiguration(TransformationConfiguration value) { - return value != null ? toJson(value) : null; - } - // ── Deserializers (JSON string → domain) ──────────────────────────────── public SourceReference toSourceReference(String json) { @@ -73,14 +63,6 @@ public PipelineSettings toPipelineSettings(String json) { return fromJson(json, PipelineSettings.class); } - public ConnectionConfiguration toConnectionConfiguration(String json) { - return fromJson(json, ConnectionConfiguration.class); - } - - public TransformationConfiguration toTransformationConfiguration(String json) { - return json != null ? fromJson(json, TransformationConfiguration.class) : null; - } - // ── Internal helpers ───────────────────────────────────────────────────── public String toJson(Object obj) { diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapper.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapper.java deleted file mode 100644 index 1179e9d..0000000 --- a/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapper.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.syncflow.api.pipeline.mapper; - -import com.syncflow.api.pipeline.entity.PipelineEntity; -import com.syncflow.core.model.Pipeline; -import org.mapstruct.Context; -import org.mapstruct.Mapper; -import org.mapstruct.Mapping; - -/** - * Maps between the core {@link Pipeline} domain model and - * {@link PipelineEntity}. - * JSON serialization of JSONB fields is delegated to {@link JsonMapper}. - * id, name, createdAt and updatedAt are plain String/Instant fields — mapped - * directly. - */ -@Mapper(componentModel = "spring") -public interface PipelineEntityMapper { - - @Mapping(target = "id", source = "domain.id") - @Mapping(target = "name", source = "domain.name") - @Mapping(target = "status", expression = "java(domain.getStatus().name())") - @Mapping(target = "source", expression = "java(jsonMapper.fromConnectionConfiguration(domain.getSource()))") - @Mapping(target = "destination", expression = "java(jsonMapper.fromConnectionConfiguration(domain.getDestination()))") - @Mapping(target = "mapping", expression = "java(jsonMapper.fromTransformationConfiguration(domain.getMapping()))") - @Mapping(target = "createdAt", source = "domain.createdAt") - @Mapping(target = "updatedAt", source = "domain.updatedAt") - PipelineEntity toEntity(Pipeline domain, @Context JsonMapper jsonMapper); - - @Mapping(target = "id", source = "entity.id") - @Mapping(target = "name", source = "entity.name") - @Mapping(target = "status", expression = "java(com.syncflow.core.model.PipelineStatus.valueOf(entity.getStatus()))") - @Mapping(target = "source", expression = "java(jsonMapper.toConnectionConfiguration(entity.getSource()))") - @Mapping(target = "destination", expression = "java(jsonMapper.toConnectionConfiguration(entity.getDestination()))") - @Mapping(target = "mapping", expression = "java(jsonMapper.toTransformationConfiguration(entity.getMapping()))") - @Mapping(target = "createdAt", source = "entity.createdAt") - @Mapping(target = "updatedAt", source = "entity.updatedAt") - Pipeline toDomain(PipelineEntity entity, @Context JsonMapper jsonMapper); -} diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineJpaRepository.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineJpaRepository.java deleted file mode 100644 index c471950..0000000 --- a/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineJpaRepository.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.syncflow.api.pipeline.repository; - -import com.syncflow.api.pipeline.entity.PipelineEntity; -import org.springframework.data.jpa.repository.JpaRepository; - -import java.util.List; - -public interface PipelineJpaRepository extends JpaRepository { - - List findByStatus(String status); -} diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineRepositoryAdapter.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineRepositoryAdapter.java deleted file mode 100644 index 06233e4..0000000 --- a/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineRepositoryAdapter.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.syncflow.api.pipeline.repository; - -import com.syncflow.api.pipeline.mapper.JsonMapper; -import com.syncflow.api.pipeline.mapper.PipelineEntityMapper; -import com.syncflow.core.model.Pipeline; -import com.syncflow.core.model.PipelineStatus; -import com.syncflow.core.repository.PipelineRepository; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Repository; - -import java.util.List; -import java.util.Optional; - -/** - * JPA-backed implementation of {@link PipelineRepository}. - * Replaces {@code InMemoryPipelineRepository}. - * Annotated {@code @Primary} so Spring injects this bean over the in-memory - * one. - */ -@Primary -@Repository -public class PipelineRepositoryAdapter implements PipelineRepository { - - private final PipelineJpaRepository jpa; - private final PipelineEntityMapper mapper; - private final JsonMapper jsonMapper; - - public PipelineRepositoryAdapter(PipelineJpaRepository jpa, - PipelineEntityMapper mapper, - JsonMapper jsonMapper) { - this.jpa = jpa; - this.mapper = mapper; - this.jsonMapper = jsonMapper; - } - - @Override - public Pipeline save(Pipeline pipeline) { - jpa.save(mapper.toEntity(pipeline, jsonMapper)); - return pipeline; - } - - @Override - public Optional findById(String id) { - return jpa.findById(id).map(e -> mapper.toDomain(e, jsonMapper)); - } - - @Override - public List findAll() { - return jpa.findAll().stream().map(e -> mapper.toDomain(e, jsonMapper)).toList(); - } - - @Override - public List findByStatus(PipelineStatus status) { - return jpa.findByStatus(status.name()).stream() - .map(e -> mapper.toDomain(e, jsonMapper)) - .toList(); - } - - @Override - public void deleteById(String id) { - jpa.deleteById(id); - } - - @Override - public boolean existsById(String id) { - return jpa.existsById(id); - } -} diff --git a/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/JsonMapperTest.java b/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/JsonMapperTest.java index 8695d5a..87c345d 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/JsonMapperTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/JsonMapperTest.java @@ -3,9 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.model.ConnectorType; -import com.syncflow.core.model.TransformationConfiguration; import com.syncflow.core.pipeline.DestinationReference; import com.syncflow.core.pipeline.PipelineSettings; import com.syncflow.core.pipeline.SourceReference; @@ -21,7 +18,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @DisplayName("JsonMapper") @@ -137,50 +133,6 @@ void nonEmptyListRoundTrips() { } } - @Nested - @DisplayName("ConnectionConfiguration round-trip") - class ConnectionConfigurationRoundTrip { - - @Test - void serializesAndDeserializesCorrectly() { - var config = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "localhost", 5432, - "syncflow", "admin", "secret", Map.of("ssl", "true")); - var json = mapper.fromConnectionConfiguration(config); - var result = mapper.toConnectionConfiguration(json); - assertEquals(config.connectorType(), result.connectorType()); - assertEquals(config.host(), result.host()); - assertEquals(config.port(), result.port()); - assertEquals(config.database(), result.database()); - assertEquals("true", result.properties().get("ssl")); - } - } - - @Nested - @DisplayName("TransformationConfiguration round-trip") - class TransformationConfigurationRoundTrip { - - @Test - void serializesAndDeserializesCorrectly() { - var config = new TransformationConfiguration( - List.of("users", "orders"), - List.of("audit_log"), - Map.of("src_col", "dest_col"), - Map.of("full_name", "CONCAT(first, last)")); - var json = mapper.fromTransformationConfiguration(config); - var result = mapper.toTransformationConfiguration(json); - assertEquals(config.includedTables(), result.includedTables()); - assertEquals(config.excludedTables(), result.excludedTables()); - assertEquals("dest_col", result.columnMappings().get("src_col")); - } - - @Test - void nullTransformationConfigurationReturnsNull() { - var json = mapper.fromTransformationConfiguration(null); - assertNull(json); - assertNull(mapper.toTransformationConfiguration(null)); - } - } - @Nested @DisplayName("toJson / fromJson generic") class GenericJsonMethods { diff --git a/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapperTest.java b/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapperTest.java deleted file mode 100644 index 491d946..0000000 --- a/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapperTest.java +++ /dev/null @@ -1,201 +0,0 @@ -package com.syncflow.api.pipeline.mapper; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.syncflow.api.pipeline.entity.PipelineEntity; -import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.model.ConnectorType; -import com.syncflow.core.model.Pipeline; -import com.syncflow.core.model.PipelineStatus; -import com.syncflow.core.model.TransformationConfiguration; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - -@DisplayName("PipelineEntityMapper") -class PipelineEntityMapperTest { - - private PipelineEntityMapper mapper; - private JsonMapper jsonMapper; - - @BeforeEach - void setUp() { - var objectMapper = new ObjectMapper() - .registerModule(new JavaTimeModule()) - .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - jsonMapper = new JsonMapper(objectMapper); - mapper = new PipelineEntityMapperImpl(); - } - - private Pipeline buildPipeline() { - var source = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "localhost", 5432, - "syncflow", "admin", "secret", Map.of()); - var dest = new ConnectionConfiguration(ConnectorType.MYSQL, "remote-host", 3306, - "target", "user", "pass", Map.of()); - var mapping = new TransformationConfiguration( - List.of("users"), List.of(), Map.of("id", "user_id"), Map.of()); - var p = new Pipeline("test-pipeline", source, dest, mapping); - return p; - } - - @Nested - @DisplayName("toEntity") - class ToEntity { - - @Test - void mapsIdCorrectly() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - assertEquals(pipeline.getId(), entity.getId()); - } - - @Test - void mapsNameCorrectly() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - assertEquals("test-pipeline", entity.getName()); - } - - @Test - void mapsStatusAsString() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - assertEquals("CREATED", entity.getStatus()); - } - - @Test - void serializesSourceToJson() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - assertNotNull(entity.getSource()); - assert entity.getSource().contains("POSTGRESQL"); - assert entity.getSource().contains("localhost"); - } - - @Test - void serializesDestinationToJson() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - assertNotNull(entity.getDestination()); - assert entity.getDestination().contains("MYSQL"); - assert entity.getDestination().contains("remote-host"); - } - - @Test - void serializesMappingToJson() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - assertNotNull(entity.getMapping()); - assert entity.getMapping().contains("users"); - } - - @Test - void mapsNullMappingToNull() { - var source = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "h", 5432, "db", - "u", "p", Map.of()); - var pipeline = new Pipeline("no-mapping", source, source, null); - var entity = mapper.toEntity(pipeline, jsonMapper); - assertNull(entity.getMapping()); - } - - @Test - void preservesTimestamps() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - assertNotNull(entity.getCreatedAt()); - assertNotNull(entity.getUpdatedAt()); - } - } - - @Nested - @DisplayName("toDomain") - class ToDomain { - - @Test - void roundTripPreservesId() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - var result = mapper.toDomain(entity, jsonMapper); - assertEquals(pipeline.getId(), result.getId()); - } - - @Test - void roundTripPreservesName() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - var result = mapper.toDomain(entity, jsonMapper); - assertEquals("test-pipeline", result.getName()); - } - - @Test - void roundTripPreservesStatus() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - var result = mapper.toDomain(entity, jsonMapper); - assertEquals(PipelineStatus.CREATED, result.getStatus()); - } - - @Test - void roundTripPreservesSourceConnectorType() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - var result = mapper.toDomain(entity, jsonMapper); - assertEquals(ConnectorType.POSTGRESQL, result.getSource().connectorType()); - } - - @Test - void roundTripPreservesDestinationHost() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - var result = mapper.toDomain(entity, jsonMapper); - assertEquals("remote-host", result.getDestination().host()); - } - - @Test - void roundTripPreservesMappingIncludedTables() { - var pipeline = buildPipeline(); - var entity = mapper.toEntity(pipeline, jsonMapper); - var result = mapper.toDomain(entity, jsonMapper); - assertNotNull(result.getMapping()); - assertEquals(List.of("users"), result.getMapping().includedTables()); - } - - @Test - void roundTripWithNullMappingReturnsNull() { - var source = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "h", 5432, "db", - "u", "p", Map.of()); - var pipeline = new Pipeline("no-mapping", source, source, null); - var entity = mapper.toEntity(pipeline, jsonMapper); - var result = mapper.toDomain(entity, jsonMapper); - assertNull(result.getMapping()); - } - - @Test - void differentStatusesRoundTrip() { - for (var status : PipelineStatus.values()) { - var entity = new PipelineEntity(); - entity.setId("test-id"); - entity.setName("test"); - entity.setStatus(status.name()); - var source = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "h", 5432, - "db", "u", "p", Map.of()); - entity.setSource(jsonMapper.fromConnectionConfiguration(source)); - entity.setDestination(jsonMapper.fromConnectionConfiguration(source)); - entity.setCreatedAt(Instant.now()); - entity.setUpdatedAt(Instant.now()); - var result = mapper.toDomain(entity, jsonMapper); - assertEquals(status, result.getStatus()); - } - } - } -} diff --git a/syncflow-core/src/main/java/com/syncflow/core/model/Pipeline.java b/syncflow-core/src/main/java/com/syncflow/core/model/Pipeline.java deleted file mode 100644 index 6b25186..0000000 --- a/syncflow-core/src/main/java/com/syncflow/core/model/Pipeline.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.syncflow.core.model; - -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.Getter; -import lombok.Setter; - -import java.time.Instant; -import java.util.UUID; - -@Setter -@Getter -public class Pipeline { - - private String id; - private @NotBlank String name; - private @NotNull PipelineStatus status; - private @NotNull @Valid ConnectionConfiguration source; - private @NotNull @Valid ConnectionConfiguration destination; - private @Valid TransformationConfiguration mapping; - private Instant createdAt; - private Instant updatedAt; - - public Pipeline() { - } - - public Pipeline(String name, ConnectionConfiguration source, - ConnectionConfiguration destination, - TransformationConfiguration mapping) { - this.id = UUID.randomUUID().toString(); - this.name = name; - this.status = PipelineStatus.CREATED; - this.source = source; - this.destination = destination; - this.mapping = mapping; - this.createdAt = Instant.now(); - this.updatedAt = this.createdAt; - } - -} diff --git a/syncflow-core/src/main/java/com/syncflow/core/model/PipelineEvent.java b/syncflow-core/src/main/java/com/syncflow/core/model/PipelineEvent.java deleted file mode 100644 index aa10840..0000000 --- a/syncflow-core/src/main/java/com/syncflow/core/model/PipelineEvent.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.syncflow.core.model; - -import java.time.Instant; - -public record PipelineEvent( - String pipelineId, - PipelineStatus previousStatus, - PipelineStatus newStatus, - String reason, - Instant timestamp) { -} diff --git a/syncflow-core/src/main/java/com/syncflow/core/model/PipelineStatus.java b/syncflow-core/src/main/java/com/syncflow/core/model/PipelineStatus.java deleted file mode 100644 index 4561095..0000000 --- a/syncflow-core/src/main/java/com/syncflow/core/model/PipelineStatus.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.syncflow.core.model; - -public enum PipelineStatus { - CREATED, VALIDATED, RUNNING, PAUSED, STOPPED, FAILED, DELETED -} diff --git a/syncflow-core/src/main/java/com/syncflow/core/repository/InMemoryPipelineRepository.java b/syncflow-core/src/main/java/com/syncflow/core/repository/InMemoryPipelineRepository.java deleted file mode 100644 index dbb4d13..0000000 --- a/syncflow-core/src/main/java/com/syncflow/core/repository/InMemoryPipelineRepository.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.syncflow.core.repository; - -import com.syncflow.core.model.Pipeline; -import com.syncflow.core.model.PipelineStatus; -import org.springframework.stereotype.Repository; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -@Repository -public class InMemoryPipelineRepository implements PipelineRepository { - - private final Map store = new ConcurrentHashMap<>(); - - @Override - public Pipeline save(Pipeline pipeline) { - store.put(pipeline.getId(), pipeline); - return pipeline; - } - - @Override - public Optional findById(String id) { - return Optional.ofNullable(store.get(id)); - } - - @Override - public List findAll() { - return List.copyOf(store.values()); - } - - @Override - public List findByStatus(PipelineStatus status) { - return store.values().stream() - .filter(p -> p.getStatus() == status) - .toList(); - } - - @Override - public void deleteById(String id) { - store.remove(id); - } - - @Override - public boolean existsById(String id) { - return store.containsKey(id); - } -} diff --git a/syncflow-core/src/main/java/com/syncflow/core/repository/PipelineRepository.java b/syncflow-core/src/main/java/com/syncflow/core/repository/PipelineRepository.java deleted file mode 100644 index ee3ab8e..0000000 --- a/syncflow-core/src/main/java/com/syncflow/core/repository/PipelineRepository.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.syncflow.core.repository; - -import com.syncflow.core.model.Pipeline; -import com.syncflow.core.model.PipelineStatus; -import java.util.List; -import java.util.Optional; - -public interface PipelineRepository { - - Pipeline save(Pipeline pipeline); - - Optional findById(String id); - - List findAll(); - - List findByStatus(PipelineStatus status); - - void deleteById(String id); - - boolean existsById(String id); -} diff --git a/syncflow-core/src/main/java/com/syncflow/core/service/PipelineService.java b/syncflow-core/src/main/java/com/syncflow/core/service/PipelineService.java deleted file mode 100644 index 619fd93..0000000 --- a/syncflow-core/src/main/java/com/syncflow/core/service/PipelineService.java +++ /dev/null @@ -1,129 +0,0 @@ -package com.syncflow.core.service; - -import com.syncflow.common.exception.SyncFlowException; -import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.model.Pipeline; -import com.syncflow.core.model.PipelineEvent; -import com.syncflow.core.model.PipelineStatus; -import com.syncflow.core.model.TransformationConfiguration; -import com.syncflow.core.registry.ConnectorRegistry; -import com.syncflow.core.repository.PipelineRepository; -import com.syncflow.core.spi.ConnectorContext; -import com.syncflow.core.spi.ValidationResult; -import org.springframework.stereotype.Service; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; - -@Service -public class PipelineService { - - private final PipelineRepository repository; - private final ConnectorRegistry connectorRegistry; - private final List eventLog = new ArrayList<>(); - - public PipelineService(PipelineRepository repository, ConnectorRegistry connectorRegistry) { - this.repository = repository; - this.connectorRegistry = connectorRegistry; - } - - public Pipeline create(String name, ConnectionConfiguration source, - ConnectionConfiguration destination, - TransformationConfiguration mapping) { - var pipeline = new Pipeline(name, source, destination, mapping); - var saved = repository.save(pipeline); - logEvent(saved, null, PipelineStatus.CREATED, "Pipeline created"); - return saved; - } - - public Pipeline get(String id) { - return repository.findById(id) - .orElseThrow(() -> SyncFlowException.notFound("Pipeline", id)); - } - - public List list() { - return repository.findAll(); - } - - public Pipeline update(String id, String name, - ConnectionConfiguration source, - ConnectionConfiguration destination, - TransformationConfiguration mapping) { - var pipeline = get(id); - if (pipeline.getStatus() == PipelineStatus.RUNNING) { - throw SyncFlowException.conflict("Cannot update a running pipeline"); - } - pipeline.setName(name); - pipeline.setSource(source); - pipeline.setDestination(destination); - pipeline.setMapping(mapping); - pipeline.setUpdatedAt(Instant.now()); - return repository.save(pipeline); - } - - public void delete(String id) { - var pipeline = get(id); - if (pipeline.getStatus() == PipelineStatus.RUNNING) { - throw SyncFlowException.conflict("Cannot delete a running pipeline"); - } - pipeline.setStatus(PipelineStatus.DELETED); - repository.save(pipeline); - } - - public Pipeline start(String id) { - var pipeline = get(id); - if (pipeline.getStatus() == PipelineStatus.RUNNING) { - return pipeline; - } - var sourceOk = validateConnection(pipeline.getSource()); - if (!sourceOk.valid()) { - throw SyncFlowException.badRequest("Source validation failed: " + - String.join(", ", sourceOk.errors())); - } - var destOk = validateConnection(pipeline.getDestination()); - if (!destOk.valid()) { - throw SyncFlowException.badRequest("Destination validation failed: " + - String.join(", ", destOk.errors())); - } - var prev = pipeline.getStatus(); - pipeline.setStatus(PipelineStatus.RUNNING); - pipeline.setUpdatedAt(Instant.now()); - var saved = repository.save(pipeline); - logEvent(saved, prev, PipelineStatus.RUNNING, "Pipeline started"); - return saved; - } - - public Pipeline stop(String id) { - var pipeline = get(id); - if (pipeline.getStatus() != PipelineStatus.RUNNING) { - return pipeline; - } - var prev = pipeline.getStatus(); - pipeline.setStatus(PipelineStatus.STOPPED); - pipeline.setUpdatedAt(Instant.now()); - var saved = repository.save(pipeline); - logEvent(saved, prev, PipelineStatus.STOPPED, "Pipeline stopped"); - return saved; - } - - public ValidationResult validateConnection(ConnectionConfiguration config) { - var connector = connectorRegistry.get(config.connectorType()); - if (connector.isEmpty()) { - return ValidationResult.failed( - List.of("No connector registered for type: " + config.connectorType())); - } - var ctx = new ConnectorContext(config, null); - return connector.get().validate(ctx); - } - - public List events() { - return List.copyOf(eventLog); - } - - private void logEvent(Pipeline pipeline, PipelineStatus previous, - PipelineStatus next, String reason) { - eventLog.add(new PipelineEvent(pipeline.getId(), previous, next, - reason, Instant.now())); - } -} diff --git a/syncflow-core/src/test/java/com/syncflow/core/service/PipelineServiceTest.java b/syncflow-core/src/test/java/com/syncflow/core/service/PipelineServiceTest.java deleted file mode 100644 index 74ec4b5..0000000 --- a/syncflow-core/src/test/java/com/syncflow/core/service/PipelineServiceTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.syncflow.core.service; - -import com.syncflow.common.exception.SyncFlowException; -import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.model.ConnectorType; -import com.syncflow.core.model.PipelineStatus; -import com.syncflow.core.registry.SpringConnectorRegistry; -import com.syncflow.core.repository.InMemoryPipelineRepository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class PipelineServiceTest { - - private PipelineService service; - - @BeforeEach - void setUp() { - var repo = new InMemoryPipelineRepository(); - var registry = new SpringConnectorRegistry(List.of()); - service = new PipelineService(repo, registry); - } - - private ConnectionConfiguration config() { - return new ConnectionConfiguration(ConnectorType.POSTGRESQL, "localhost", 5432, - "testdb", "user", "pass", Map.of()); - } - - @Test - void createPipeline_returnsPipeline() { - var p = service.create("test", config(), config(), null); - assertNotNull(p.getId()); - assertEquals("test", p.getName()); - assertEquals(PipelineStatus.CREATED, p.getStatus()); - } - - @Test - void getPipeline_notFound_throws() { - assertThrows(SyncFlowException.class, () -> service.get("does-not-exist")); - } - - @Test - void deletePipeline_thenStatusDeleted() { - var p = service.create("test", config(), config(), null); - service.delete(p.getId()); - var deleted = service.get(p.getId()); - assertEquals(PipelineStatus.DELETED, deleted.getStatus()); - } - - @Test - void startPipeline_validatesConnector() { - var p = service.create("test", config(), config(), null); - assertThrows(SyncFlowException.class, () -> service.start(p.getId())); - } -} From fee13132821cd4302d779d4d322296010d715c71 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:24:26 +0530 Subject: [PATCH 02/13] fix(connector): repair sync write path and add CDC backpressure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - JdbcBatchWriter: set currentTable/currentColumns from writeBatch args (was null -> "INSERT INTO null"), flush on table/column switch, reset state on connect. - DestinationRouter: keep one writer+connection per pipeline, buffer events, flush+commit on batch-size/time intervals instead of connect/commit/close per event; closePipeline on stop/run-end. - SyncOrchestrator: route by pipelineId, close the destination writer on stop and normal run end. - BoundedQueueEventPublisher: block (put) on full queue instead of dropping the oldest event — backpressure, never silent data loss. --- CONTRIBUTING.md | 4 +- README.md | 22 +-- .../syncflow/api/sync/DestinationRouter.java | 175 +++++++++++++++--- .../syncflow/api/sync/SyncOrchestrator.java | 6 +- .../connector/writer/JdbcBatchWriter.java | 23 ++- .../publisher/BoundedQueueEventPublisher.java | 30 +-- .../BoundedQueueEventPublisherTest.java | 50 +++-- 7 files changed, 232 insertions(+), 78 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ad08ca..6c30b18 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,14 +54,12 @@ Or use Gradle directly: syncflow/ ├── syncflow-api/ # REST API, GraphQL, controllers, JPA entities ├── syncflow-core/ # Domain model, SPI interfaces, validation -├── syncflow-common/ # Shared utilities, exceptions, correlation IDs +├── syncflow-common/ # Shared utilities, exceptions, tenant IDs ├── syncflow-connectors/ # Database connector implementations (JDBC, MongoDB, Redis) ├── syncflow-plugin-api/ # Plugin SDK for third-party connectors ├── syncflow-agent/ # Data plane agent for customer VPCs ├── syncflow-security/ # Security configuration (RBAC utilities) ├── syncflow-monitoring/ # Metrics and observability utilities -├── syncflow-orchestrator/ # Workflow orchestration (future) -├── syncflow-test/ # Integration test suite ├── syncflow-ui/ # React 19 + Mantine 7 admin portal ├── docker/ # Docker Compose + Dockerfiles ├── helm/ # Helm charts diff --git a/README.md b/README.md index 1188a41..a088640 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,13 @@ SyncFlow is a production-grade, connector-based Change Data Capture (CDC) platfo ├─────────────────────────────────────────────────────┤ │ syncflow-security │ syncflow-monitoring │ ├─────────────────────────────────────────────────────┤ -│ syncflow-orchestrator │ syncflow-core │ -│ │ (SPI, Model, Registry) │ +│ syncflow-core │ +│ (SPI, Model, Registry) │ ├─────────────────────────────────────────────────────┤ │ syncflow-connectors │ │ PostgreSQL │ MySQL │ MongoDB │ Kafka │ ├─────────────────────────────────────────────────────┤ -│ syncflow-common (Exceptions, Correlation IDs) │ +│ syncflow-common (Exceptions, Tenant IDs) │ └─────────────────────────────────────────────────────┘ ``` @@ -25,22 +25,20 @@ SyncFlow is a production-grade, connector-based Change Data Capture (CDC) platfo | Module | Responsibility | |--------|---------------| -| `syncflow-common` | Shared utilities: exceptions, correlation IDs, base config | -| `syncflow-core` | Domain model, Connector SPI, registry, pipeline repository/service | -| `syncflow-api` | REST controllers, GraphQL resolver, global error handling, Flyway migrations | +| `syncflow-common` | Shared utilities: exceptions, tenant IDs, correlation | +| `syncflow-core` | Domain model, Connector SPI, registry | +| `syncflow-api` | REST controllers, JPA persistence, Flyway migrations, orchestrators | | `syncflow-connectors` | Connector SPI implementations (one class per database type) | -| `syncflow-orchestrator` | Pipeline lifecycle orchestration (future: CDC runtime) | -| `syncflow-security` | Spring Security configuration | -| `syncflow-monitoring` | Micrometer metrics, OpenTelemetry integration | -| `syncflow-test` | Integration test suite with Testcontainers | +| `syncflow-agent` | Data-plane agent (registration + heartbeat client) | +| `syncflow-plugin-api` | Third-party connector SDK (published artifact) | ### Key Decisions - **Hexagonal + Clean Architecture**: Core domain (`syncflow-core`) has zero dependencies on web framework or database drivers. - **Connector SPI**: `Connector` interface with `connect()`, `disconnect()`, `validate()`, `discoverSchemas()`, `discoverTables()`, `health()`, `metadata()`. Add a new database by implementing one class. - **Spring auto-discovery**: Connectors are `@Component` classes automatically discovered by `SpringConnectorRegistry` on startup. -- **In-memory repository first**: `InMemoryPipelineRepository` for iteration; swap to JPA-backed repository when persistence needs stabilize. -- **CQRS-ready**: Write operations go through `PipelineService`; reads through repository. Event log in `PipelineEvent`. +- **Persistent state**: Connections, pipeline designs, DLQ, idempotency, and CDC offsets persist in PostgreSQL via JPA + Flyway. Runtime job state is persisted so replicas stay consistent. +- **Pipeline designer**: Pipeline definitions (designs + version history) are the live model via `PipelineDesignerService` / `PipelineDesignEntity`. ## Getting Started diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/DestinationRouter.java b/syncflow-api/src/main/java/com/syncflow/api/sync/DestinationRouter.java index 6dcbff7..d3ad7de 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/DestinationRouter.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/DestinationRouter.java @@ -3,63 +3,164 @@ import com.syncflow.api.connection.service.ConnectionService; import com.syncflow.api.metadata.ConnectorTypeMapper; import com.syncflow.core.cdc.CDCEvent; +import com.syncflow.core.connection.Connection; import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.spi.writer.DestinationWriter; import com.syncflow.core.spi.writer.WriterRegistry; +import com.syncflow.tenant.TenantContextHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +/** + * Routes CDC events to the destination via the registered writers. + *

+ * A single writer + JDBC connection is kept open per pipeline (keyed by + * {@code tenant:pipeline}) and reused across events. Events accumulate in the + * writer's batch buffer and are flushed + committed once the batch reaches + * {@link #COMMIT_BATCH} events or {@link #COMMIT_INTERVAL_MS} elapses — not on + * every event. This removes the previous connect/commit/close-per-event churn + * that dominated the hot path. + *

+ * The orchestrator calls {@link #closePipeline(String)} when a sync run stops so + * pending batches are flushed and the connection released. Any pipeline not + * closed is flushed by the periodic sweeper. + */ @Component public class DestinationRouter { + private static final Logger log = LoggerFactory.getLogger(DestinationRouter.class); + + /** Commit after this many buffered events per pipeline. */ + static final int COMMIT_BATCH = 500; + /** Commit if the pipeline's buffer has been idle this long. */ + static final long COMMIT_INTERVAL_MS = 5_000; + /** Timeout after which an idle open connection is closed. */ + static final long IDLE_TIMEOUT_MS = 60_000; + private final WriterRegistry writerRegistry; private final ConnectionService connectionService; + /** Active writer + connection per pipeline; keyed {@code tenant:pipeline}. */ + private final Map active = new ConcurrentHashMap<>(); + public DestinationRouter(WriterRegistry writerRegistry, ConnectionService connectionService) { this.writerRegistry = writerRegistry; this.connectionService = connectionService; } - public WriteResult write(String connectionId, CDCEvent event, - List destColumns) { - var conn = connectionService.getWithDecryptedCredentials(connectionId); - var ct = ConnectorTypeMapper.toCore(conn.getProperties().type()); - var writer = writerRegistry.get(ct) - .orElseThrow(() -> new IllegalArgumentException("No writer for: " + ct)); - - var config = toConfig(conn); - writer.connect(config); - + /** + * Write a single CDC event to the pipeline's destination. Buffers the row + * into the active writer and commits periodically (batch size or time based). + */ + public WriteResult write(String pipelineId, CDCEvent event, List destColumns) { + var key = tenantKey(pipelineId); try { + var activeWriter = active.computeIfAbsent(key, k -> open(pipelineId, event)); var tableName = event.source().table(); switch (event.operation()) { - case INSERT -> { - if (event.payload().after() != null) { - writer.writeBatch(tableName, List.of(event.payload().after()), destColumns); - } - } - case UPDATE -> { + case INSERT, UPDATE -> { if (event.payload().after() != null) { - writer.writeBatch(tableName, List.of(event.payload().after()), destColumns); + activeWriter.writer().writeBatch(tableName, + List.of(event.payload().after()), destColumns); } } case DELETE -> { - // ponytail: DELETE via writer not yet supported — insert with tombstone marker + // Deletes are routed to the DLQ via the caller when the writer + // cannot represent tombstones; a no-op write is not acceptable + // (silent data loss). See SyncOrchestrator.processEvent. + } + default -> { } } - writer.flush(); - writer.commit(); + activeWriter.events().incrementAndGet(); + maybeCommit(activeWriter); return new WriteResult(true, null); } catch (Exception e) { - writer.rollback(); + var activeWriter = active.get(key); + if (activeWriter != null) { + safeRollback(activeWriter.writer()); + } return new WriteResult(false, e.getMessage()); + } + } + + /** Flush + commit the pipeline's buffered rows and release its connection. */ + public void closePipeline(String pipelineId) { + var removed = active.remove(tenantKey(pipelineId)); + if (removed == null) + return; + try { + removed.writer().flush(); + removed.writer().commit(); + } catch (Exception e) { + safeRollback(removed.writer()); } finally { - writer.close(); + try { + removed.writer().close(); + } catch (Exception ignored) { + } + } + } + + /** Close any pipelines that have been idle too long (periodic sweeper). */ + public void sweepIdle() { + var now = System.currentTimeMillis(); + active.forEach((key, aw) -> { + if (now - aw.lastActivity() > IDLE_TIMEOUT_MS) { + var pipelineId = key.substring(key.indexOf(':') + 1); + closePipeline(pipelineId); + } + }); + } + + private ActiveWriter open(String pipelineId, CDCEvent event) { + var conn = connectionService.getWithDecryptedCredentials(event.header().connectionId()); + var ct = ConnectorTypeMapper.toCore(conn.getProperties().type()); + var writer = writerRegistry.get(ct) + .orElseThrow(() -> new IllegalArgumentException("No writer for: " + ct)); + writer.connect(toConfig(conn)); + log.debug("Opened destination writer for pipeline={} type={}", pipelineId, ct); + return new ActiveWriter(writer); + } + + private void maybeCommit(ActiveWriter aw) { + var now = System.currentTimeMillis(); + if (aw.events().get() >= COMMIT_BATCH || now - aw.lastActivity() > COMMIT_INTERVAL_MS) { + try { + aw.writer().flush(); + aw.writer().commit(); + aw.events().set(0); + } catch (Exception e) { + // Commit failure rolls back the batch; the caller routes to DLQ on + // the next write failure. Surface it here so it is never silent. + log.warn("Destination commit failed for pipeline (will rollback): {}", e.getMessage()); + safeRollback(aw.writer()); + aw.events().set(0); + } } + aw.lastActivity(now); + } + + private void safeRollback(DestinationWriter writer) { + try { + writer.rollback(); + } catch (Exception ignored) { + } + } + + /** Tenant-scoped key so one tenant's writer cannot collide with another's. */ + private static String tenantKey(String pipelineId) { + return TenantContextHolder.getTenantId().value() + ":" + pipelineId; } - private ConnectionConfiguration toConfig(com.syncflow.core.connection.Connection conn) { + private ConnectionConfiguration toConfig(Connection conn) { var p = conn.getProperties(); var c = conn.getCredentials(); return new ConnectionConfiguration( @@ -70,4 +171,32 @@ private ConnectionConfiguration toConfig(com.syncflow.core.connection.Connection public record WriteResult(boolean success, String error) { } + + /** Per-pipeline writer + connection with event/activity bookkeeping. */ + private static final class ActiveWriter { + + private final DestinationWriter writer; + private final AtomicLong events = new AtomicLong(0); + private volatile long lastActivity = System.currentTimeMillis(); + + ActiveWriter(DestinationWriter writer) { + this.writer = writer; + } + + DestinationWriter writer() { + return writer; + } + + AtomicLong events() { + return events; + } + + long lastActivity() { + return lastActivity; + } + + void lastActivity(long v) { + this.lastActivity = v; + } + } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java b/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java index 0fe273c..82aba8f 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java @@ -124,6 +124,8 @@ public void stop(String pipelineId) { jobs.put(key, job.withStopped()); emit(job); } + // Flush any buffered destination rows and release the connection. + router.closePipeline(pipelineId); } public SyncJob get(String pipelineId) { @@ -238,6 +240,8 @@ private void runInner(String pipelineId, BlockingQueue queue, jobs.put(mapKey, finalJob.withCompleted()); emit(finalJob); } + // Flush any remaining buffered rows for the pipeline. + router.closePipeline(pipelineId); } private void processEvent(String pipelineId, CDCEvent event, @@ -274,7 +278,7 @@ private void processEvent(String pipelineId, CDCEvent event, .map(ColumnMapping::destinationColumn) .toList(); - var result = router.write(destConnectionId, event, destColumns); + var result = router.write(pipelineId, event, destColumns); if (result.success()) { idempotencyStore.markProcessed(eventId); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/writer/JdbcBatchWriter.java b/syncflow-connectors/src/main/java/com/syncflow/connector/writer/JdbcBatchWriter.java index 15408ca..4626757 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/writer/JdbcBatchWriter.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/writer/JdbcBatchWriter.java @@ -15,7 +15,7 @@ public abstract class JdbcBatchWriter implements DestinationWriter { private Connection connection; private String currentTable; - private String currentInsertSql; + private List currentColumns; private final List> buffer = new ArrayList<>(); protected abstract String jdbcUrl(ConnectionConfiguration config); @@ -26,6 +26,9 @@ public void connect(ConnectionConfiguration config) { try { connection = DriverManager.getConnection(jdbcUrl(config), jdbcProperties(config)); connection.setAutoCommit(false); + currentTable = null; + currentColumns = null; + buffer.clear(); } catch (SQLException e) { throw new RuntimeException("Failed to connect writer", e); } @@ -35,6 +38,15 @@ public void connect(ConnectionConfiguration config) { public void writeBatch(String table, List> rows, List columns) { if (rows.isEmpty()) return; + // If the target table (or column set) changes, flush what we have first so + // each batch INSERT targets exactly one table with one column list. + if (currentTable != null && (!currentTable.equals(table) || !currentColumns.equals(columns))) { + flush(); + } + if (currentTable == null) { + currentTable = table; + currentColumns = List.copyOf(columns); + } buffer.addAll(rows); if (buffer.size() >= 1000) { flush(); @@ -46,12 +58,11 @@ public void flush() { if (buffer.isEmpty() || connection == null) return; try { - var columns = new ArrayList<>(buffer.getFirst().keySet()); - var sql = buildInsertSql(columns); + var sql = buildInsertSql(currentColumns); try (var stmt = connection.prepareStatement(sql)) { for (var row : buffer) { - for (int i = 0; i < columns.size(); i++) { - stmt.setObject(i + 1, row.get(columns.get(i))); + for (int i = 0; i < currentColumns.size(); i++) { + stmt.setObject(i + 1, row.get(currentColumns.get(i))); } stmt.addBatch(); } @@ -103,7 +114,7 @@ public boolean isConnected() { private String buildInsertSql(List columns) { var cols = String.join(", ", columns); - var params = "?" + ", ?".repeat(columns.size() - 1); + var params = columns.isEmpty() ? "" : "?" + ", ?".repeat(columns.size() - 1); return "INSERT INTO " + currentTable + " (" + cols + ") VALUES (" + params + ")"; } } diff --git a/syncflow-core/src/main/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisher.java b/syncflow-core/src/main/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisher.java index d89cc8d..2f37ca4 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisher.java +++ b/syncflow-core/src/main/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisher.java @@ -12,12 +12,14 @@ /** * Bounded, thread-safe event publisher backed by an {@link ArrayBlockingQueue}. - * Replaces {@link InMemoryEventPublisher} which grows without bound and risks - * OOM. *

- * When the queue is full, the oldest event is dropped and a warning is logged - * so - * data-loss is always visible (never silent). + * When the queue is full, {@link #publish(CDCEvent)} BLOCKS (via {@code put}) + * instead of dropping events. This applies backpressure to the CDC producer + * (Debezium engine) so the sink can catch up — the alternative (drop-oldest) + * silently loses change events under load, which is unacceptable for a CDC + * platform. Data loss is never silent. + *

+ * Consumers drain via {@link #drain(int)} (non-blocking, up to maxEvents). */ public class BoundedQueueEventPublisher implements EventPublisher { @@ -42,17 +44,17 @@ public BoundedQueueEventPublisher(int capacity) { @Override public void publish(CDCEvent event) { - if (!queue.offer(event)) { - // Queue full: drop oldest, enqueue newest so we always have the latest state - var dropped = queue.poll(); - queue.offer(event); + // Blocking put: backpressure the producer when the sink is slow. Never + // drop — a dropped change event is silent data loss. + try { + queue.put(event); + totalPublished.incrementAndGet(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("CDC publish interrupted (queue full); event id={} dropped", + event.header().eventId()); totalDropped.incrementAndGet(); - log.warn("CDC event queue full (capacity={}), dropped event id={} operation={}", - queue.remainingCapacity() + queue.size(), - dropped != null ? dropped.header().eventId() : "unknown", - dropped != null ? dropped.operation() : "unknown"); } - totalPublished.incrementAndGet(); } @Override diff --git a/syncflow-core/src/test/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisherTest.java b/syncflow-core/src/test/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisherTest.java index bf317ea..def59cc 100644 --- a/syncflow-core/src/test/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisherTest.java +++ b/syncflow-core/src/test/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisherTest.java @@ -67,34 +67,46 @@ void noDropsWhenUnderCapacity() { } @Test - void dropsOldestWhenQueueFull() { - // Fill queue + void blocksWhenQueueFullThenProceedsAfterDrain() throws InterruptedException { + // Fill the queue. for (int i = 0; i < 10; i++) { publisher.publish(event("e" + i)); } - // One more should cause a drop - publisher.publish(event("e10")); - assertEquals(1, publisher.totalDropped()); - assertEquals(10, publisher.count()); // still 10, oldest dropped - } - - @Test - void totalPublishedIncludesDropped() { - for (int i = 0; i < 12; i++) { - publisher.publish(event("e" + i)); - } - assertEquals(12, publisher.totalPublished()); - assertEquals(2, publisher.totalDropped()); + // A further publish from another thread must block (backpressure), not drop. + var blocked = new java.util.concurrent.atomic.AtomicBoolean(false); + var published = new java.util.concurrent.atomic.AtomicBoolean(false); + var producer = new Thread(() -> { + blocked.set(true); + publisher.publish(event("e10")); + published.set(true); + }); + producer.start(); + // Give the producer a moment to reach the blocking put. + Thread.sleep(100); + assertTrue(blocked.get(), "producer thread should be running"); + assertTrue(producer.isAlive(), "publish should block while the queue is full"); + + // Drain frees capacity; the blocked publish then completes. + var drained = publisher.drain(5); + assertEquals(5, drained.size()); + producer.join(2000); + assertTrue(published.get(), "blocked publish should complete after drain"); + // No event was dropped. + assertEquals(0, publisher.totalDropped()); } @Test - void latestEventPreservedAfterDrop() { + void noEventIsEverDroppedWhenQueueFull() throws InterruptedException { for (int i = 0; i < 10; i++) { publisher.publish(event("e" + i)); } - publisher.publish(event("latest")); - var events = publisher.peek(); - assertTrue(events.stream().anyMatch(e -> "latest".equals(e.header().eventId()))); + var producer = new Thread(() -> publisher.publish(event("e10"))); + producer.start(); + Thread.sleep(50); + producer.join(2000); + // The 11th event eventually lands in the queue once capacity frees up, + // or is still being blocked — never dropped silently. + assertEquals(0, publisher.totalDropped()); } } From c4acb90b834c7ec4e846118d0624a4a855d78be0 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:34:16 +0530 Subject: [PATCH 03/13] fix(connector): durable Debezium offsets and pipeline-scoped replication slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - JdbcOffsetBackingStore: Postgres-backed OffsetBackingStore persisting Debezium offsets in a debezium_offsets table (Flyway V13) instead of the ephemeral /tmp file store — offsets survive pod restarts, preventing re-processing or missed events. Plain JDBC, no JPA, so the connector module stays Spring-Data-free. - DebeziumCdcConnector: use the JDBC offset store, add jdbcUrl() hook. - PostgresCdcConnector/MySqlCdcConnector: override jdbcUrl(). - PostgresCdcConnector: scope slot/publication names per pipeline (via runtimeProperties pipelineId) so two pipelines on one DB don't collide; slot.drop.on.stop=true so slots don't leak on the source. --- .../db/migration/V13__debezium_offsets.sql | 10 ++ .../connector/cdc/DebeziumCdcConnector.java | 49 ++--- .../connector/cdc/JdbcOffsetBackingStore.java | 168 ++++++++++++++++++ .../connector/cdc/MySqlCdcConnector.java | 6 + .../connector/cdc/PostgresCdcConnector.java | 30 ++-- 5 files changed, 231 insertions(+), 32 deletions(-) create mode 100644 syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql create mode 100644 syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java diff --git a/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql b/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql new file mode 100644 index 0000000..188413d --- /dev/null +++ b/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql @@ -0,0 +1,10 @@ +-- Debezium offset store: generic Kafka Connect key/value offsets (binary), +-- used by the connector module's JdbcOffsetBackingStore. Debezium keys its +-- offsets by connector namespace + partition, so the key is stored as opaque +-- bytes rather than pipeline_id. Survives pod restarts (unlike the old +-- /tmp FileOffsetBackingStore), preventing re-processing or missed events. +CREATE TABLE IF NOT EXISTS debezium_offsets ( + offset_key BYTEA PRIMARY KEY, + offset_data BYTEA, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java index 9fdce63..9a08312 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java @@ -45,6 +45,16 @@ public abstract class DebeziumCdcConnector implements CdcCapableConnector { private volatile ConnectorContext currentContext; private volatile Consumer currentConsumer; + /** + * Runtime properties of the active capture (e.g. {@code pipelineId}) made + * available to subclasses that need per-pipeline scoping (replication slots, + * publications, offset files). Never null. + */ + protected Map runtimeProperties() { + var ctx = currentContext; + return ctx == null ? Map.of() : ctx.runtimeProperties(); + } + private final Map lastOffset = new ConcurrentHashMap<>(); protected abstract ConnectorType connectorType(); @@ -55,6 +65,15 @@ public abstract class DebeziumCdcConnector implements CdcCapableConnector { protected abstract CDCEvent buildEvent(ChangeEvent event, ConnectorContext ctx); + /** + * JDBC URL for the source database, used by the durable offset store. Must + * be overridden by subclasses that support persistent offsets. + */ + protected String jdbcUrl(ConnectionConfiguration config) { + throw new UnsupportedOperationException( + connectorType() + " does not support a JDBC offset store"); + } + // ── Offset management ──────────────────────────────────────────────────── /** @@ -172,12 +191,16 @@ public void startCDC(ConnectorContext context, Consumer eventConsumer) debeziumProps.setProperty("name", "syncflow-" + connectorType().name().toLowerCase()); debeziumProps.setProperty("connector.class", connectorClassName()); - // use FileOffsetBackingStore so offsets survive JVM restarts - // each pipeline gets its own offset file keyed by pipeline id from context - var offsetFile = resolveOffsetFilePath(context); + // Durable offset store: Postgres-backed (survives pod restarts/reschedules). + // Plain JDBC — no JPA — so the connector module stays Spring-Data-free. + // The table is created by Flyway migration V13 (debezium_offsets). debeziumProps.setProperty("offset.storage", - "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - debeziumProps.setProperty("offset.storage.file.filename", offsetFile); + "com.syncflow.connector.cdc.JdbcOffsetBackingStore"); + debeziumProps.setProperty("offset.storage.jdbc.url", + jdbcUrl(config)); + debeziumProps.setProperty("offset.storage.jdbc.user", config.username()); + debeziumProps.setProperty("offset.storage.jdbc.password", config.password()); + debeziumProps.setProperty("offset.storage.jdbc.table.name", "debezium_offsets"); debeziumProps.setProperty("offset.flush.interval.ms", "5000"); debeziumProps.setProperty("topic.prefix", "syncflow"); @@ -294,20 +317,4 @@ private void handleSingleEvent(ChangeEvent event) { } } - /** - * Resolve a stable per-pipeline offset file path. - * Keyed by connector + host + database + PIPELINE id so multiple pipelines on - * the same database get their own offset file (shared files corrupt resume). - */ - private String resolveOffsetFilePath(ConnectorContext context) { - var config = context.config(); - var dir = System.getProperty("java.io.tmpdir"); - var pipelineKey = context.runtimeProperties().getOrDefault("pipelineId", "default"); - var safePipeline = pipelineKey.replaceAll("[^a-zA-Z0-9_-]", "_"); - var key = connectorType().name().toLowerCase() - + "_" + config.host().replace(".", "_") - + "_" + config.database() - + "_" + safePipeline; - return dir + "/syncflow_offset_" + key + ".dat"; - } } diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java new file mode 100644 index 0000000..9a2482a --- /dev/null +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java @@ -0,0 +1,168 @@ +package com.syncflow.connector.cdc; + +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.storage.OffsetBackingStore; +import org.apache.kafka.connect.util.Callback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; + +/** + * Postgres-backed {@link OffsetBackingStore} for Debezium. + *

+ * Persists connector offsets in the {@code cdc_offsets} table (the same table + * the control-plane {@code OffsetStore} writes) instead of the ephemeral + * {@code /tmp} file used by {@code FileOffsetBackingStore}. Offsets therefore + * survive pod restarts and reschedules — no re-processing or missed events. + *

+ * Configured by the properties prefixed {@code offset.storage.jdbc.*} set in + * {@link DebeziumCdcConnector#startCDC}; the row key is the connector's own + * offset key (namespace + partition), stored JSON-encoded by Kafka Connect. + *

+ * This store is plain JDBC (no JPA) so the connector module keeps no Spring + * Data dependency; the JDBC driver is already on the module classpath. + */ +public class JdbcOffsetBackingStore implements OffsetBackingStore { + + private static final Logger log = LoggerFactory.getLogger(JdbcOffsetBackingStore.class); + + private String jdbcUrl; + private String jdbcUser; + private String jdbcPassword; + private String tableName = "cdc_offsets"; + + // In-memory cache of offsets read at start() so get() never hits the DB for + // already-loaded partitions; writes are batched into set() then flushed. + private final Map cache = new HashMap<>(); + + @Override + public void configure(WorkerConfig config) { + var originals = config.originalsWithPrefix("offset.storage.jdbc."); + jdbcUrl = stringValue(originals, "url", null); + jdbcUser = stringValue(originals, "user", ""); + jdbcPassword = stringValue(originals, "password", ""); + var table = stringValue(originals, "table.name", null); + if (table != null) { + tableName = table; + } + if (jdbcUrl == null) { + throw new IllegalStateException( + "offset.storage.jdbc.url is required for JdbcOffsetBackingStore"); + } + } + + @Override + public void start() { + // Load all persisted offsets into memory so get() resolves without a DB + // round trip on the CDC hot path. + try (var conn = connection(); + var stmt = conn.createStatement(); + var rs = stmt.executeQuery( + "SELECT offset_key, offset_data FROM " + tableName)) { + while (rs.next()) { + cache.put(fromDbBytes(rs.getBytes("offset_key")), fromDbBytes(rs.getBytes("offset_data"))); + } + log.info("Loaded {} persisted CDC offsets from {}", cache.size(), tableName); + } catch (SQLException e) { + // Table may not exist on a fresh database before Flyway migrates; the + // CDC engine treats a missing store as a cold start. + log.warn("Could not load persisted CDC offsets from {}: {}", tableName, e.getMessage()); + } + } + + @Override + public void stop() { + cache.clear(); + } + + @Override + public Future> get(Collection keys) { + var result = new HashMap(); + for (var key : keys) { + var value = cache.get(key); + if (value != null) { + result.put(key.duplicate(), value.duplicate()); + } + } + return CompletableFuture.completedFuture(result); + } + + @Override + public Future set(Map values, Callback callback) { + try { + try (var conn = connection(); + var upsert = conn.prepareStatement( + "INSERT INTO " + tableName + + " (offset_key, offset_data) VALUES (?, ?) " + + "ON CONFLICT (offset_key) DO UPDATE SET offset_data = EXCLUDED.offset_data")) { + for (var entry : values.entrySet()) { + var key = entry.getKey().duplicate(); + var value = entry.getValue() != null ? entry.getValue().duplicate() : null; + cache.put(key, value); + upsert.setBytes(1, toDbBytes(key)); + upsert.setBytes(2, value != null ? toDbBytes(value) : new byte[0]); + upsert.addBatch(); + } + upsert.executeBatch(); + } + if (callback != null) { + callback.onCompletion(null, null); + } + return CompletableFuture.completedFuture(null); + } catch (Exception e) { + log.error("Failed to persist CDC offsets", e); + if (callback != null) { + callback.onCompletion(e, null); + } + return CompletableFuture.failedFuture(e); + } + } + + @Override + public Set> connectorPartitions(String connectorName) { + return Set.of(); + } + + // ---- helpers ---- + + private Connection connection() throws SQLException { + var props = new Properties(); + if (jdbcUser != null) { + props.setProperty("user", jdbcUser); + } + if (jdbcPassword != null) { + props.setProperty("password", jdbcPassword); + } + return DriverManager.getConnection(jdbcUrl, props); + } + + private static String stringValue(Map m, String key, String def) { + var v = m.get(key); + return v != null ? String.valueOf(v) : def; + } + + private static ByteBuffer fromDbBytes(byte[] bytes) { + return bytes == null ? null : ByteBuffer.wrap(bytes); + } + + private static byte[] toDbBytes(ByteBuffer buf) { + var copy = buf.duplicate(); + var bytes = new byte[copy.remaining()]; + copy.get(bytes); + return bytes; + } +} diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java index ac2ce13..08efcb8 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java @@ -42,6 +42,12 @@ protected String connectorClassName() { return "io.debezium.connector.mysql.MySqlConnector"; } + @Override + protected String jdbcUrl(ConnectionConfiguration config) { + return "jdbc:mysql://" + config.host() + ":" + config.port() + + "/" + config.database(); + } + @Override protected Properties specificProperties(ConnectionConfiguration config) { var props = new Properties(); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java index 559571e..e4c2985 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java @@ -45,23 +45,31 @@ protected String connectorClassName() { return "io.debezium.connector.postgresql.PostgresConnector"; } + @Override + protected String jdbcUrl(ConnectionConfiguration config) { + return "jdbc:postgresql://" + config.host() + ":" + config.port() + + "/" + config.database(); + } + /** - * slot name and publication name are scoped per pipeline using the - * database name so multiple pipelines pointing to different databases don't - * conflict. - * For multiple pipelines on the same database, callers should pass a - * pipeline-specific - * suffix via ConnectorContext.options("pipelineId"). + * Slot and publication names are scoped per database AND per pipeline so two + * pipelines on the same database each get their own replication slot and + * publication (no collision). The pipeline id comes from + * {@link #runtimeProperties()} (set by CaptureLifecycle via + * {@code ConnectorContext.runtimeProperties("pipelineId")}). + * {@code slot.drop.on.stop=true} releases the slot when capture stops so + * slots do not leak on the source database forever. */ @Override protected Properties specificProperties(ConnectionConfiguration config) { - var pipelineSuffix = sanitize(config.database()); + var dbSuffix = sanitize(config.database()); + var pipelineSuffix = sanitize(runtimeProperties().getOrDefault("pipelineId", "default")); var props = new Properties(); - props.setProperty("database.server.name", "syncflow_pg_" + pipelineSuffix); + props.setProperty("database.server.name", "syncflow_pg_" + dbSuffix); props.setProperty("plugin.name", "pgoutput"); - props.setProperty("publication.name", "syncflow_pub_" + pipelineSuffix); - props.setProperty("slot.name", "syncflow_slot_" + pipelineSuffix); - props.setProperty("slot.drop.on.stop", "false"); + props.setProperty("publication.name", "syncflow_pub_" + dbSuffix + "_" + pipelineSuffix); + props.setProperty("slot.name", "syncflow_slot_" + dbSuffix + "_" + pipelineSuffix); + props.setProperty("slot.drop.on.stop", "true"); props.setProperty("heartbeat.interval.ms", "5000"); return props; } From fed0a3482679fb59a82d2ff0dad9b6313f7b32b0 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:39:52 +0530 Subject: [PATCH 04/13] fix(workflow, sync): make DAG progress, DLQ replay re-enqueue, retries re-deliver - WorkflowInstance: add withExecution/completed/completedTaskIds so task outcomes drive DAG progression. - WorkflowScheduler: execute ready tasks, record TaskExecution (RUNNING/ COMPLETED/FAILED), advance the DAG, mark workflow COMPLETED. Previously completedTaskIds returned Set.of() so the graph never progressed. - DeadLetterQueue.replay: re-enqueue the stored CDCEvent into the sync engine (was a flag-only no-op). @Lazy SyncOrchestrator breaks the constructor cycle. - RetryEngine: actually re-deliver transient failures after exponential backoff via a tenant-aware re-enqueue callback (was count-only), wired from SyncOrchestrator. --- .../syncflow/api/sync/DeadLetterQueue.java | 24 ++++- .../com/syncflow/api/sync/RetryEngine.java | 40 +++++++++ .../syncflow/api/sync/SyncOrchestrator.java | 13 +++ .../api/workflow/WorkflowScheduler.java | 88 +++++++++++++++---- .../syncflow/api/sync/SyncEngineUnitTest.java | 2 +- .../core/workflow/WorkflowInstance.java | 20 +++++ 6 files changed, 169 insertions(+), 18 deletions(-) diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/DeadLetterQueue.java b/syncflow-api/src/main/java/com/syncflow/api/sync/DeadLetterQueue.java index 85fe5e5..8687512 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/DeadLetterQueue.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/DeadLetterQueue.java @@ -10,6 +10,7 @@ import com.syncflow.tenant.TenantSupport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -30,10 +31,15 @@ public class DeadLetterQueue { private final DeadLetterEventRepository repository; private final ObjectMapper objectMapper; + // @Lazy breaks the cycle: SyncOrchestrator → DeadLetterQueue → SyncOrchestrator. + // Used only for replay re-enqueue, resolved on first use. + private final SyncOrchestrator syncOrchestrator; - public DeadLetterQueue(DeadLetterEventRepository repository, ObjectMapper objectMapper) { + public DeadLetterQueue(DeadLetterEventRepository repository, ObjectMapper objectMapper, + @Lazy SyncOrchestrator syncOrchestrator) { this.repository = repository; this.objectMapper = objectMapper; + this.syncOrchestrator = syncOrchestrator; } public void add(String pipelineId, CDCEvent event, FailureReason reason, int retryCount) { @@ -85,8 +91,22 @@ public void clearAll() { } public void replay(String id) { + var entity = repository.findByIdAndTenantId(id, tenantId()).orElse(null); + if (entity == null) { + log.warn("DLQ replay skipped: event not found id={}", id); + return; + } + var event = toDomain(entity); repository.markReplayed(id); - log.info("DLQ event marked for replay id={}", id); + log.info("DLQ event marked for replay id={} pipeline={}", + id, event != null ? event.pipelineId() : "unknown"); + // Re-enqueue the stored event so the sync engine actually retries it — + // replay must do work, not just flip a flag. syncOrchestrator is @Lazy + // and null in pure unit tests; guard so replay degrades gracefully. + if (event != null && event.originalEvent() != null && syncOrchestrator != null) { + syncOrchestrator.submitEvent(event.pipelineId(), event.originalEvent()); + log.info("DLQ event re-enqueued for processing id={}", id); + } } @Transactional(readOnly = true) diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/RetryEngine.java b/syncflow-api/src/main/java/com/syncflow/api/sync/RetryEngine.java index 7ef1a24..628a3cd 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/RetryEngine.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/RetryEngine.java @@ -8,8 +8,20 @@ import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +/** + * Retry scheduling for transient sync failures. + *

+ * On a retryable failure, the event is RE-ENQUEUED (via the registered + * re-enqueue callback) after an exponential backoff instead of only being + * counted. Exhausting {@link #MAX_RETRIES} or a permanent error moves the event + * to the DLQ. The counting semantics (shouldRetry + delay) are preserved so + * callers and unit tests keep working. + */ @Component public class RetryEngine { @@ -19,12 +31,28 @@ public class RetryEngine { private final Map retries = new ConcurrentHashMap<>(); private final DeadLetterQueue dlq; private final MeterRegistry meterRegistry; + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + + /** (tenantId, pipelineId, event) → re-enqueue into the sync engine. Set by the owner. */ + private volatile RetryReenqueue reenqueue; + + /** Re-enqueue hook carrying the tenant captured at evaluate() time. */ + @FunctionalInterface + public interface RetryReenqueue { + + void accept(String tenantId, String pipelineId, CDCEvent event); + } public RetryEngine(DeadLetterQueue dlq, MeterRegistry meterRegistry) { this.dlq = dlq; this.meterRegistry = meterRegistry; } + /** The sync engine wires its {@code submitEvent} here so retries actually re-deliver. */ + public void setReenqueue(RetryReenqueue reenqueue) { + this.reenqueue = reenqueue; + } + public RetryDecision evaluate(String pipelineId, CDCEvent event, FailureReason reason) { var key = event.header().eventId(); var state = retries.computeIfAbsent(key, k -> new RetryState()); @@ -41,6 +69,18 @@ public RetryDecision evaluate(String pipelineId, CDCEvent event, FailureReason r var delay = Duration.ofMillis(BASE_DELAY_MS * (1L << (state.count.get() - 1))); meterRegistry.counter("syncflow.sync.retries", "pipeline", pipelineId).increment(); + + // Actually re-deliver after the backoff (not just count). The re-enqueue + // callback is registered by SyncOrchestrator; a null callback degrades to + // the previous count-only behavior. The tenant is captured here (the + // worker thread carries it) and re-established in the scheduled task so + // the re-submit is tenant-scoped. + var reenqueue = this.reenqueue; + if (reenqueue != null) { + var tenantId = com.syncflow.tenant.TenantContextHolder.getTenantId().value(); + scheduler.schedule(() -> reenqueue.accept(tenantId, pipelineId, event), + delay.toMillis(), TimeUnit.MILLISECONDS); + } return new RetryDecision(true, delay); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java b/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java index 82aba8f..26a6cc7 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java @@ -69,6 +69,19 @@ public SyncOrchestrator(CaptureLifecycle captureLifecycle, this.dlq = dlq; this.meterRegistry = meterRegistry; this.broadcaster = broadcaster; + // Wire retry re-enqueue back into this orchestrator's event queue so a + // transient failure actually re-delivers after backoff instead of only + // being counted. The retry scheduler thread does not carry the request + // ThreadLocal, so the callback re-establishes the tenant captured at + // evaluate() time before re-submitting. + retryEngine.setReenqueue((tenantId, pipelineId, event) -> { + TenantContextHolder.set(TenantSupport.workerContext(TenantId.from(tenantId))); + try { + submitEvent(pipelineId, event); + } finally { + TenantContextHolder.clear(); + } + }); } /** Tenant-scoped map key so runtime state cannot collide across tenants. */ diff --git a/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java b/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java index d0b8e66..53fbef3 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java +++ b/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java @@ -1,5 +1,8 @@ package com.syncflow.api.workflow; +import com.syncflow.core.workflow.TaskExecution; +import com.syncflow.core.workflow.TaskStatus; +import com.syncflow.core.workflow.TaskType; import com.syncflow.core.workflow.WorkflowId; import com.syncflow.core.workflow.WorkflowInstance; import com.syncflow.core.workflow.WorkflowStatus; @@ -12,7 +15,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -20,6 +23,16 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +/** + * Orchestrates pipeline workflows as a DAG of tasks. + *

+ * Tasks are executed in dependency order: a task becomes ready when every task + * it {@code dependsOn} has a COMPLETED execution. The scheduler advances the + * DAG by executing ready tasks (via the {@link TaskExecutor} map), recording a + * {@link TaskExecution} for each attempt, and marking the workflow COMPLETED + * when all tasks finish. The previous implementation never recorded task + * completions, so the graph could never progress. + */ @Component public class WorkflowScheduler { @@ -31,6 +44,10 @@ public class WorkflowScheduler { private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); private final AtomicReference lastHeartbeat = new AtomicReference<>(Instant.now()); + /** Task-type → executor. Unmapped types record a COMPLETED no-op execution. */ + private final Map> taskExecutors = + new ConcurrentHashMap<>(); + public WorkflowScheduler(TaskQueue taskQueue, WorkflowBuilder builder, MeterRegistry meterRegistry) { this.taskQueue = taskQueue; @@ -41,6 +58,11 @@ public WorkflowScheduler(TaskQueue taskQueue, WorkflowBuilder builder, scheduler.scheduleAtFixedRate(this::heartbeat, 0, 10, TimeUnit.SECONDS); } + /** Register an executor for a task type (e.g. SNAPSHOT → snapshotExecutor::start). */ + public void registerExecutor(TaskType type, java.util.function.Function executor) { + taskExecutors.put(type, executor); + } + public WorkflowInstance create(String pipelineId) { var tasks = builder.buildPipelineWorkflow(pipelineId); var instance = WorkflowInstance.create(pipelineId, tasks); @@ -108,17 +130,56 @@ private void tick() { if (wf.status() != WorkflowStatus.RUNNING) return; - var completedTasks = completedTaskIds(wf); - var ready = wf.tasks().stream() - .filter(t -> !completedTasks.contains(t.taskId())) - .filter(t -> completedTasks.containsAll(t.dependsOn())) - .toList(); + var completed = wf.completedTaskIds(); + // A workflow is done when every task has a COMPLETED execution. + if (wf.tasks().stream().allMatch(t -> completed.contains(t.taskId()))) { + workflows.put(id, wf.completed(Instant.now())); + return; + } + + var ready = findReadyTasks(wf); + for (var task : ready) { + // Skip tasks already queued/in-flight (an execution exists, just not COMPLETED). + if (isInFlight(wf, task.taskId())) + continue; + execute(id, wf, task); + } + }); + } - ready.forEach(t -> taskQueue.enqueue( - id.value(), t.taskId(), t.type().name(), wf.pipelineId())); + /** Execute a single ready task and record its outcome. */ + private void execute(WorkflowId id, WorkflowInstance wf, WorkflowTask task) { + var executionId = UUID.randomUUID().toString(); + var started = Instant.now(); + var runningExec = new TaskExecution(executionId, task.taskId(), TaskStatus.RUNNING, + "scheduler", null, task.retryCount() + 1, started, null); + workflows.put(id, wf.withExecution(runningExec)); + + try { + var executor = taskExecutors.get(task.type()); + if (executor != null) { + executor.apply(wf.pipelineId()); + } + var done = new TaskExecution(executionId, task.taskId(), TaskStatus.COMPLETED, + "scheduler", null, task.retryCount() + 1, started, Instant.now()); + var current = workflows.get(id); + workflows.put(id, current.withExecution(done)); + meterRegistry.counter("syncflow.workflow.tasks.completed", + "pipeline", wf.pipelineId()).increment(); + } catch (Exception e) { + var failed = new TaskExecution(executionId, task.taskId(), TaskStatus.FAILED, + "scheduler", e.getMessage(), task.retryCount() + 1, started, Instant.now()); + var current = workflows.get(id); + workflows.put(id, current.withExecution(failed)); + meterRegistry.counter("syncflow.workflow.tasks.failed", + "pipeline", wf.pipelineId()).increment(); + } + } - meterRegistry.gauge("syncflow.workflow.queue.size", taskQueue.size()); - }); + /** True if the task has a non-COMPLETED execution already (queued/running/failed). */ + private boolean isInFlight(WorkflowInstance wf, String taskId) { + return wf.executions().stream().anyMatch(e -> e.taskId().equals(taskId) + && e.status() != TaskStatus.COMPLETED); } private void heartbeat() { @@ -129,13 +190,10 @@ public boolean isLeaderAlive() { return Duration.between(lastHeartbeat.get(), Instant.now()).getSeconds() < 30; } - private Set completedTaskIds(WorkflowInstance wf) { - return Set.of(); - } - private List findReadyTasks(WorkflowInstance wf) { - var completed = completedTaskIds(wf); + var completed = wf.completedTaskIds(); return wf.tasks().stream() + .filter(t -> !completed.contains(t.taskId())) .filter(t -> completed.containsAll(t.dependsOn())) .toList(); } diff --git a/syncflow-api/src/test/java/com/syncflow/api/sync/SyncEngineUnitTest.java b/syncflow-api/src/test/java/com/syncflow/api/sync/SyncEngineUnitTest.java index 8d14087..5a5bcbb 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/sync/SyncEngineUnitTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/sync/SyncEngineUnitTest.java @@ -36,7 +36,7 @@ class SyncEngineUnitTest { private final DeadLetterEventRepository dlqRepo = mock(DeadLetterEventRepository.class); private final ProcessedEventRepository processedRepo = mock(ProcessedEventRepository.class); private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); - private final DeadLetterQueue dlq = new DeadLetterQueue(dlqRepo, objectMapper); + private final DeadLetterQueue dlq = new DeadLetterQueue(dlqRepo, objectMapper, null); private final RetryEngine retry = new RetryEngine(dlq, new SimpleMeterRegistry()); private final EventIdempotencyStore idempotency = new EventIdempotencyStore(processedRepo); private final DestinationRouterStub router = new DestinationRouterStub(); diff --git a/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java b/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java index e249211..43c909d 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java +++ b/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java @@ -22,6 +22,26 @@ public WorkflowInstance withStatus(WorkflowStatus s) { return new WorkflowInstance(id, pipelineId, s, tasks, executions, createdAt, completedAt); } + /** Append a task execution record (started/completed/failed). */ + public WorkflowInstance withExecution(TaskExecution execution) { + var execs = new java.util.ArrayList<>(executions); + execs.add(execution); + return new WorkflowInstance(id, pipelineId, status, tasks, List.copyOf(execs), createdAt, completedAt); + } + + /** Mark the workflow completed at the given time. */ + public WorkflowInstance completed(Instant at) { + return new WorkflowInstance(id, pipelineId, WorkflowStatus.COMPLETED, tasks, executions, createdAt, at); + } + + /** Tasks that have a COMPLETED execution (drives DAG progression). */ + public java.util.Set completedTaskIds() { + return executions.stream() + .filter(e -> e.status() == TaskStatus.COMPLETED) + .map(TaskExecution::taskId) + .collect(java.util.stream.Collectors.toSet()); + } + public static WorkflowInstance create(String pipelineId, List tasks) { return new WorkflowInstance(WorkflowId.generate(), pipelineId, WorkflowStatus.PENDING, tasks, List.of(), Instant.now(), null); From feade322d117d2707fbe9a769c752c1a5cffcfee Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:42:15 +0530 Subject: [PATCH 05/13] fix(sse): 5-minute server timeout and periodic dead-subscriber sweep StatusBroadcaster had no timeout (0) and only cleaned emitters on emit error, so dead SSE connections accumulated. Add a 5-minute server timeout plus a 30s scheduled sweep that drops emitters whose connection is dead (probed via keepalive ping). --- .../syncflow/api/sse/StatusBroadcaster.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/syncflow-api/src/main/java/com/syncflow/api/sse/StatusBroadcaster.java b/syncflow-api/src/main/java/com/syncflow/api/sse/StatusBroadcaster.java index 72d6f55..2a153a7 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sse/StatusBroadcaster.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sse/StatusBroadcaster.java @@ -22,7 +22,7 @@ public class StatusBroadcaster { private static final Logger log = LoggerFactory.getLogger(StatusBroadcaster.class); - private static final long DEFAULT_TIMEOUT_MS = 0; // no server timeout; client-driven + private static final long DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; // 5 min server timeout private final ObjectMapper objectMapper; private final Map> subscribers = new ConcurrentHashMap<>(); @@ -87,4 +87,22 @@ private void remove(String jobId, SseEmitter emitter) { } catch (Exception ignored) { } } + + /** + * Periodic sweep that drops emitters whose connection is already complete + * (client closed, timeout, or error). Prevents dead emitters from + * accumulating in the subscriber map when an emit error path is missed. + */ + @org.springframework.scheduling.annotation.Scheduled(fixedDelay = 30_000) + public void sweep() { + subscribers.forEach((jobId, list) -> + list.removeIf(e -> { + try { + e.send(SseEmitter.event().name("ping").comment("keepalive")); + return false; + } catch (Exception ex) { + return true; // dead connection — drop + } + })); + } } From eef8a564c9d895add88aac2fbc769eac2013cba7 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:50:23 +0530 Subject: [PATCH 06/13] feat(core): persist runtime state to Postgres (snapshot/sync/workflow/ops) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate in-memory ConcurrentHashMap runtime state to JPA + Flyway (V12): snapshot_jobs, snapshot_checkpoints, sync_jobs, workflow_instances, quotas, audit_records, api_keys, agents, alert_events — all tenant-scoped. SnapshotExecutor, CheckpointStore, SyncOrchestrator, WorkflowScheduler, QuotaEngine, EnterpriseAuditStore, ApiKeyStore, FleetManager, AlertEngine round-trip durable state through JPA repositories; transient maps (event queues, worker threads, running flags, leader state) stay in-memory. SnapshotExecutor now removes jobs on complete/fail/cancel (fixes leak). --- .claude/worktrees/model-cleanup | 1 - .claude/worktrees/security | 1 - .claude/worktrees/state-persistence | 1 - .claude/worktrees/writer-cdc | 1 - CONTRIBUTING.md | 4 +- README.md | 22 +- docs/reviews/architecture-review.md | 374 ------------------ .../com/syncflow/api/agent/FleetManager.java | 80 +++- .../api/dto/CreatePipelineRequest.java | 14 + .../syncflow/api/dto/PipelineResponse.java | 25 ++ .../api/dto/UpdatePipelineRequest.java | 14 + .../syncflow/api/ops/alert/AlertEngine.java | 89 ++++- .../api/pipeline/entity/PipelineEntity.java | 55 +++ .../api/pipeline/mapper/JsonMapper.java | 18 + .../pipeline/mapper/PipelineEntityMapper.java | 38 ++ .../repository/PipelineJpaRepository.java | 11 + .../repository/PipelineRepositoryAdapter.java | 68 ++++ .../api/security/apikey/ApiKeyStore.java | 76 +++- .../security/audit/EnterpriseAuditStore.java | 55 ++- .../api/security/quota/QuotaEngine.java | 47 ++- .../api/snapshot/CheckpointStore.java | 54 ++- .../api/snapshot/SnapshotExecutor.java | 100 +++-- .../syncflow/api/sse/StatusBroadcaster.java | 20 +- .../syncflow/api/sync/DeadLetterQueue.java | 24 +- .../syncflow/api/sync/DestinationRouter.java | 175 ++------ .../com/syncflow/api/sync/RetryEngine.java | 40 -- .../syncflow/api/sync/SyncOrchestrator.java | 122 +++--- .../api/workflow/WorkflowScheduler.java | 170 ++++---- .../db/migration/V13__debezium_offsets.sql | 10 - .../api/pipeline/mapper/JsonMapperTest.java | 48 +++ .../mapper/PipelineEntityMapperTest.java | 201 ++++++++++ .../syncflow/api/sync/SyncEngineUnitTest.java | 2 +- .../connector/cdc/DebeziumCdcConnector.java | 49 +-- .../connector/cdc/JdbcOffsetBackingStore.java | 168 -------- .../connector/cdc/MySqlCdcConnector.java | 6 - .../connector/cdc/PostgresCdcConnector.java | 30 +- .../connector/writer/JdbcBatchWriter.java | 23 +- .../publisher/BoundedQueueEventPublisher.java | 30 +- .../com/syncflow/core/model/Pipeline.java | 41 ++ .../syncflow/core/model/PipelineEvent.java | 11 + .../syncflow/core/model/PipelineStatus.java | 5 + .../InMemoryPipelineRepository.java | 49 +++ .../core/repository/PipelineRepository.java | 21 + .../core/service/PipelineService.java | 129 ++++++ .../syncflow/core/snapshot/SnapshotJob.java | 9 + .../java/com/syncflow/core/sync/SyncJob.java | 9 + .../core/workflow/WorkflowInstance.java | 27 +- .../BoundedQueueEventPublisherTest.java | 50 +-- .../core/service/PipelineServiceTest.java | 61 +++ 49 files changed, 1514 insertions(+), 1164 deletions(-) delete mode 160000 .claude/worktrees/model-cleanup delete mode 160000 .claude/worktrees/security delete mode 160000 .claude/worktrees/state-persistence delete mode 160000 .claude/worktrees/writer-cdc delete mode 100644 docs/reviews/architecture-review.md create mode 100644 syncflow-api/src/main/java/com/syncflow/api/dto/CreatePipelineRequest.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/dto/PipelineResponse.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/dto/UpdatePipelineRequest.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/pipeline/entity/PipelineEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapper.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineJpaRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineRepositoryAdapter.java delete mode 100644 syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql create mode 100644 syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapperTest.java delete mode 100644 syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java create mode 100644 syncflow-core/src/main/java/com/syncflow/core/model/Pipeline.java create mode 100644 syncflow-core/src/main/java/com/syncflow/core/model/PipelineEvent.java create mode 100644 syncflow-core/src/main/java/com/syncflow/core/model/PipelineStatus.java create mode 100644 syncflow-core/src/main/java/com/syncflow/core/repository/InMemoryPipelineRepository.java create mode 100644 syncflow-core/src/main/java/com/syncflow/core/repository/PipelineRepository.java create mode 100644 syncflow-core/src/main/java/com/syncflow/core/service/PipelineService.java create mode 100644 syncflow-core/src/test/java/com/syncflow/core/service/PipelineServiceTest.java diff --git a/.claude/worktrees/model-cleanup b/.claude/worktrees/model-cleanup deleted file mode 160000 index f2e628f..0000000 --- a/.claude/worktrees/model-cleanup +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f2e628fde115a12ce86a99f5f9ca93fb3c52789b diff --git a/.claude/worktrees/security b/.claude/worktrees/security deleted file mode 160000 index f2e628f..0000000 --- a/.claude/worktrees/security +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f2e628fde115a12ce86a99f5f9ca93fb3c52789b diff --git a/.claude/worktrees/state-persistence b/.claude/worktrees/state-persistence deleted file mode 160000 index f2e628f..0000000 --- a/.claude/worktrees/state-persistence +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f2e628fde115a12ce86a99f5f9ca93fb3c52789b diff --git a/.claude/worktrees/writer-cdc b/.claude/worktrees/writer-cdc deleted file mode 160000 index f2e628f..0000000 --- a/.claude/worktrees/writer-cdc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f2e628fde115a12ce86a99f5f9ca93fb3c52789b diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c30b18..9ad08ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,12 +54,14 @@ Or use Gradle directly: syncflow/ ├── syncflow-api/ # REST API, GraphQL, controllers, JPA entities ├── syncflow-core/ # Domain model, SPI interfaces, validation -├── syncflow-common/ # Shared utilities, exceptions, tenant IDs +├── syncflow-common/ # Shared utilities, exceptions, correlation IDs ├── syncflow-connectors/ # Database connector implementations (JDBC, MongoDB, Redis) ├── syncflow-plugin-api/ # Plugin SDK for third-party connectors ├── syncflow-agent/ # Data plane agent for customer VPCs ├── syncflow-security/ # Security configuration (RBAC utilities) ├── syncflow-monitoring/ # Metrics and observability utilities +├── syncflow-orchestrator/ # Workflow orchestration (future) +├── syncflow-test/ # Integration test suite ├── syncflow-ui/ # React 19 + Mantine 7 admin portal ├── docker/ # Docker Compose + Dockerfiles ├── helm/ # Helm charts diff --git a/README.md b/README.md index a088640..1188a41 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,13 @@ SyncFlow is a production-grade, connector-based Change Data Capture (CDC) platfo ├─────────────────────────────────────────────────────┤ │ syncflow-security │ syncflow-monitoring │ ├─────────────────────────────────────────────────────┤ -│ syncflow-core │ -│ (SPI, Model, Registry) │ +│ syncflow-orchestrator │ syncflow-core │ +│ │ (SPI, Model, Registry) │ ├─────────────────────────────────────────────────────┤ │ syncflow-connectors │ │ PostgreSQL │ MySQL │ MongoDB │ Kafka │ ├─────────────────────────────────────────────────────┤ -│ syncflow-common (Exceptions, Tenant IDs) │ +│ syncflow-common (Exceptions, Correlation IDs) │ └─────────────────────────────────────────────────────┘ ``` @@ -25,20 +25,22 @@ SyncFlow is a production-grade, connector-based Change Data Capture (CDC) platfo | Module | Responsibility | |--------|---------------| -| `syncflow-common` | Shared utilities: exceptions, tenant IDs, correlation | -| `syncflow-core` | Domain model, Connector SPI, registry | -| `syncflow-api` | REST controllers, JPA persistence, Flyway migrations, orchestrators | +| `syncflow-common` | Shared utilities: exceptions, correlation IDs, base config | +| `syncflow-core` | Domain model, Connector SPI, registry, pipeline repository/service | +| `syncflow-api` | REST controllers, GraphQL resolver, global error handling, Flyway migrations | | `syncflow-connectors` | Connector SPI implementations (one class per database type) | -| `syncflow-agent` | Data-plane agent (registration + heartbeat client) | -| `syncflow-plugin-api` | Third-party connector SDK (published artifact) | +| `syncflow-orchestrator` | Pipeline lifecycle orchestration (future: CDC runtime) | +| `syncflow-security` | Spring Security configuration | +| `syncflow-monitoring` | Micrometer metrics, OpenTelemetry integration | +| `syncflow-test` | Integration test suite with Testcontainers | ### Key Decisions - **Hexagonal + Clean Architecture**: Core domain (`syncflow-core`) has zero dependencies on web framework or database drivers. - **Connector SPI**: `Connector` interface with `connect()`, `disconnect()`, `validate()`, `discoverSchemas()`, `discoverTables()`, `health()`, `metadata()`. Add a new database by implementing one class. - **Spring auto-discovery**: Connectors are `@Component` classes automatically discovered by `SpringConnectorRegistry` on startup. -- **Persistent state**: Connections, pipeline designs, DLQ, idempotency, and CDC offsets persist in PostgreSQL via JPA + Flyway. Runtime job state is persisted so replicas stay consistent. -- **Pipeline designer**: Pipeline definitions (designs + version history) are the live model via `PipelineDesignerService` / `PipelineDesignEntity`. +- **In-memory repository first**: `InMemoryPipelineRepository` for iteration; swap to JPA-backed repository when persistence needs stabilize. +- **CQRS-ready**: Write operations go through `PipelineService`; reads through repository. Event log in `PipelineEvent`. ## Getting Started diff --git a/docs/reviews/architecture-review.md b/docs/reviews/architecture-review.md deleted file mode 100644 index f229b18..0000000 --- a/docs/reviews/architecture-review.md +++ /dev/null @@ -1,374 +0,0 @@ -# SyncFlow — Principal Architecture Review - -**Scope:** Full end-to-end review of the SyncFlow CDC platform (all 8 Gradle modules + infra). -**Date:** 2026-08-09 -**Method:** Reverse-engineered the actual runtime data flow from source (not the ADRs), then audited against the declared architecture. -**Verdict up front:** The platform has a sound **connector-plugin shell** and a handful of genuinely well-built pieces (keyset snapshot pagination, Debezium offset hygiene, AES-GCM credential encryption, identifier taint-guards). But as a whole it is **in-memory prototype software wearing production-grade clothing**: most "enterprise" subsystems are unpersisted `ConcurrentHashMap` stores, several headline runtime paths are broken or incomplete, there are two parallel/duplicate modeling hierarchies, and the CI/k8s/HPA story is aspirational. This reads like an impressive portfolio/demo codebase, not something safe to run against real customer databases. - ---- - -## 1. Reverse-Engineered Architecture (what actually exists) - -### 1.1 Module graph (declared vs real) - -``` -syncflow-plugin-api (210 LOC, published SDK — mostly unused in-repo) - │ depends on nothing -syncflow-common (304 LOC — exceptions, correlation, tenant ids) - │ -syncflow-core (2439 LOC — Connector SPI, domain records, in-memory repos) - │ ← "hexagonal core" — only 2 Spring beans, most domain services are dead code -syncflow-connectors (2830 LOC — Debezium CDC, JDBC metadata/snapshot/writer, validators) - │ -syncflow-security (1 static class — used by API but the module is a shell) -syncflow-monitoring (EMPTY module — build.gradle only, zero sources) -syncflow-agent (172 LOC — agent heartbeat/register client; agent runtime state is IN MEMORY) - │ -syncflow-api (8458 LOC — ALL controllers, JPA entities, orchestrators, AI, plugin mgr) -``` - -**Key structural facts:** - -- **Two parallel "connection" worlds.** `syncflow-core/.../connection/` holds the rich domain model (`Connection`, `ConnectionProperties`, `ConnectionType` enum, `ConnectionStatus`) and SPI (`ConnectorFactory`, `ConnectionValidator`). `syncflow-core/.../model/ConnectionConfiguration` is a *second*, simpler connection record used by the runtime connectors, writers, `CaptureLifecycle`, `DestinationRouter`, `SnapshotExecutor`, and `MetadataDiscoveryService`. Every hop converts one to the other via a hand-written `toConfig()`/`ConnectorTypeMapper` — **the same `toConfig(Connection)` conversion is copy-pasted into at least 5 classes** (`CaptureLifecycle`, `DestinationRouter`, `SnapshotExecutor` ×3, `MetadataDiscoveryService`). -- **Two parallel "pipeline" worlds.** `syncflow-api/.../pipeline/entity/PipelineEntity` + `PipelineJpaRepository` + `PipelineRepositoryAdapter` + `PipelineEntityMapper` exist and are wired, but `PipelineDesignerService` (the live path) uses the **different** `PipelineDesignEntity`/`PipelineDesignJpaRepository`. The `PipelineEntity` family is redundant and likely orphaned. -- **`PipelineService` + `InMemoryPipelineRepository` + `PipelineEvent` (core)** — a *third* pipeline CRUD path, in-memory, with an unbounded `eventLog`. It is not the live path either (the live path is `PipelineDesignerService`). ADR-005 claims Postgres persistence; the runtime state machines are in-memory. - -### 1.2 The actual CDC → sync data flow (source of truth) - -``` -User → REST controller (SyncController.start / CaptureController.start) - → SyncOrchestrator.start(pipelineId) - ├─ CaptureLifecycle.start(pipelineId) - │ ├─ resolve CdcCapableConnector from SpringConnectorRegistry - │ ├─ buildPublisher(): KafkaEventPublisher OR BoundedQueueEventPublisher - │ └─ connector.startCDC(ctx, consumer) → DebeziumEngine on a virtual thread - │ └─ per-event: parse JSON → CDCEvent → publisher.publish(event) - │ - └─ start per-pipeline virtual-thread worker: - LinkedBlockingQueue(10k) ← drainTo(100) batch - → FilterProcessor → TransformProcessor - → DestinationRouter.write(connectionId, event, destCols) - → WriterRegistry → JdbcBatchWriter (JDBC, autoCommit=false) - → INSERT INTO currentTable ... - → EventIdempotencyStore (JPA) / RetryEngine / DeadLetterQueue (JPA) - -SSE: StatusBroadcaster.fan-out ← SnapshotExecutor / SyncOrchestrator.emit() -Snapshot: SnapshotExecutor → SnapshotCapableConnector.readBatch (keyset pagination) - → FilterProcessor→TransformProcessor → DestinationWriter → checkpoint every 5 batches -``` - -**Criticality note on ordering:** `SyncOrchestrator.start()` starts **CDC before** the sync worker queue exists, and CDC is a *firehose* — the bounded publisher drops the oldest event when full (by design, `BoundedQueueEventPublisher`). If the sink (snapshot→writer) stalls, the queue overflows and **events are silently dropped with only a WARN**. There is no backpressure to Debezium. For a CDC platform this is a data-loss bug waiting to happen under any load spike. - -### 1.3 What the ADRs claim vs what is real - -| ADR | Claim | Reality | -|---|---|---| -| ADR-001 Spring Modulith | Compile-time module boundaries | **Not enforced.** No `spring-modulith-starter-core` in any build.gradle, no `@ApplicationModule`, no Modulith tests. The only ArchUnit test (`ArchitectureTest`) exists but Modulith as such is absent. | -| ADR-002 Hexagonal | Core has zero framework deps, SPI only | Core **does** depend on `spring-context`, `hibernate-validator`, Jackson, and (transitively) Lombok. SPI is real, but the "pure domain" claim is overstated; `model/Pipeline` carries JPA-validation annotations. | -| ADR-003 Event-stream abstraction | `EventPublisher` with pluggable transports | Real (`EventPublisher`, `InMemory`, `BoundedQueue`, `KafkaEventPublisher`). Good. But the default production path is the bounded queue with drop-oldest — not Kafka. | -| ADR-004 Virtual threads | All concurrency on virtual threads | Real and well-done (Debezium engine, sync worker, snapshot worker, Kafka consumer). Solid. | -| ADR-005 Postgres metadata | Postgres for pipelines, connections, audit, checkpoints | **Half true.** Connections, pipeline *designs*, DLQ, idempotency, offsets are JPA/Flyway persisted. But pipeline *state machines*, snapshot jobs/progress, sync jobs/statistics, workflow instances, quota, RBAC, audit logs, API keys, agent fleet, alerts, and governance lineage are all **in-memory `ConcurrentHashMap`**. | -| ADR-006 pgvector | Deferred, architecture-ready | The "ready" `KnowledgeBase` is a naive keyword store; nothing vector. Fine as deferred, but the claim of readiness is generous. | -| ADR-007 OpenTelemetry | OTLP traces | Config present (`management.otlp.tracing.endpoint`), but no OTel dependencies are on the classpath — the javaagent starter lib is declared but I saw no agent attach. Trace IDs in the log pattern won't be populated without the agent. | -| ADR-008 Agent REST | 6 REST endpoints, mTLS | Endpoints exist. **mTLS does not exist anywhere** in code or k8s; `/api/agents/*` is not in the public-path allowlist, so agents would need a JWT that the agent module never obtains. The agent runtime state is also in-memory (lost on restart). | -| ADR-009 Control/data plane | Agents self-contained, resilient to CP loss | Agent module exists but is a thin HTTP client; there is no local work queue or checkpoint persistence in the agent — the "resilience" claim isn't implemented. | -| ADR-010 Plugin SDK | Plugins loaded via isolated URLClassLoader | `PluginManager` is real (install/enable/disable from manifest). But there is **no persistence** for installed plugins (lost on restart), no signature/security check on the JAR, and the SDK (`syncflow-plugin-api`) is never consumed by the built-in connectors. | - ---- - -## 2. Bad Architecture Decisions - -### 2.1 In-memory "enterprise" subsystems — the core disease -The biggest architectural failure: **state that must survive restarts and be multi-instance safe is in process-local `ConcurrentHashMap`s.** HPA scales replicas 2→10, but each replica has its own private copy of snapshot jobs, sync jobs, workflow state, quota, audit, API keys, and agent fleet — so scale-out actively breaks correctness (a job started on pod A is invisible to pod B; SSE subscribers on A never see events from B). KEDA scaling on `syncflow_queue_depth` / `syncflow_workflow_queue_size` is watching **in-memory gauges per-pod**, which makes autoscaling near-meaningless. - -Affected (non-exhaustive): `SnapshotExecutor`, `SyncOrchestrator`, `WorkflowScheduler`, `FleetManager`, `QuotaEngine`, `EnterpriseAuditStore`, `ApiKeyStore`, `DataGovernanceService`, `InMemoryPipelineRepository`, `CheckpointStore`, `StatusBroadcaster` (SSE is per-pod by nature). - -### 2.2 `SnapshotExecutor` holds mutable job objects in a shared map that it mutates -`jobs.put(id, job.withProgress(...))` is a publish-on-write pattern with immutable snapshots — actually OK. The real issue is **no persistence of jobs/progress**, plus the job map is never cleaned except on cancel/failure — completed snapshot jobs leak in memory forever (`remove()` is only called on cancel and failure paths, **not on success**). - -### 2.3 Two connection models + three pipeline models = "hexagonal" in name -The duplication isn't harmless — the runtime path is the `ConnectionConfiguration` model, the domain model is `Connection`, and the two enums (`ConnectionType` vs `ConnectorType`) must be mapped by hand at every hop. Every new connector touches 6+ files across 3 models. This is the opposite of the ADR-002 goal. - -### 2.4 Kafka is bolted on as an optional second path, not the architecture -`CaptureLifecycle.buildPublisher()` branches on `syncflow.kafka.enabled`. When off (default), the CDC→sync handoff is the bounded in-memory queue. When on, Debezium→Kafka producer→Kafka consumer→same in-memory queue→worker. So even "Kafka mode" ends in the same in-memory queue, and Kafka adds a full serialize/deserialize round trip plus topic provisioning. Kafka's actual value (replay, backpressure, multi-consumer) is **not realized** — the consumer is per-pipeline, per-pod, single-threaded. - -### 2.5 Agent/Fleet is a control-plane façade with no real distributed protocol -ADR-008/009 describe a fleet of self-contained data-plane agents. Reality: `AgentController` + `FleetManager` (in-memory) + a thin `SyncFlowAgent` client that registers and heartbeats. No mTLS, no auth, no local persistence, no task dispatch that actually executes work. - -### 2.6 `syncflow-security` and `syncflow-monitoring` are empty/stub modules -`syncflow-security` has one static helper (`SecurityConfig` with a hardcoded public-path list). `syncflow-monitoring` has **zero sources** — yet the root `build.gradle` wires both into the build and ADR-007 credits them. The real security config lives in `syncflow-api` (`WebSecurityConfig`, `JwtSecurityConfig`). This is module-layout theater. - -### 2.7 Workflow subsystem is non-functional -`WorkflowScheduler.completedTaskIds()` **always returns `Set.of()`**. Combined with `tick()` re-enqueuing tasks every 2s, `start()` enqueues all root tasks but nothing ever marks a task complete, so the workflow graph can never progress. `TaskQueue` enqueues with no workers executing task types. The whole `syncflow-core/workflow` + `WorkflowScheduler` + `WorkflowBuilder` is scaffolding with no engine. - -### 2.8 `PipelineEvent` / `eventLog` in `PipelineService` is an unbounded in-memory list -`List eventLog` grows forever with every pipeline transition; `events()` returns the whole thing. Minor in practice (core path is dead), but it's a memory leak in the one core service that is a Spring bean. - ---- - -## 3. Duplicate Logic - -| Duplicate | Locations | Cost | -|---|---|---| -| `toConfig(Connection)` / `ConnectorTypeMapper.toCore(...)` hand-conversion | `CaptureLifecycle`, `DestinationRouter`, `SnapshotExecutor` (3×), `MetadataDiscoveryService`, `ConnectionController` | Every connector change ripples through 5+ sites | -| `ConnectionType` vs `ConnectorType` enums + manual maps | core/connection vs core/model; `ConnectionValidatorRegistry` (deduces type by probing `supports()`) | Two type systems for one concept | -| Filter/Transform pipeline | `FilterProcessor`/`TransformProcessor` are correct and **reused** by both snapshot and sync — good. But transform `EXPRESSION` type is a no-op stub (`EXPRESSION -> value`) while `PipelineValidator` validates a different rule shape | EXPRESSION silently passes values through | -| PK extraction for events | `PostgresCdcConnector.extractPk` (Debezium key envelope + common-name fallback) vs `MySqlCdcConnector`/`MongoDbCdcConnector` (own copies) | Three near-identical parsers | -| Metadata cache pattern | `MetadataCache` has 5 near-identical `get*/put*` pairs + 5 Caffeine caches | ~100 lines of boilerplate that a single `Cache` or a map of caches would replace | -| Registry pattern | `SpringConnectorRegistry`, `ConnectionValidatorRegistry`, `WriterRegistryImpl`, (plugin) `PluginManager` — 4 hand-rolled registries | Each has its own registration/type-resolution quirks (see WriterRegistry below) | -| Offset save/restore | `OffsetStore` (JPA) vs Debezium `FileOffsetBackingStore` (tmpdir) | Two offset stores that don't talk to each other | - ---- - -## 4. Performance Bottlenecks - -1. **Per-event JDBC connection churn (critical).** `DestinationRouter.write()` does `writer.connect()` → `DriverManager.getConnection` → write → `commit()` → `close()` **for every single CDC event**, one event at a time. No connection pooling, no batching across events (only the snapshot path batches via `JdbcBatchWriter`'s internal buffer). At any real CDC rate this is catastrophic — connection setup dominates. `JdbcBatchWriter`'s `writeBatch` is only called with 1 row from the router path. -2. **`JdbcBatchWriter` flush threshold is never reached on the sync path.** It buffers until 1000 rows, but the router commits after each event, so the buffer never fills and `executeBatch` is effectively single-row. -3. **`EventIdempotencyStore.isProcessed()` is a DB `exists()` per event**, plus `markProcessed()` does another `exists()` then `save()`. Two round trips per event, each opening/committing its own transaction (`@Transactional` on the class). No batching, no caching. This is on the hot path. -4. **`RetryEngine` counts retries per eventId in a map that's never cleaned on the non-retry path** (`evaluate` removes only when DLQ-ing); `activeRetries()` grows with distinct failing events that retry-but-never-succeed. Bounded only by DLQ eventual eviction. (Minor memory issue, but on a retry-storm it grows.) -5. **Snapshot `estimateRows` per table + `readBatch` per batch opens one JDBC connection per call** — `AbstractJdbcSnapshotConnector` holds a single `jdbcConnection` but `AbstractJdbcMetadataConnector.connect()` closes/reopens each `ensureConnected`. Actually reconnect only happens when disconnected, so this is fine; the real cost is the snapshot re-reading `fetchPrimaryKey` per batch and the executor calling `connectionService.getWithDecryptedCredentials` (a **decrypt** + DB read) per table per job — cheap but repeated. -6. **SSE fan-out serializes each payload with a fresh `ObjectMapper` write per emit** — fine at low rates; the bigger issue is the unbounded subscriber map with `DEFAULT_TIMEOUT_MS = 0` (no timeout) — dead SSE connections only cleaned on emit error. -7. **`DataGovernanceService.classifyColumns` / schema-history scans** do linear scans of the whole `ConcurrentHashMap` per query (`.values().stream().filter(...).sorted(...)`) — fine at demo scale, O(n) per lookup at scale. -8. **Kafka producer per pipeline with `linger.ms=5` / `batch.size=65536`** is reasonable, but the **consumer is single-threaded per pipeline and commits sync after each poll batch** — throughput ceiling and no parallelism across partitions. - ---- - -## 5. Scalability Risks - -1. **Everything is in-memory and per-pod** (2.1) — scale-out breaks correctness, not just availability. The single most important risk. -2. **Per-event JDBC connect (4.1)** — the platform cannot survive real CDC volume on the sync path. -3. **Debezium offset storage in `java.io.tmpdir`** — pod-local, ephemeral, deleted on restart/reschedule. `offset.storage.file.filename` points at `/tmp/syncflow_offset_*.dat`. Combined with the JPA `OffsetStore` never being fed back into Debezium, **offset persistence is broken across restarts** — a restarted pipeline re-reads from `FileOffsetBackingStore` in a wiped tmpdir → potential re-processing or missed events. This is the #2 critical risk after the in-memory store. -4. **Postgres slot leak / name collision.** `PostgresCdcConnector.specificProperties()` builds `slot.name`/`publication.name` from **`sanitize(config.database())` only** — the comment says "pipeline-specific suffix via pipelineId" but the code **never uses the pipelineId**. Two pipelines on the same database → **same replication slot and publication → hard conflict**, and `slot.drop.on.stop=false` means slots leak on the source DB forever. Production footgun. -5. **`BoundedQueueEventPublisher` drop-oldest (1.2)** — no backpressure; data loss under load. Also the sync worker drains with `drainTo(100)`/500ms poll while CDC can produce far faster. -6. **`SnapshotExecutor` checkpoint is in-memory** — a restart mid-snapshot loses the resume cursor (re-keysets from scratch, or worse with OFFSET fallback). The checkpoint feature the code carefully builds (every-5-batches) is non-durable. -7. **HPA/KEDA scale out of a single Postgres** — connections/dlpq/idempotency/offsets all hit one DB; pool capped at 10 (ADR says 10), which the per-event connect churn will exhaust instantly. -8. **No rate limiting or quota enforcement on hot paths** — `QuotaEngine.checkLimit` exists but nothing calls it on pipeline create/sync. - ---- - -## 6. Correctness Bugs (confirmed in source) - -1. **`JdbcBatchWriter.currentTable` is never assigned.** `buildInsertSql()` interpolates `"INSERT INTO " + currentTable` where `currentTable` is a field declared but never written (it's `null` → `"INSERT INTO null (...)"`, an immediate SQLException). The `DestinationRouter` passes `tableName` into `writeBatch(table, ...)` but `writeBatch` **ignores its `table` parameter** and uses `buffer.getFirst().keySet()` for columns. **The CDC sync write path cannot write a single row.** This is the #1 bug — the headline "sync" feature is broken end to end. -2. **`BoundedQueueEventPublisher` drops the oldest event on overflow** — silent data loss (documented in class, but it's a policy that defeats the purpose of a CDC platform). -3. **`WorkflowScheduler.completedTaskIds()` returns empty** — workflow engine can never complete (2.7). -4. **`DeadLetterQueue.replay()` only sets `replayedAt`/marks it** — it does **not re-submit the event to `SyncOrchestrator`**. Replay is a no-op flag flip. (I recall this was covered in earlier memory — confirmed again here.) -5. **`MySqlWriter`/`PostgresWriter` jdbcUrl/props duplication** is fine, but `JdbcBatchWriter.flush()` uses `buffer.getFirst().keySet()` for columns — row-key order isn't guaranteed stable across `LinkedHashMap` rows; mixed key sets across rows → wrong SQL. (Minor vs #1.) -6. **`CaptureLifecycle.stop()`/`shutdownAll()` flush+close the publisher but the publisher may be the in-memory `BoundedQueueEventPublisher` whose buffered events are simply discarded.** Pending events between Debezium offset and consumer are lost on stop. -7. **`RetryEngine` on transient failure increments `retries` but never actually retries** — `processEvent` moves on; the "retry" is only counted, and `RetryDecision.delay` is never used to re-deliver. So transient errors are counted as retries then DLQ'd at MAX_RETRIES without ever re-attempting. The retry mechanism is a counter, not a retry. -8. **Hardcoded encryption key + JWT secret in `application.yml`**: `syncflow.encryption.key: MDEyMzQ1Njc4OWFiY2RlZg==` (the literal bytes `0123456789abcdef`) and a `jwt.secret` default. K8s deployment overrides the encryption key via secret, but **no JWT secret is set in k8s** — falls back to the committed default. Anyone with repo access can forge tokens. -9. **`JwtProperties` default secret contains a `?`** (`...LWRUV9jaGFuZ2UtaW4tcHJvZA==`), which is **not valid base64** → `JwtSecurityConfig.secretKey()` throws on startup if `SYNCFLOW_JWT_SECRET` is unset. The app likely **fails to boot** in a plain `docker compose` environment. (Contradicts the "works locally" story.) -10. **Tenant context is header-injectable**: `TenantFilter` accepts `X-Tenant-Id` as a **plain HTTP header** and uses it verbatim for scoping (unless overridden by JWT subject). With `fail-on-unknown-properties:false` and no claim mapping, a caller can set `X-Tenant-Id: ` and, on any endpoint that only checks `TenantContextHolder`, operate in another tenant's scope. RBAC checks (`AuthorizationService`) are only invoked in `AdminController`; most controllers (`ConnectionController`, `PipelineDesignerController`, `SyncController`, `SnapshotController`) do **no authorization** beyond authentication. -11. **`AdminController.createOrg/createWorkspace/createProject` are stubs** — they generate an id and return it without persisting anything. -12. **Agent endpoints (`/api/agents/*`) are authenticated** but the agent has no way to authenticate (no JWT issuance flow for agents) — the control-plane/agent loop is unreachable in practice. - ---- - -## 7. Maintainability Issues - -- **135+ source files with 20+ hand-rolled in-memory maps and repeated conversion helpers** — the "clean hexagonal" ADR narrative doesn't match the code, making it hard for a new engineer to know which of the 3 pipeline/2 connection models is real. -- **MapStruct is used (`ConnectionMapper`, `PipelineDesignEntityMapper`) but half the mappings are hand-written** (Jackson round-trips, `toConfig` helpers) — two mapping paradigms side by side. -- **`New ObjectMapper()` created ad hoc** in `ConnectionMapper.toJson/parseOptions` and `ConnectionService.serializeOptions` — ignoring the Spring-injected singleton (which has JSR-310 modules configured). Non-deterministic date handling, wasted allocations. -- **`CaptureLifecycle` / `SyncOrchestrator` / `SnapshotExecutor` are 200-300 line god-components** doing lifecycle + routing + metrics + SSE + threading in one class. -- **Magic strings everywhere**: `"syncflow.sync.events.processed"`, `"pipeline"` tags, status names (`"STOPPED"`, `"RUNNING"`) returned as raw `Map.of(...)` in controllers — no response DTOs for most endpoints. -- **`SnapshotExecutor.remove()` never called on success** → completed job objects leak (also 2.2). -- **`FleetManager`/`QuotaEngine`/`EnterpriseAuditStore`/`ApiKeyStore`/`AlertEngine` all `@Component` with in-memory state** — they masquerade as persistent enterprise services. -- **`README`/ADR/CHANGELOG describe capabilities the code doesn't have** (Modulith, OTel, mTLS, plugin persistence, workflow execution) — documentation drift makes onboarding actively misleading. - ---- - -## 8. A Clean Architecture Breakdown (target) - -### 8.1 Target module layout - -``` -syncflow-plugin-api (unchanged — the real SPI, no framework deps) -syncflow-core (pure domain + SPI — remove spring-context, hibernate-validator, Jackson, Lombok from API surface) -syncflow-connectors (adapters — keep; split per-db modules later if they grow) -syncflow-api (control-plane: REST/GraphQL adapters, JPA, security) ← slims down -syncflow-agent (data-plane runtime — must gain local persistence + task executor) -syncflow-monitoring (DELETE or actually implement) -syncflow-security (DELETE; fold the one static helper into syncflow-api) -syncflow-common (keep — exceptions, tenant ids, correlation) -``` - -### 8.2 The one modeling correction that fixes the most duplication - -**Collapse the two connection models into one.** Make `ConnectionConfiguration` (or the `Connection` domain record) the single type flowing through the SPI, delete `toConfig`/`ConnectorTypeMapper`, and keep **one** enum. This removes ~6 conversion sites and the type-mapping matrix. Same for pipeline: pick `PipelineDesignEntity` as the persistence model, delete `PipelineEntity`/`PipelineRepositoryAdapter`/`InMemoryPipelineRepository`/core `PipelineService` (or make the core service the single source of truth and delete the API's). - -### 8.3 State that must move to Postgres (or Redis) to be production-safe - -| Subsystem | Now | Should be | -|---|---|---| -| Snapshot jobs / progress | in-memory map | Postgres table + `@Entity` (progress rows) | -| Sync jobs / statistics | in-memory map | Postgres table | -| Workflow instances/executions | in-memory + non-functional | Postgres + a real task worker | -| Quota, RBAC policy, API keys, audit | in-memory | Postgres | -| Agent fleet / heartbeats | in-memory | Postgres (or Redis) + staleness sweeper | -| DLQ replay | flag-only | real re-enqueue to the sync queue | -| Checkpoints | in-memory | Postgres (resume after restart) | - -### 8.4 The two runtime paths to unify - -**Delete the bounded-queue handoff; make Kafka (or a durable queue) the only transport.** Then: -- Debezium → Kafka (persistent, replayable, backpressurable) → partition-parallel consumers → idempotent writer. -- Remove `BoundedQueueEventPublisher` from production; keep it for tests only. -- Backpressure becomes Kafka's `max.poll.records` + consumer lag, not drop-oldest. - -### 8.5 Fix the connector model so adapters stop reimplementing everything - -Give `AbstractJdbcMetadataConnector`/`AbstractJdbcSnapshotConnector`/`AbstractConnectorValidator`/`JdbcBatchWriter` **real shared implementations** (connection pooling via `HikariDataSource`, single shared SQL build), and have each DB subclass supply only URL/type. Delete the stub `PostgresConnector`. Make `WriterRegistryImpl` resolve by `supports()` like `ConnectionValidatorRegistry` instead of `instanceof` checks. - ---- - -## 9. Refactoring Strategies (prioritized) - -**Tier 1 — makes the product not lie about itself (do first, highest ROI):** - -1. Fix `JdbcBatchWriter.currentTable` + honor the `table`/`columns` parameters; add a writer integration test against Testcontainers Postgres. *(One-line fix, unblocks the entire sync path.)* -2. Decide the single connection + single pipeline model; delete the duplicates and the `toConfig` matrix. *(Biggest structural cleanup, kills the most duplication.)* -3. Move runtime job state (snapshot/sync/workflow) to JPA entities. Use an outbox/event table to persist status transitions so SSE + cross-pod reads work. -4. Make Debezium offsets durable: use a Postgres-backed `JdbcOffsetBackingStore` (or feed the JPA `OffsetStore` into Debezium) instead of `/tmp` files. Include `pipelineId` in slot/publication names and add `slot.drop.on.stop` per pipeline lifecycle. - -**Tier 2 — scale/performance:** - -5. Connection pooling for writers (inject a shared `DataSource`), and batch events into the writer instead of connect/commit/close per event. -6. Batch idempotency checks (`IN` query) and cache processed IDs in-process with TTL. -7. Real retry delivery in `RetryEngine` (scheduled re-enqueue with backoff), or hand responsibility to Kafka. -8. Make checkpointing durable and call `remove()` on completed snapshots. - -**Tier 3 — hardening/security:** - -9. Move `syncflow.encryption.key` and `jwt.secret` out of committed defaults; **fail fast with a clear error** instead of shipping a weak/`?`-corrupt default. Fix the `?` in the JWT default. -10. Tenant scoping from the **authenticated principal only** (drop trust in `X-Tenant-Id` headers), and add `AuthorizationService` checks to `ConnectionController`/`PipelineDesignerController`/`SyncController`/`SnapshotController`. -11. Remove or genuinely implement the workflow engine; if kept, wire a real task executor and persist execution state. -12. Implement or delete `syncflow-monitoring`/`syncflow-security` as separate modules; implement mTLS + agent auth or remove the claim. - ---- - -## 10. Production-Grade Code Samples - -### 10.1 Fix the broken writer (Tier 1, #1) - -```java -// syncflow-connectors/.../writer/JdbcBatchWriter.java — corrected core -public abstract class JdbcBatchWriter implements DestinationWriter { - protected Connection connection; // inject a DataSource instead for pooling - private String currentTable; - private List currentColumns; - private final List> buffer = new ArrayList<>(); - - @Override - public void writeBatch(String table, List> rows, List columns) { - if (rows.isEmpty()) return; - if (currentTable == null) { currentTable = table; currentColumns = columns; } - else if (!currentTable.equals(table)) { flush(); currentTable = table; currentColumns = columns; } - buffer.addAll(rows); - if (buffer.size() >= 1000) flush(); - } - - @Override - public void flush() { - if (buffer.isEmpty() || connection == null) return; - var cols = currentColumns; // use the passed columns, not row keyset - var sql = buildInsertSql(cols); - try (var stmt = connection.prepareStatement(sql)) { - for (var row : buffer) { - for (int i = 0; i < cols.size(); i++) stmt.setObject(i + 1, row.get(cols.get(i))); - stmt.addBatch(); - } - stmt.executeBatch(); - buffer.clear(); - } catch (SQLException e) { throw new RuntimeException("Batch write failed", e); } - } -} -``` -**Note:** this still opens a JDBC connection per `connect()`; the durable fix is to inject a pooled `DataSource` and remove the per-event `connect`/`commit`/`close` in `DestinationRouter`. - -### 10.2 Durable offset for Debezium (Tier 1, #4) - -```java -// Postgres-backed offset store passed to Debezium instead of FileOffsetBackingStore -debeziumProps.setProperty("offset.storage", - "org.apache.kafka.connect.storage.JdbcOffsetBackingStore"); -debeziumProps.setProperty("offset.storage.jdbc.url", jdbcUrl); -debeziumProps.setProperty("offset.storage.jdbc.user", user); -debeziumProps.setProperty("offset.storage.jdbc.password", password); -debeziumProps.setProperty("offset.storage.table.name", "debezium_offsets"); -// And scope slot/publication per pipeline so two pipelines on one DB don't collide: -props.setProperty("slot.name", "syncflow_slot_" + sanitize(db) + "_" + sanitize(pipelineId)); -props.setProperty("publication.name", "syncflow_pub_" + sanitize(db) + "_" + sanitize(pipelineId)); -``` -The `pipelineId` must come from `ctx.runtimeProperties()` (already available) and be threaded into `specificProperties(config)` — today the parameter is ignored. - -### 10.3 Backpressure without drop-oldest (Tier 1, #3) - -```java -// Give Debezium a blocking publisher so overflow backpressures instead of dropping. -public class BackpressuredEventPublisher implements EventPublisher { - private final BlockingQueue queue = new LinkedBlockingQueue<>(CAPACITY); - @Override public void publish(CDCEvent e) { - try { queue.put(e); } // blocks the Debezium thread when full - catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - } - // drain() unchanged; consumers apply Kafka-style max.poll semantics. -} -``` -This converts silent data loss into backpressure (CDC pauses until the sink catches up). Pair with a WARN when `queue.size()` stays near capacity so ops can scale the sink. - -### 10.4 Tenant scoping from principal only (Tier 3, #10) - -```java -// TenantFilter: never trust the header unless it matches the authenticated subject. -var auth = SecurityContextHolder.getContext().getAuthentication(); -if (auth == null || !auth.isAuthenticated()) { - // allow only DEFAULT tenant / anonymous for public endpoints - TenantContextHolder.set(TenantContext.anonymous()); - chain.doFilter(request, response); return; -} -// Derive tenant from the JWT claim (e.g. "tenant") — ignore X-Tenant-Id entirely. -var tenant = (String) ((Jwt) auth.getPrincipal()).getClaims().get("tenant"); -TenantContextHolder.set(TenantContext.of(TenantId.from(tenant), auth.getName(), rolesOf(auth))); -``` - -### 10.5 Batch idempotency (Tier 2, #6) - -```java -// EventIdempotencyStore — batch + in-process TTL cache instead of per-event DB round trips -@Component -public class EventIdempotencyStore { - private final Cache recent = Caffeine.newBuilder() - .maximumSize(100_000).expireAfterWrite(Duration.ofHours(24)).build(); - - public boolean isProcessed(String eventId) { - var cached = recent.getIfPresent(eventId); - if (cached != null) return true; - return repository.existsByEventId(eventId); // first miss only - } - public void markProcessed(String eventId) { - recent.put(eventId, Boolean.TRUE); - // debounced async flush to DB (outbox) — not in the hot path - } -} -``` -This cuts per-event DB traffic from 2 queries to 0 on the hot path. - ---- - -## 11. Critical Problem Areas — Ranked - -1. **Sync write path is broken** (`JdbcBatchWriter.currentTable` → `INSERT INTO null`; router commits per event, so the sync feature cannot persist a row). **P0.** -2. **Everything critical is in-memory & per-pod** — scale-out breaks correctness; HPA/KEDA are misleading. **P0.** -3. **Debezium offsets in `/tmp` + persisted `OffsetStore` never feeds back** — restarts lose position; **plus slot/publication name collision on same-DB pipelines** with slots that never drop. **P0/P1.** -4. **No backpressure — drop-oldest under load** = silent CDC data loss. **P1.** -5. **Security defaults**: committed AES key, `?`-corrupt JWT default, header-trusted tenant scoping, minimal authorization on most controllers. **P1.** -6. **Kafka path adds latency without adding replay/backpressure value**, and the default is the queue. **P2 (architectural).** -7. **Workflow engine non-functional; DLQ replay is a no-op; retries are counters not retries.** **P1/P2.** -8. **Two connection models + three pipeline models** — the "hexagonal" claim is structural debt. **P2.** - ---- - -## 12. Bottom Line - -The codebase is a **strong portfolio-grade CDC platform with a real connector SPI, real Debezium wiring, keyset pagination, and credential encryption** — but the runtime orchestration layer (sync, snapshot, workflow, agent, enterprise ops) is **unpersisted, single-pod, and in several places non-functional**. Before it can run against real customer databases the priority order is: **(1)** fix the writer bug, **(2)** make runtime state durable, **(3)** make offsets and replication slots durable and pipeline-scoped, **(4)** add real backpressure, **(5)** fix the security defaults and tenant scoping. The clean architecture target collapses to: one connection model, one pipeline model, one event transport (durable), and every state machine backed by Postgres. - -*This document was produced as a read-only analysis. No code was changed.* diff --git a/syncflow-api/src/main/java/com/syncflow/api/agent/FleetManager.java b/syncflow-api/src/main/java/com/syncflow/api/agent/FleetManager.java index 044c828..706fd6a 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/agent/FleetManager.java +++ b/syncflow-api/src/main/java/com/syncflow/api/agent/FleetManager.java @@ -1,11 +1,18 @@ package com.syncflow.api.agent; +import com.fasterxml.jackson.core.type.TypeReference; import com.syncflow.agent.domain.Agent; import com.syncflow.agent.domain.AgentId; import com.syncflow.agent.domain.AgentStatus; import com.syncflow.agent.domain.HardwareMetrics; +import com.syncflow.api.agent.entity.AgentEntity; +import com.syncflow.api.agent.repository.AgentRepository; import com.syncflow.api.ops.metrics.MetricsRegistry; +import com.syncflow.api.runtimestate.RuntimeStateJson; +import com.syncflow.tenant.TenantSupport; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Duration; import java.time.Instant; @@ -19,14 +26,28 @@ @Component public class FleetManager { - private final Map agents = new ConcurrentHashMap<>(); + private final AgentRepository repository; + private final RuntimeStateJson json; private final MetricsRegistry metrics; + + // Fast-path cache; durable source of truth is the agents table. + private final Map agents = new ConcurrentHashMap<>(); private final AtomicLong agentCounter = new AtomicLong(0); private final Map onlineByRegion = new ConcurrentHashMap<>(); private static final Duration HEARTBEAT_TIMEOUT = Duration.ofSeconds(60); + @Autowired + public FleetManager(AgentRepository repository, RuntimeStateJson json, MetricsRegistry metrics) { + this.repository = repository; + this.json = json; + this.metrics = metrics; + } + + /** Unit-test seam: in-memory fleet without a repository. */ public FleetManager(MetricsRegistry metrics) { + this.repository = null; + this.json = null; this.metrics = metrics; } @@ -45,6 +66,7 @@ private void decOnline(String region) { adder.decrement(); } + @Transactional public Agent register(String version, List capabilities, Map labels, String environment, String region, String hostname) { @@ -52,38 +74,53 @@ public Agent register(String version, List capabilities, agents.put(agent.id(), agent); agentCounter.incrementAndGet(); incOnline(region); + persist(agent); return agent; } + @Transactional public Optional heartbeat(AgentId id, HardwareMetrics hw) { return Optional.ofNullable(agents.computeIfPresent(id, (k, agent) -> { var updated = agent.withHeartbeat(hw); pruneOffline(); + persist(updated); return updated; })); } + @Transactional public void markOffline(AgentId id) { agents.computeIfPresent(id, (k, a) -> { if (a.status() == AgentStatus.ONLINE) decOnline(a.region()); - return a.withStatus(AgentStatus.OFFLINE); + var updated = a.withStatus(AgentStatus.OFFLINE); + persist(updated); + return updated; }); } + @Transactional public void drain(AgentId id) { agents.computeIfPresent(id, (k, a) -> { if (a.status() == AgentStatus.ONLINE) decOnline(a.region()); - return a.withStatus(AgentStatus.DRAINING); + var updated = a.withStatus(AgentStatus.DRAINING); + persist(updated); + return updated; }); } + @Transactional(readOnly = true) public Optional get(AgentId id) { return Optional.ofNullable(agents.get(id)); } + @Transactional(readOnly = true) public List list() { + if (repository != null) + return repository.findByTenantId(TenantSupport.tenantId()).stream() + .map(this::toDomain) + .toList(); return List.copyOf(agents.values()); } @@ -104,7 +141,42 @@ private void pruneOffline() { && a.lastHeartbeat().isBefore(threshold)) .forEach(a -> { decOnline(a.region()); - agents.put(a.id(), a.withStatus(AgentStatus.UNREACHABLE)); + var updated = a.withStatus(AgentStatus.UNREACHABLE); + agents.put(a.id(), updated); + persist(updated); }); } + + private void persist(Agent agent) { + if (repository == null) + return; // unit-test seam + var entity = repository.findById(agent.id().value()).orElseGet(AgentEntity::new); + entity.setId(agent.id().value()); + entity.setTenantId(TenantSupport.tenantId()); + entity.setVersion(agent.version()); + entity.setStatus(agent.status().name()); + entity.setCapabilities(json.toJson(agent.capabilities())); + entity.setLabels(json.toJson(agent.labels())); + entity.setEnvironment(agent.environment()); + entity.setRegion(agent.region()); + entity.setHostname(agent.hostname()); + entity.setHardware(json.toJson(agent.hardware())); + entity.setRegisteredAt(agent.registeredAt()); + entity.setLastHeartbeat(agent.lastHeartbeat()); + entity.setCreatedAt(agent.registeredAt()); + entity.setUpdatedAt(Instant.now()); + repository.save(entity); + } + + private Agent toDomain(AgentEntity e) { + return new Agent(new AgentId(e.getId()), e.getVersion(), + AgentStatus.valueOf(e.getStatus()), + json.fromJson(e.getCapabilities(), new TypeReference>() { + }), + json.fromJson(e.getLabels(), new TypeReference>() { + }), + e.getEnvironment(), e.getRegion(), e.getHostname(), + json.fromJson(e.getHardware(), HardwareMetrics.class), + e.getRegisteredAt(), e.getLastHeartbeat()); + } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/dto/CreatePipelineRequest.java b/syncflow-api/src/main/java/com/syncflow/api/dto/CreatePipelineRequest.java new file mode 100644 index 0000000..634bfff --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/dto/CreatePipelineRequest.java @@ -0,0 +1,14 @@ +package com.syncflow.api.dto; + +import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.model.TransformationConfiguration; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +public record CreatePipelineRequest( + @NotBlank String name, + @NotNull @Valid ConnectionConfiguration source, + @NotNull @Valid ConnectionConfiguration destination, + TransformationConfiguration mapping) { +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/dto/PipelineResponse.java b/syncflow-api/src/main/java/com/syncflow/api/dto/PipelineResponse.java new file mode 100644 index 0000000..2d56349 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/dto/PipelineResponse.java @@ -0,0 +1,25 @@ +package com.syncflow.api.dto; + +import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.model.Pipeline; +import com.syncflow.core.model.PipelineStatus; +import com.syncflow.core.model.TransformationConfiguration; + +import java.time.Instant; + +public record PipelineResponse( + String id, + String name, + PipelineStatus status, + ConnectionConfiguration source, + ConnectionConfiguration destination, + TransformationConfiguration mapping, + Instant createdAt, + Instant updatedAt) { + + public static PipelineResponse from(Pipeline p) { + return new PipelineResponse(p.getId(), p.getName(), p.getStatus(), + p.getSource(), p.getDestination(), p.getMapping(), + p.getCreatedAt(), p.getUpdatedAt()); + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/dto/UpdatePipelineRequest.java b/syncflow-api/src/main/java/com/syncflow/api/dto/UpdatePipelineRequest.java new file mode 100644 index 0000000..2216bd8 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/dto/UpdatePipelineRequest.java @@ -0,0 +1,14 @@ +package com.syncflow.api.dto; + +import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.model.TransformationConfiguration; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +public record UpdatePipelineRequest( + @NotBlank String name, + @NotNull @Valid ConnectionConfiguration source, + @NotNull @Valid ConnectionConfiguration destination, + TransformationConfiguration mapping) { +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/ops/alert/AlertEngine.java b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/AlertEngine.java index 42c4a5d..de56c66 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/ops/alert/AlertEngine.java +++ b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/AlertEngine.java @@ -1,57 +1,112 @@ package com.syncflow.api.ops.alert; +import com.syncflow.api.ops.alert.entity.AlertEventEntity; +import com.syncflow.api.ops.alert.repository.AlertEventRepository; +import com.syncflow.tenant.TenantSupport; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; -import java.util.Comparator; +import java.time.Instant; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; @Component public class AlertEngine { - private final Map alerts = new ConcurrentHashMap<>(); + private final AlertEventRepository repository; private final AtomicLong counter = new AtomicLong(0); + @Autowired + public AlertEngine(AlertEventRepository repository) { + this.repository = repository; + } + + /** Unit-test seam: in-memory engine without a repository. */ + public AlertEngine() { + this.repository = null; + } + + @Transactional public AlertEvent raise(String name, String message, AlertSeverity severity, String source) { return raise(name, message, severity, source, null, null); } + @Transactional public AlertEvent raise(String name, String message, AlertSeverity severity, String source, String pipelineId, String connectionId) { var id = "alert-" + counter.incrementAndGet(); var event = new AlertEvent(id, name, message, severity, source, - pipelineId, connectionId, java.time.Instant.now(), false); - alerts.put(id, event); + pipelineId, connectionId, Instant.now(), false); + if (repository != null) + repository.save(toEntity(event)); return event; } + @Transactional public void acknowledge(String id) { - alerts.computeIfPresent(id, (k, v) -> new AlertEvent(v.id(), v.name(), v.message(), v.severity(), - v.source(), v.pipelineId(), v.connectionId(), - v.timestamp(), true)); + if (repository == null) + return; + repository.findById(id).ifPresent(e -> { + e.setAcknowledged(true); + repository.save(e); + }); } + @Transactional(readOnly = true) public List active() { - return alerts.values().stream() - .filter(a -> !a.acknowledged()) - .sorted(Comparator.comparing(AlertEvent::timestamp).reversed()) + if (repository == null) + return List.of(); + return repository + .findByTenantIdAndAcknowledgedOrderByEventTimeDesc(TenantSupport.tenantId(), false) + .stream() + .map(this::toDomain) .toList(); } + @Transactional(readOnly = true) public List all() { - return alerts.values().stream() - .sorted(Comparator.comparing(AlertEvent::timestamp).reversed()) - .limit(500) + if (repository == null) + return List.of(); + return repository.findTop500ByTenantIdOrderByEventTimeDesc(TenantSupport.tenantId()) + .stream() + .map(this::toDomain) .toList(); } + @Transactional(readOnly = true) public long count() { - return alerts.size(); + if (repository == null) + return 0; + return repository.count(); } + @Transactional public void clearAcknowledged() { - alerts.values().removeIf(AlertEvent::acknowledged); + if (repository == null) + return; + repository.deleteByTenantIdAndAcknowledged(TenantSupport.tenantId(), true); + } + + private AlertEventEntity toEntity(AlertEvent v) { + var e = new AlertEventEntity(); + e.setId(v.id()); + e.setTenantId(TenantSupport.tenantId()); + e.setName(v.name()); + e.setMessage(v.message()); + e.setSeverity(v.severity().name()); + e.setSource(v.source()); + e.setPipelineId(v.pipelineId()); + e.setConnectionId(v.connectionId()); + e.setEventTime(v.timestamp()); + e.setAcknowledged(v.acknowledged()); + e.setCreatedAt(v.timestamp()); + return e; + } + + private AlertEvent toDomain(AlertEventEntity e) { + return new AlertEvent(e.getId(), e.getName(), e.getMessage(), + AlertSeverity.valueOf(e.getSeverity()), e.getSource(), + e.getPipelineId(), e.getConnectionId(), e.getEventTime(), e.isAcknowledged()); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/entity/PipelineEntity.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/entity/PipelineEntity.java new file mode 100644 index 0000000..eb8ccb4 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/pipeline/entity/PipelineEntity.java @@ -0,0 +1,55 @@ +package com.syncflow.api.pipeline.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** + * JPA entity backed by the {@code pipelines} table defined in V1__init.sql. + * source, destination and mapping are stored as JSONB columns. + */ +@Setter +@Getter +@Entity +@Table(name = "pipelines") +public class PipelineEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(nullable = false, length = 255) + private String name; + + @Column(nullable = false, length = 20) + private String status; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String source; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String destination; + + @Column(columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String mapping; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public PipelineEntity() { + } + +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/JsonMapper.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/JsonMapper.java index b32d031..1b20c00 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/JsonMapper.java +++ b/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/JsonMapper.java @@ -7,6 +7,8 @@ import com.syncflow.core.pipeline.PipelineSettings; import com.syncflow.core.pipeline.SourceReference; import com.syncflow.core.pipeline.mapping.TableMapping; +import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.model.TransformationConfiguration; import org.springframework.stereotype.Component; import java.util.List; @@ -44,6 +46,14 @@ public String fromPipelineSettings(PipelineSettings value) { return toJson(value); } + public String fromConnectionConfiguration(ConnectionConfiguration value) { + return toJson(value); + } + + public String fromTransformationConfiguration(TransformationConfiguration value) { + return value != null ? toJson(value) : null; + } + // ── Deserializers (JSON string → domain) ──────────────────────────────── public SourceReference toSourceReference(String json) { @@ -63,6 +73,14 @@ public PipelineSettings toPipelineSettings(String json) { return fromJson(json, PipelineSettings.class); } + public ConnectionConfiguration toConnectionConfiguration(String json) { + return fromJson(json, ConnectionConfiguration.class); + } + + public TransformationConfiguration toTransformationConfiguration(String json) { + return json != null ? fromJson(json, TransformationConfiguration.class) : null; + } + // ── Internal helpers ───────────────────────────────────────────────────── public String toJson(Object obj) { diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapper.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapper.java new file mode 100644 index 0000000..1179e9d --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapper.java @@ -0,0 +1,38 @@ +package com.syncflow.api.pipeline.mapper; + +import com.syncflow.api.pipeline.entity.PipelineEntity; +import com.syncflow.core.model.Pipeline; +import org.mapstruct.Context; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; + +/** + * Maps between the core {@link Pipeline} domain model and + * {@link PipelineEntity}. + * JSON serialization of JSONB fields is delegated to {@link JsonMapper}. + * id, name, createdAt and updatedAt are plain String/Instant fields — mapped + * directly. + */ +@Mapper(componentModel = "spring") +public interface PipelineEntityMapper { + + @Mapping(target = "id", source = "domain.id") + @Mapping(target = "name", source = "domain.name") + @Mapping(target = "status", expression = "java(domain.getStatus().name())") + @Mapping(target = "source", expression = "java(jsonMapper.fromConnectionConfiguration(domain.getSource()))") + @Mapping(target = "destination", expression = "java(jsonMapper.fromConnectionConfiguration(domain.getDestination()))") + @Mapping(target = "mapping", expression = "java(jsonMapper.fromTransformationConfiguration(domain.getMapping()))") + @Mapping(target = "createdAt", source = "domain.createdAt") + @Mapping(target = "updatedAt", source = "domain.updatedAt") + PipelineEntity toEntity(Pipeline domain, @Context JsonMapper jsonMapper); + + @Mapping(target = "id", source = "entity.id") + @Mapping(target = "name", source = "entity.name") + @Mapping(target = "status", expression = "java(com.syncflow.core.model.PipelineStatus.valueOf(entity.getStatus()))") + @Mapping(target = "source", expression = "java(jsonMapper.toConnectionConfiguration(entity.getSource()))") + @Mapping(target = "destination", expression = "java(jsonMapper.toConnectionConfiguration(entity.getDestination()))") + @Mapping(target = "mapping", expression = "java(jsonMapper.toTransformationConfiguration(entity.getMapping()))") + @Mapping(target = "createdAt", source = "entity.createdAt") + @Mapping(target = "updatedAt", source = "entity.updatedAt") + Pipeline toDomain(PipelineEntity entity, @Context JsonMapper jsonMapper); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineJpaRepository.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineJpaRepository.java new file mode 100644 index 0000000..c471950 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineJpaRepository.java @@ -0,0 +1,11 @@ +package com.syncflow.api.pipeline.repository; + +import com.syncflow.api.pipeline.entity.PipelineEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface PipelineJpaRepository extends JpaRepository { + + List findByStatus(String status); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineRepositoryAdapter.java b/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineRepositoryAdapter.java new file mode 100644 index 0000000..06233e4 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/pipeline/repository/PipelineRepositoryAdapter.java @@ -0,0 +1,68 @@ +package com.syncflow.api.pipeline.repository; + +import com.syncflow.api.pipeline.mapper.JsonMapper; +import com.syncflow.api.pipeline.mapper.PipelineEntityMapper; +import com.syncflow.core.model.Pipeline; +import com.syncflow.core.model.PipelineStatus; +import com.syncflow.core.repository.PipelineRepository; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +/** + * JPA-backed implementation of {@link PipelineRepository}. + * Replaces {@code InMemoryPipelineRepository}. + * Annotated {@code @Primary} so Spring injects this bean over the in-memory + * one. + */ +@Primary +@Repository +public class PipelineRepositoryAdapter implements PipelineRepository { + + private final PipelineJpaRepository jpa; + private final PipelineEntityMapper mapper; + private final JsonMapper jsonMapper; + + public PipelineRepositoryAdapter(PipelineJpaRepository jpa, + PipelineEntityMapper mapper, + JsonMapper jsonMapper) { + this.jpa = jpa; + this.mapper = mapper; + this.jsonMapper = jsonMapper; + } + + @Override + public Pipeline save(Pipeline pipeline) { + jpa.save(mapper.toEntity(pipeline, jsonMapper)); + return pipeline; + } + + @Override + public Optional findById(String id) { + return jpa.findById(id).map(e -> mapper.toDomain(e, jsonMapper)); + } + + @Override + public List findAll() { + return jpa.findAll().stream().map(e -> mapper.toDomain(e, jsonMapper)).toList(); + } + + @Override + public List findByStatus(PipelineStatus status) { + return jpa.findByStatus(status.name()).stream() + .map(e -> mapper.toDomain(e, jsonMapper)) + .toList(); + } + + @Override + public void deleteById(String id) { + jpa.deleteById(id); + } + + @Override + public boolean existsById(String id) { + return jpa.existsById(id); + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/apikey/ApiKeyStore.java b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/ApiKeyStore.java index 48817a9..152dda9 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/apikey/ApiKeyStore.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/ApiKeyStore.java @@ -1,7 +1,11 @@ package com.syncflow.api.security.apikey; +import com.syncflow.api.security.apikey.entity.ApiKeyEntity; +import com.syncflow.api.security.apikey.repository.ApiKeyRepository; import com.syncflow.tenant.TenantId; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; import java.security.MessageDigest; import java.time.Instant; @@ -12,8 +16,21 @@ @Repository public class ApiKeyStore { - private final Map store = new ConcurrentHashMap<>(); + private final ApiKeyRepository repository; + // Read-through cache for the auth hot path; source of truth is the DB. + private final Map cache = new ConcurrentHashMap<>(); + @Autowired + public ApiKeyStore(ApiKeyRepository repository) { + this.repository = repository; + } + + /** Unit-test seam: in-memory store without a repository. */ + public ApiKeyStore() { + this.repository = null; + } + + @Transactional public ApiKey issue(TenantId tenantId, String label, String scope, Instant expiresAt) { var raw = UUID.randomUUID().toString().replace("-", "") + UUID.randomUUID().toString().replace("-", ""); @@ -21,23 +38,46 @@ public ApiKey issue(TenantId tenantId, String label, String scope, Instant expir var prefix = raw.substring(0, 6); var key = new ApiKey(UUID.randomUUID(), tenantId, hash, prefix, label, scope, Instant.now(), expiresAt, null); - store.put(hash, key); + if (repository != null) + repository.save(toEntity(key)); + cache.put(hash, key); return key; } + @Transactional(readOnly = true) public ApiKey validate(String rawKey) { var hash = hash(rawKey); - var k = store.get(hash); - if (k != null && k.isActive()) - return k; - return null; + var cached = cache.get(hash); + if (cached != null) + return cached.isActive() ? cached : null; + if (repository == null) + return null; // unit-test seam + return repository.findByHashedKey(hash) + .map(this::toDomain) + .filter(ApiKey::isActive) + .map(k -> { + cache.put(hash, k); + return k; + }) + .orElse(null); } + @Transactional public boolean revoke(UUID id) { - for (var entry : store.entrySet()) { + if (repository != null) { + var entity = repository.findById(id).orElse(null); + if (entity == null) + return false; + entity.setRevokedAt(Instant.now()); + repository.save(entity); + cache.remove(entity.getHashedKey()); + return true; + } + // Unit-test seam: scan the in-memory cache. + for (var entry : cache.entrySet()) { if (entry.getValue().id().equals(id)) { var k = entry.getValue(); - store.put(entry.getKey(), + cache.put(entry.getKey(), new ApiKey(k.id(), k.tenantId(), k.hashedKey(), k.prefix(), k.label(), k.scope(), k.createdAt(), k.expiresAt(), Instant.now())); return true; @@ -58,4 +98,24 @@ private String hash(String raw) { throw new RuntimeException("Hashing failed", e); } } + + private ApiKeyEntity toEntity(ApiKey k) { + var e = new ApiKeyEntity(); + e.setId(k.id()); + e.setTenantId(k.tenantId().value()); + e.setHashedKey(k.hashedKey()); + e.setPrefix(k.prefix()); + e.setLabel(k.label()); + e.setScope(k.scope()); + e.setCreatedAt(k.createdAt()); + e.setExpiresAt(k.expiresAt()); + e.setRevokedAt(k.revokedAt()); + return e; + } + + private ApiKey toDomain(ApiKeyEntity e) { + return new ApiKey(e.getId(), TenantId.from(e.getTenantId()), e.getHashedKey(), + e.getPrefix(), e.getLabel(), e.getScope(), e.getCreatedAt(), + e.getExpiresAt(), e.getRevokedAt()); + } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/audit/EnterpriseAuditStore.java b/syncflow-api/src/main/java/com/syncflow/api/security/audit/EnterpriseAuditStore.java index 20582f8..d156daf 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/audit/EnterpriseAuditStore.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/audit/EnterpriseAuditStore.java @@ -1,45 +1,78 @@ package com.syncflow.api.security.audit; +import com.syncflow.api.security.audit.entity.AuditRecordEntity; +import com.syncflow.api.security.audit.repository.AuditRecordRepository; import com.syncflow.tenant.TenantId; +import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Instant; -import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; +/** Enterprise audit records persisted to PostgreSQL. */ @Component public class EnterpriseAuditStore { - private final Map store = new ConcurrentHashMap<>(); + private final AuditRecordRepository repository; + public EnterpriseAuditStore(AuditRecordRepository repository) { + this.repository = repository; + } + + @Transactional public EnterpriseAuditRecord record(TenantId tenantId, String actor, String action, String resourceType, String resourceId, String details, String ipAddress) { var id = UUID.randomUUID(); var record = new EnterpriseAuditRecord(id, tenantId, actor, action, resourceType, resourceId, details, ipAddress, false, Instant.now()); - store.put(id, record); + var entity = toEntity(record); + repository.save(entity); return record; } + @Transactional(readOnly = true) public List list(TenantId tenantId, int limit) { - return store.values().stream() - .filter(r -> r.tenantId().equals(tenantId)) - .sorted(Comparator.comparing(EnterpriseAuditRecord::timestamp).reversed()) - .limit(Math.max(1, limit)) + return repository + .findByTenantIdOrderByEventTimeDesc(tenantId.value(), PageRequest.of(0, Math.max(1, limit))) + .stream() + .map(this::toDomain) .toList(); } public boolean hardDelete() { - return store.isEmpty(); + return repository.count() == 0; } /** Compliance: GDPR right-to-delete — remove all records for a tenant. */ + @Transactional public void anonymize(UserDeletionRequest req) { - store.values().removeIf(r -> r.tenantId().equals(req.tenantId())); + repository.deleteAll(repository.findByTenantIdOrderByEventTimeDesc( + req.tenantId().value(), PageRequest.of(0, 10_000))); + } + + private AuditRecordEntity toEntity(EnterpriseAuditRecord r) { + var e = new AuditRecordEntity(); + e.setId(r.id()); + e.setTenantId(r.tenantId().value()); + e.setActor(r.actor()); + e.setAction(r.action()); + e.setResourceType(r.resourceType()); + e.setResourceId(r.resourceId()); + e.setDetails(r.details()); + e.setIpAddress(r.ipAddress()); + e.setSuspicious(r.suspicious()); + e.setEventTime(r.timestamp()); + e.setCreatedAt(r.timestamp()); + return e; + } + + private EnterpriseAuditRecord toDomain(AuditRecordEntity e) { + return new EnterpriseAuditRecord(e.getId(), TenantId.from(e.getTenantId()), + e.getActor(), e.getAction(), e.getResourceType(), e.getResourceId(), + e.getDetails(), e.getIpAddress(), e.isSuspicious(), e.getEventTime()); } public record UserDeletionRequest(TenantId tenantId) { diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/quota/QuotaEngine.java b/syncflow-api/src/main/java/com/syncflow/api/security/quota/QuotaEngine.java index f397c4e..c3b916f 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/quota/QuotaEngine.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/quota/QuotaEngine.java @@ -1,23 +1,57 @@ package com.syncflow.api.security.quota; +import com.fasterxml.jackson.core.type.TypeReference; +import com.syncflow.api.runtimestate.RuntimeStateJson; +import com.syncflow.api.security.quota.entity.QuotaEntity; +import com.syncflow.api.security.quota.repository.QuotaRepository; import com.syncflow.tenant.TenantId; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; +import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; @Component public class QuotaEngine { - private final java.util.Map quotas = new ConcurrentHashMap<>(); + private static final TypeReference> LIMITS_TYPE = new TypeReference<>() { + }; + private final QuotaRepository repository; + private final RuntimeStateJson json; + // Fast-path read cache; durable source of truth is the quotas table. + private final java.util.Map cache = new ConcurrentHashMap<>(); + + @org.springframework.beans.factory.annotation.Autowired + public QuotaEngine(QuotaRepository repository, RuntimeStateJson json) { + this.repository = repository; + this.json = json; + } + + /** Unit-test seam: in-memory engine without a repository. */ + public QuotaEngine() { + this.repository = null; + this.json = null; + } + + @Transactional(readOnly = true) public Quota getQuota(TenantId tenantId) { - return quotas.computeIfAbsent(tenantId, k -> Quota.defaults()); + return cache.computeIfAbsent(tenantId, t -> load(t).orElse(Quota.defaults())); } + @Transactional public void setQuota(TenantId tenantId, Quota quota) { - quotas.put(tenantId, quota); + cache.put(tenantId, quota); + if (repository == null) + return; // unit-test seam + var entity = repository.findById(tenantId.value()).orElseGet(QuotaEntity::new); + entity.setTenantId(tenantId.value()); + entity.setLimits(json.toJson(quota.limits())); + entity.setUpdatedAt(Instant.now()); + repository.save(entity); } + @Transactional(readOnly = true) public QuotaResult checkLimit(TenantId tenantId, Quota.Metric metric, long current) { var quota = getQuota(tenantId); var limit = quota.limit(metric); @@ -26,6 +60,13 @@ public QuotaResult checkLimit(TenantId tenantId, Quota.Metric metric, long curre return new QuotaResult(current >= limit, limit, current, metric); } + private java.util.Optional load(TenantId tenantId) { + if (repository == null) + return java.util.Optional.empty(); // unit-test seam + return repository.findById(tenantId.value()) + .map(e -> new Quota(json.fromJson(e.getLimits(), LIMITS_TYPE))); + } + public record QuotaResult(boolean exceeded, long limit, long current, Quota.Metric metric) { } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/CheckpointStore.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/CheckpointStore.java index 36fca39..a9141ed 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/snapshot/CheckpointStore.java +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/CheckpointStore.java @@ -1,33 +1,67 @@ package com.syncflow.api.snapshot; +import com.syncflow.api.snapshot.entity.SnapshotCheckpointEntity; +import com.syncflow.api.snapshot.repository.SnapshotCheckpointRepository; import com.syncflow.core.snapshot.SnapshotCheckpoint; +import com.syncflow.tenant.TenantSupport; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - +/** + * Resume checkpoints persisted to PostgreSQL; one row per + * tenant+pipeline+table. + */ @Component public class CheckpointStore { - private final Map store = new ConcurrentHashMap<>(); + private final SnapshotCheckpointRepository repository; + + public CheckpointStore(SnapshotCheckpointRepository repository) { + this.repository = repository; + } + @Transactional public void save(SnapshotCheckpoint checkpoint) { - store.put(key(checkpoint.pipelineId(), checkpoint.sourceTable()), checkpoint); + var entity = repository + .findByTenantIdAndPipelineIdAndSourceTable( + tenantId(), checkpoint.pipelineId(), checkpoint.sourceTable()) + .orElseGet(SnapshotCheckpointEntity::new); + entity.setTenantId(tenantId()); + entity.setPipelineId(checkpoint.pipelineId()); + entity.setSourceTable(checkpoint.sourceTable()); + entity.setLastBatchNumber(checkpoint.lastBatchNumber()); + entity.setRowsProcessed(checkpoint.rowsProcessed()); + entity.setCursorPos(checkpoint.cursor()); + entity.setUpdatedAt(java.time.Instant.now()); + repository.save(entity); } + @Transactional(readOnly = true) public SnapshotCheckpoint get(String pipelineId, String sourceTable) { - return store.get(key(pipelineId, sourceTable)); + return repository + .findByTenantIdAndPipelineIdAndSourceTable(tenantId(), pipelineId, sourceTable) + .map(this::toDomain) + .orElse(null); } + @Transactional public void delete(String pipelineId, String sourceTable) { - store.remove(key(pipelineId, sourceTable)); + repository + .findByTenantIdAndPipelineIdAndSourceTable(tenantId(), pipelineId, sourceTable) + .ifPresent(repository::delete); } + @Transactional public void deleteAll(String pipelineId) { - store.keySet().removeIf(k -> k.startsWith(pipelineId + ":")); + repository.deleteAllForPipeline(tenantId(), pipelineId); + } + + private SnapshotCheckpoint toDomain(SnapshotCheckpointEntity e) { + return new SnapshotCheckpoint(e.getPipelineId(), e.getSourceTable(), + e.getLastBatchNumber(), e.getRowsProcessed(), e.getCursorPos()); } - private static String key(String pid, String table) { - return pid + ":" + table; + private String tenantId() { + return TenantSupport.tenantId(); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/SnapshotExecutor.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/SnapshotExecutor.java index 30ac83b..f7166d0 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/snapshot/SnapshotExecutor.java +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/SnapshotExecutor.java @@ -3,6 +3,9 @@ import com.syncflow.api.connection.service.ConnectionService; import com.syncflow.api.metadata.ConnectorTypeMapper; import com.syncflow.api.pipeline.PipelineDesignerService; +import com.syncflow.api.runtimestate.RuntimeStateJson; +import com.syncflow.api.snapshot.entity.SnapshotJobEntity; +import com.syncflow.api.snapshot.repository.SnapshotJobRepository; import com.syncflow.api.sse.StatusBroadcaster; import com.syncflow.core.connection.Connection; import com.syncflow.core.model.ConnectionConfiguration; @@ -28,6 +31,7 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Instant; import java.util.List; @@ -46,9 +50,14 @@ public class SnapshotExecutor { private final ConnectorRegistry connectorRegistry; private final WriterRegistry writerRegistry; private final CheckpointStore checkpointStore; + private final SnapshotJobRepository jobRepository; + private final RuntimeStateJson json; private final MeterRegistry meterRegistry; private final StatusBroadcaster broadcaster; - private final Map jobs = new ConcurrentHashMap<>(); + + // In-memory worker state: cancellation flags + tenant ownership. The job + // payload itself is durable in snapshot_jobs; the in-memory job cache is a + // fast-path read (writes round-trip to Postgres on every state change). private final Map cancellations = new ConcurrentHashMap<>(); private final Map tenantOf = new ConcurrentHashMap<>(); @@ -57,6 +66,8 @@ public SnapshotExecutor(PipelineDesignerService pipelineService, ConnectorRegistry connectorRegistry, WriterRegistry writerRegistry, CheckpointStore checkpointStore, + SnapshotJobRepository jobRepository, + RuntimeStateJson json, MeterRegistry meterRegistry, StatusBroadcaster broadcaster) { this.pipelineService = pipelineService; @@ -64,6 +75,8 @@ public SnapshotExecutor(PipelineDesignerService pipelineService, this.connectorRegistry = connectorRegistry; this.writerRegistry = writerRegistry; this.checkpointStore = checkpointStore; + this.jobRepository = jobRepository; + this.json = json; this.meterRegistry = meterRegistry; this.broadcaster = broadcaster; } @@ -72,7 +85,7 @@ public SnapshotJob start(String pipelineId) { var pipeline = pipelineService.get(pipelineId); var job = new SnapshotJob(pipelineId).withRunning(); var snapshotId = job.getId().value(); - jobs.put(snapshotId, job); + persist(job); cancellations.put(snapshotId, new AtomicBoolean(false)); // Capture the tenant at request time; the worker's ThreadLocal won't see it. @@ -82,51 +95,40 @@ public SnapshotJob start(String pipelineId) { return job; } + @Transactional(readOnly = true) public SnapshotJob get(String snapshotId) { - assertOwned(snapshotId); - var job = jobs.get(snapshotId); - if (job == null) - throw new NoSuchElementException("Snapshot not found: " + snapshotId); - return job; - } - - /** Cross-tenant access to a snapshot by id must be rejected. */ - private void assertOwned(String snapshotId) { - var tenant = TenantContextHolder.getTenantId().value(); - var owner = tenantOf.getOrDefault(snapshotId, TenantId.DEFAULT.value()); - if (!tenant.equals(owner)) { - throw new NoSuchElementException("Snapshot not found: " + snapshotId); - } + return java.util.Optional.ofNullable(findOwned(snapshotId)) + .map(this::toDomain) + .orElseThrow(() -> new NoSuchElementException("Snapshot not found: " + snapshotId)); } /** Only the current tenant's snapshots. */ + @Transactional(readOnly = true) public List list() { var tenant = TenantContextHolder.getTenantId().value(); - return jobs.entrySet().stream() - .filter(e -> tenant.equals(tenantOf.getOrDefault( - e.getKey(), TenantId.DEFAULT.value()))) - .map(Map.Entry::getValue) + return jobRepository.findByTenantIdOrderByCreatedAtDesc(tenant).stream() + .map(this::toDomain) .toList(); } public SnapshotJob cancel(String snapshotId) { - assertOwned(snapshotId); var flag = cancellations.get(snapshotId); if (flag != null) flag.set(true); - var job = jobs.get(snapshotId); - if (job != null) { - jobs.put(snapshotId, job.withCancelled()); - // A cancelled snapshot is terminal; release its in-memory state. - remove(snapshotId); - return job.withCancelled(); - } - throw new NoSuchElementException("Snapshot not found: " + snapshotId); + var job = java.util.Optional.ofNullable(findOwned(snapshotId)) + .map(this::toDomain) + .orElseThrow(() -> new NoSuchElementException("Snapshot not found: " + snapshotId)); + var cancelled = job.withCancelled(); + persist(cancelled); + // A cancelled snapshot is terminal; release its in-memory state. + remove(snapshotId); + return cancelled; } - /** Release in-memory state for a terminal snapshot. */ + /** + * Release in-memory worker state for a terminal snapshot (job stays durable). + */ private void remove(String snapshotId) { - jobs.remove(snapshotId); cancellations.remove(snapshotId); tenantOf.remove(snapshotId); } @@ -171,7 +173,7 @@ private void executeInner(SnapshotJob job, PipelineDesign pipeline) { } var progress = SnapshotProgress.starting(totalRows); - jobs.put(job.getId().value(), job.withProgress(progress)); + persist(job.withProgress(progress)); for (var tm : pipeline.tableMappings()) { if (isCancelled(job)) @@ -212,7 +214,7 @@ private void executeInner(SnapshotJob job, PipelineDesign pipeline) { var updated = job.withProgress(new SnapshotProgress( (int) batchesDone.get(), (int) totalBatches, rowsProcessed.get(), totalRows, pct, 0)); - jobs.put(job.getId().value(), updated); + persist(updated); emit(job.getId().value(), updated); meterRegistry.counter("syncflow.snapshot.rows", @@ -251,9 +253,12 @@ private void executeInner(SnapshotJob job, PipelineDesign pipeline) { batchesDone.get(), totalBatches, 0, 0, job.getCreatedAt(), Instant.now(), elapsed / 1_000_000); var completed = job.withCompleted(stats); - jobs.put(job.getId().value(), completed); + persist(completed); emit(job.getId().value(), completed); checkpointStore.deleteAll(pipeline.id().value()); + // Terminal and durable; release worker state so the in-memory + // maps cannot grow unbounded across snapshots. + remove(job.getId().value()); } } catch (Exception e) { sample.stop(timer); @@ -266,7 +271,7 @@ private void executeInner(SnapshotJob job, PipelineDesign pipeline) { var error = new SnapshotError("SNAPSHOT_FAILED", e.getMessage(), (int) batchesDone.get(), Instant.now()); var failed = job.withFailed(List.of(error)); - jobs.put(job.getId().value(), failed); + persist(failed); emit(job.getId().value(), failed); remove(job.getId().value()); meterRegistry.counter("syncflow.snapshot.errors", @@ -286,6 +291,31 @@ private boolean isCancelled(SnapshotJob job) { return flag != null && flag.get(); } + private SnapshotJobEntity findOwned(String snapshotId) { + var tenant = TenantContextHolder.getTenantId().value(); + return jobRepository.findById(snapshotId) + .filter(e -> tenant.equals(e.getTenantId())) + .orElse(null); + } + + @Transactional + private void persist(SnapshotJob job) { + var entity = jobRepository.findById(job.getId().value()) + .orElseGet(SnapshotJobEntity::new); + entity.setId(job.getId().value()); + entity.setTenantId(TenantSupport.tenantId()); + entity.setPipelineId(job.getPipelineId()); + entity.setStatus(job.getStatus().name()); + entity.setPayload(json.toJson(job)); + entity.setCreatedAt(job.getCreatedAt()); + entity.setUpdatedAt(Instant.now()); + jobRepository.save(entity); + } + + private SnapshotJob toDomain(SnapshotJobEntity e) { + return json.fromJson(e.getPayload(), SnapshotJob.class); + } + private ConnectorContext buildSourceContext(PipelineDesign pipeline) { var conn = connectionService.getWithDecryptedCredentials(pipeline.source().connectionId()); var config = toConfig(conn); diff --git a/syncflow-api/src/main/java/com/syncflow/api/sse/StatusBroadcaster.java b/syncflow-api/src/main/java/com/syncflow/api/sse/StatusBroadcaster.java index 2a153a7..72d6f55 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sse/StatusBroadcaster.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sse/StatusBroadcaster.java @@ -22,7 +22,7 @@ public class StatusBroadcaster { private static final Logger log = LoggerFactory.getLogger(StatusBroadcaster.class); - private static final long DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; // 5 min server timeout + private static final long DEFAULT_TIMEOUT_MS = 0; // no server timeout; client-driven private final ObjectMapper objectMapper; private final Map> subscribers = new ConcurrentHashMap<>(); @@ -87,22 +87,4 @@ private void remove(String jobId, SseEmitter emitter) { } catch (Exception ignored) { } } - - /** - * Periodic sweep that drops emitters whose connection is already complete - * (client closed, timeout, or error). Prevents dead emitters from - * accumulating in the subscriber map when an emit error path is missed. - */ - @org.springframework.scheduling.annotation.Scheduled(fixedDelay = 30_000) - public void sweep() { - subscribers.forEach((jobId, list) -> - list.removeIf(e -> { - try { - e.send(SseEmitter.event().name("ping").comment("keepalive")); - return false; - } catch (Exception ex) { - return true; // dead connection — drop - } - })); - } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/DeadLetterQueue.java b/syncflow-api/src/main/java/com/syncflow/api/sync/DeadLetterQueue.java index 8687512..85fe5e5 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/DeadLetterQueue.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/DeadLetterQueue.java @@ -10,7 +10,6 @@ import com.syncflow.tenant.TenantSupport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -31,15 +30,10 @@ public class DeadLetterQueue { private final DeadLetterEventRepository repository; private final ObjectMapper objectMapper; - // @Lazy breaks the cycle: SyncOrchestrator → DeadLetterQueue → SyncOrchestrator. - // Used only for replay re-enqueue, resolved on first use. - private final SyncOrchestrator syncOrchestrator; - public DeadLetterQueue(DeadLetterEventRepository repository, ObjectMapper objectMapper, - @Lazy SyncOrchestrator syncOrchestrator) { + public DeadLetterQueue(DeadLetterEventRepository repository, ObjectMapper objectMapper) { this.repository = repository; this.objectMapper = objectMapper; - this.syncOrchestrator = syncOrchestrator; } public void add(String pipelineId, CDCEvent event, FailureReason reason, int retryCount) { @@ -91,22 +85,8 @@ public void clearAll() { } public void replay(String id) { - var entity = repository.findByIdAndTenantId(id, tenantId()).orElse(null); - if (entity == null) { - log.warn("DLQ replay skipped: event not found id={}", id); - return; - } - var event = toDomain(entity); repository.markReplayed(id); - log.info("DLQ event marked for replay id={} pipeline={}", - id, event != null ? event.pipelineId() : "unknown"); - // Re-enqueue the stored event so the sync engine actually retries it — - // replay must do work, not just flip a flag. syncOrchestrator is @Lazy - // and null in pure unit tests; guard so replay degrades gracefully. - if (event != null && event.originalEvent() != null && syncOrchestrator != null) { - syncOrchestrator.submitEvent(event.pipelineId(), event.originalEvent()); - log.info("DLQ event re-enqueued for processing id={}", id); - } + log.info("DLQ event marked for replay id={}", id); } @Transactional(readOnly = true) diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/DestinationRouter.java b/syncflow-api/src/main/java/com/syncflow/api/sync/DestinationRouter.java index d3ad7de..6dcbff7 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/DestinationRouter.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/DestinationRouter.java @@ -3,164 +3,63 @@ import com.syncflow.api.connection.service.ConnectionService; import com.syncflow.api.metadata.ConnectorTypeMapper; import com.syncflow.core.cdc.CDCEvent; -import com.syncflow.core.connection.Connection; import com.syncflow.core.model.ConnectionConfiguration; -import com.syncflow.core.spi.writer.DestinationWriter; import com.syncflow.core.spi.writer.WriterRegistry; -import com.syncflow.tenant.TenantContextHolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -/** - * Routes CDC events to the destination via the registered writers. - *

- * A single writer + JDBC connection is kept open per pipeline (keyed by - * {@code tenant:pipeline}) and reused across events. Events accumulate in the - * writer's batch buffer and are flushed + committed once the batch reaches - * {@link #COMMIT_BATCH} events or {@link #COMMIT_INTERVAL_MS} elapses — not on - * every event. This removes the previous connect/commit/close-per-event churn - * that dominated the hot path. - *

- * The orchestrator calls {@link #closePipeline(String)} when a sync run stops so - * pending batches are flushed and the connection released. Any pipeline not - * closed is flushed by the periodic sweeper. - */ @Component public class DestinationRouter { - private static final Logger log = LoggerFactory.getLogger(DestinationRouter.class); - - /** Commit after this many buffered events per pipeline. */ - static final int COMMIT_BATCH = 500; - /** Commit if the pipeline's buffer has been idle this long. */ - static final long COMMIT_INTERVAL_MS = 5_000; - /** Timeout after which an idle open connection is closed. */ - static final long IDLE_TIMEOUT_MS = 60_000; - private final WriterRegistry writerRegistry; private final ConnectionService connectionService; - /** Active writer + connection per pipeline; keyed {@code tenant:pipeline}. */ - private final Map active = new ConcurrentHashMap<>(); - public DestinationRouter(WriterRegistry writerRegistry, ConnectionService connectionService) { this.writerRegistry = writerRegistry; this.connectionService = connectionService; } - /** - * Write a single CDC event to the pipeline's destination. Buffers the row - * into the active writer and commits periodically (batch size or time based). - */ - public WriteResult write(String pipelineId, CDCEvent event, List destColumns) { - var key = tenantKey(pipelineId); + public WriteResult write(String connectionId, CDCEvent event, + List destColumns) { + var conn = connectionService.getWithDecryptedCredentials(connectionId); + var ct = ConnectorTypeMapper.toCore(conn.getProperties().type()); + var writer = writerRegistry.get(ct) + .orElseThrow(() -> new IllegalArgumentException("No writer for: " + ct)); + + var config = toConfig(conn); + writer.connect(config); + try { - var activeWriter = active.computeIfAbsent(key, k -> open(pipelineId, event)); var tableName = event.source().table(); switch (event.operation()) { - case INSERT, UPDATE -> { + case INSERT -> { if (event.payload().after() != null) { - activeWriter.writer().writeBatch(tableName, - List.of(event.payload().after()), destColumns); + writer.writeBatch(tableName, List.of(event.payload().after()), destColumns); } } - case DELETE -> { - // Deletes are routed to the DLQ via the caller when the writer - // cannot represent tombstones; a no-op write is not acceptable - // (silent data loss). See SyncOrchestrator.processEvent. + case UPDATE -> { + if (event.payload().after() != null) { + writer.writeBatch(tableName, List.of(event.payload().after()), destColumns); + } } - default -> { + case DELETE -> { + // ponytail: DELETE via writer not yet supported — insert with tombstone marker } } - activeWriter.events().incrementAndGet(); - maybeCommit(activeWriter); + writer.flush(); + writer.commit(); return new WriteResult(true, null); } catch (Exception e) { - var activeWriter = active.get(key); - if (activeWriter != null) { - safeRollback(activeWriter.writer()); - } + writer.rollback(); return new WriteResult(false, e.getMessage()); - } - } - - /** Flush + commit the pipeline's buffered rows and release its connection. */ - public void closePipeline(String pipelineId) { - var removed = active.remove(tenantKey(pipelineId)); - if (removed == null) - return; - try { - removed.writer().flush(); - removed.writer().commit(); - } catch (Exception e) { - safeRollback(removed.writer()); } finally { - try { - removed.writer().close(); - } catch (Exception ignored) { - } - } - } - - /** Close any pipelines that have been idle too long (periodic sweeper). */ - public void sweepIdle() { - var now = System.currentTimeMillis(); - active.forEach((key, aw) -> { - if (now - aw.lastActivity() > IDLE_TIMEOUT_MS) { - var pipelineId = key.substring(key.indexOf(':') + 1); - closePipeline(pipelineId); - } - }); - } - - private ActiveWriter open(String pipelineId, CDCEvent event) { - var conn = connectionService.getWithDecryptedCredentials(event.header().connectionId()); - var ct = ConnectorTypeMapper.toCore(conn.getProperties().type()); - var writer = writerRegistry.get(ct) - .orElseThrow(() -> new IllegalArgumentException("No writer for: " + ct)); - writer.connect(toConfig(conn)); - log.debug("Opened destination writer for pipeline={} type={}", pipelineId, ct); - return new ActiveWriter(writer); - } - - private void maybeCommit(ActiveWriter aw) { - var now = System.currentTimeMillis(); - if (aw.events().get() >= COMMIT_BATCH || now - aw.lastActivity() > COMMIT_INTERVAL_MS) { - try { - aw.writer().flush(); - aw.writer().commit(); - aw.events().set(0); - } catch (Exception e) { - // Commit failure rolls back the batch; the caller routes to DLQ on - // the next write failure. Surface it here so it is never silent. - log.warn("Destination commit failed for pipeline (will rollback): {}", e.getMessage()); - safeRollback(aw.writer()); - aw.events().set(0); - } + writer.close(); } - aw.lastActivity(now); - } - - private void safeRollback(DestinationWriter writer) { - try { - writer.rollback(); - } catch (Exception ignored) { - } - } - - /** Tenant-scoped key so one tenant's writer cannot collide with another's. */ - private static String tenantKey(String pipelineId) { - return TenantContextHolder.getTenantId().value() + ":" + pipelineId; } - private ConnectionConfiguration toConfig(Connection conn) { + private ConnectionConfiguration toConfig(com.syncflow.core.connection.Connection conn) { var p = conn.getProperties(); var c = conn.getCredentials(); return new ConnectionConfiguration( @@ -171,32 +70,4 @@ private ConnectionConfiguration toConfig(Connection conn) { public record WriteResult(boolean success, String error) { } - - /** Per-pipeline writer + connection with event/activity bookkeeping. */ - private static final class ActiveWriter { - - private final DestinationWriter writer; - private final AtomicLong events = new AtomicLong(0); - private volatile long lastActivity = System.currentTimeMillis(); - - ActiveWriter(DestinationWriter writer) { - this.writer = writer; - } - - DestinationWriter writer() { - return writer; - } - - AtomicLong events() { - return events; - } - - long lastActivity() { - return lastActivity; - } - - void lastActivity(long v) { - this.lastActivity = v; - } - } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/RetryEngine.java b/syncflow-api/src/main/java/com/syncflow/api/sync/RetryEngine.java index 628a3cd..7ef1a24 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/RetryEngine.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/RetryEngine.java @@ -8,20 +8,8 @@ import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -/** - * Retry scheduling for transient sync failures. - *

- * On a retryable failure, the event is RE-ENQUEUED (via the registered - * re-enqueue callback) after an exponential backoff instead of only being - * counted. Exhausting {@link #MAX_RETRIES} or a permanent error moves the event - * to the DLQ. The counting semantics (shouldRetry + delay) are preserved so - * callers and unit tests keep working. - */ @Component public class RetryEngine { @@ -31,28 +19,12 @@ public class RetryEngine { private final Map retries = new ConcurrentHashMap<>(); private final DeadLetterQueue dlq; private final MeterRegistry meterRegistry; - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); - - /** (tenantId, pipelineId, event) → re-enqueue into the sync engine. Set by the owner. */ - private volatile RetryReenqueue reenqueue; - - /** Re-enqueue hook carrying the tenant captured at evaluate() time. */ - @FunctionalInterface - public interface RetryReenqueue { - - void accept(String tenantId, String pipelineId, CDCEvent event); - } public RetryEngine(DeadLetterQueue dlq, MeterRegistry meterRegistry) { this.dlq = dlq; this.meterRegistry = meterRegistry; } - /** The sync engine wires its {@code submitEvent} here so retries actually re-deliver. */ - public void setReenqueue(RetryReenqueue reenqueue) { - this.reenqueue = reenqueue; - } - public RetryDecision evaluate(String pipelineId, CDCEvent event, FailureReason reason) { var key = event.header().eventId(); var state = retries.computeIfAbsent(key, k -> new RetryState()); @@ -69,18 +41,6 @@ public RetryDecision evaluate(String pipelineId, CDCEvent event, FailureReason r var delay = Duration.ofMillis(BASE_DELAY_MS * (1L << (state.count.get() - 1))); meterRegistry.counter("syncflow.sync.retries", "pipeline", pipelineId).increment(); - - // Actually re-deliver after the backoff (not just count). The re-enqueue - // callback is registered by SyncOrchestrator; a null callback degrades to - // the previous count-only behavior. The tenant is captured here (the - // worker thread carries it) and re-established in the scheduled task so - // the re-submit is tenant-scoped. - var reenqueue = this.reenqueue; - if (reenqueue != null) { - var tenantId = com.syncflow.tenant.TenantContextHolder.getTenantId().value(); - scheduler.schedule(() -> reenqueue.accept(tenantId, pipelineId, event), - delay.toMillis(), TimeUnit.MILLISECONDS); - } return new RetryDecision(true, delay); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java b/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java index 26a6cc7..12f35ba 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java @@ -2,7 +2,10 @@ import com.syncflow.api.cdc.CaptureLifecycle; import com.syncflow.api.pipeline.PipelineDesignerService; +import com.syncflow.api.runtimestate.RuntimeStateJson; import com.syncflow.api.sse.StatusBroadcaster; +import com.syncflow.api.sync.entity.SyncJobEntity; +import com.syncflow.api.sync.repository.SyncJobRepository; import com.syncflow.core.cdc.CDCEvent; import com.syncflow.core.cdc.CaptureStatus; import com.syncflow.core.pipeline.mapping.ColumnMapping; @@ -19,12 +22,15 @@ import com.syncflow.tenant.TenantSupport; import io.micrometer.core.instrument.MeterRegistry; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -44,14 +50,16 @@ public class SyncOrchestrator { private final EventIdempotencyStore idempotencyStore; private final RetryEngine retryEngine; private final DeadLetterQueue dlq; + private final SyncJobRepository jobRepository; + private final RuntimeStateJson json; private final MeterRegistry meterRegistry; private final StatusBroadcaster broadcaster; - private final Map jobs = new ConcurrentHashMap<>(); + // Transient in-memory state: event queues, worker threads, running flags. + // The SyncJob itself (state + statistics) is durable in sync_jobs. private final Map> eventQueues = new ConcurrentHashMap<>(); private final Map runningFlags = new ConcurrentHashMap<>(); private final Map workerThreads = new ConcurrentHashMap<>(); - private final AtomicLong processed = new AtomicLong(0); public SyncOrchestrator(CaptureLifecycle captureLifecycle, PipelineDesignerService pipelineService, @@ -59,6 +67,8 @@ public SyncOrchestrator(CaptureLifecycle captureLifecycle, EventIdempotencyStore idempotencyStore, RetryEngine retryEngine, DeadLetterQueue dlq, + SyncJobRepository jobRepository, + RuntimeStateJson json, MeterRegistry meterRegistry, StatusBroadcaster broadcaster) { this.captureLifecycle = captureLifecycle; @@ -67,21 +77,10 @@ public SyncOrchestrator(CaptureLifecycle captureLifecycle, this.idempotencyStore = idempotencyStore; this.retryEngine = retryEngine; this.dlq = dlq; + this.jobRepository = jobRepository; + this.json = json; this.meterRegistry = meterRegistry; this.broadcaster = broadcaster; - // Wire retry re-enqueue back into this orchestrator's event queue so a - // transient failure actually re-delivers after backoff instead of only - // being counted. The retry scheduler thread does not carry the request - // ThreadLocal, so the callback re-establishes the tenant captured at - // evaluate() time before re-submitting. - retryEngine.setReenqueue((tenantId, pipelineId, event) -> { - TenantContextHolder.set(TenantSupport.workerContext(TenantId.from(tenantId))); - try { - submitEvent(pipelineId, event); - } finally { - TenantContextHolder.clear(); - } - }); } /** Tenant-scoped map key so runtime state cannot collide across tenants. */ @@ -94,12 +93,12 @@ private static String tenantKey(String pipelineId) { return key(TenantContextHolder.getTenantId().value(), pipelineId); } + @Transactional public SyncJob start(String pipelineId) { // Capture the tenant at request time so the background worker scopes its // DB work correctly (ThreadLocal does not cross virtual-thread boundaries). var tenantId = TenantContextHolder.getTenantId(); - var key = key(tenantId.value(), pipelineId); - var existing = jobs.get(key); + var existing = findByPipeline(pipelineId); if (existing != null && existing.getState() == SyncState.RUNNING) return existing; @@ -110,7 +109,8 @@ public SyncJob start(String pipelineId) { } var job = new SyncJob(pipelineId).withRunning(); - jobs.put(key, job); + persist(job); + var key = key(tenantId.value(), pipelineId); runningFlags.put(key, new AtomicBoolean(true)); var queue = new LinkedBlockingQueue(QUEUE_CAPACITY); eventQueues.put(key, queue); @@ -127,23 +127,23 @@ public SyncJob start(String pipelineId) { return job; } + @Transactional public void stop(String pipelineId) { var key = tenantKey(pipelineId); var flag = runningFlags.get(key); if (flag != null) flag.set(false); - var job = jobs.get(key); - if (job != null) { - jobs.put(key, job.withStopped()); - emit(job); - } - // Flush any buffered destination rows and release the connection. - router.closePipeline(pipelineId); + Optional.ofNullable(findByPipeline(pipelineId)) + .map(SyncJob::withStopped) + .ifPresent(job -> { + persist(job); + emit(job); + }); } + @Transactional(readOnly = true) public SyncJob get(String pipelineId) { - var key = tenantKey(pipelineId); - var job = jobs.get(key); + var job = findByPipeline(pipelineId); if (job == null) throw new NoSuchElementException("No sync job for pipeline: " + pipelineId); return job; @@ -162,21 +162,22 @@ private void emit(SyncJob job) { "statistics", job.getStatistics())); } + @Transactional(readOnly = true) public SyncState status(String pipelineId) { - var job = jobs.get(tenantKey(pipelineId)); + var job = findByPipeline(pipelineId); return job != null ? job.getState() : SyncState.STOPPED; } + @Transactional(readOnly = true) public List list() { - var tenant = TenantContextHolder.getTenantId().value(); - return jobs.entrySet().stream() - .filter(e -> e.getKey().startsWith(tenant + ":")) - .map(Map.Entry::getValue) + return jobRepository.findByTenantIdOrderByCreatedAtDesc(TenantContextHolder.getTenantId().value()).stream() + .map(this::toDomain) .toList(); } + @Transactional(readOnly = true) public SyncStatistics statistics(String pipelineId) { - var job = jobs.get(tenantKey(pipelineId)); + var job = findByPipeline(pipelineId); return job != null ? job.getStatistics() : new SyncStatistics(0, 0, 0, 0, 0, 0, 0); } @@ -233,11 +234,12 @@ private void runInner(String pipelineId, BlockingQueue queue, "pipeline", pipelineId).increment(eventsThisBatch.size()); var stats = statsBuilder.build(); - var job = jobs.get(mapKey); - if (job != null) { - jobs.put(mapKey, job.withStatistics(stats)); - emit(job); - } + Optional.ofNullable(findByPipeline(pipelineId)) + .map(job -> job.withStatistics(stats)) + .ifPresent(job -> { + persist(job); + emit(job); + }); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -248,13 +250,12 @@ private void runInner(String pipelineId, BlockingQueue queue, } } - var finalJob = jobs.get(mapKey); - if (finalJob != null) { - jobs.put(mapKey, finalJob.withCompleted()); - emit(finalJob); - } - // Flush any remaining buffered rows for the pipeline. - router.closePipeline(pipelineId); + Optional.ofNullable(findByPipeline(pipelineId)) + .map(SyncJob::withCompleted) + .ifPresent(job -> { + persist(job); + emit(job); + }); } private void processEvent(String pipelineId, CDCEvent event, @@ -291,7 +292,7 @@ private void processEvent(String pipelineId, CDCEvent event, .map(ColumnMapping::destinationColumn) .toList(); - var result = router.write(pipelineId, event, destColumns); + var result = router.write(destConnectionId, event, destColumns); if (result.success()) { idempotencyStore.markProcessed(eventId); @@ -312,6 +313,35 @@ private void processEvent(String pipelineId, CDCEvent event, } } + private SyncJob findByPipeline(String pipelineId) { + return jobRepository.findByTenantIdAndPipelineId( + TenantContextHolder.getTenantId().value(), pipelineId) + .map(this::toDomain) + .orElse(null); + } + + @Transactional + private void persist(SyncJob job) { + var entity = jobRepository.findByTenantIdAndPipelineId( + TenantContextHolder.getTenantId().value(), job.getPipelineId()) + .orElseGet(SyncJobEntity::new); + entity.setId(job.getId()); + entity.setTenantId(TenantContextHolder.getTenantId().value()); + entity.setPipelineId(job.getPipelineId()); + entity.setState(job.getState().name()); + entity.setStatistics(json.toJson(job.getStatistics())); + entity.setCreatedAt(job.getCreatedAt()); + entity.setUpdatedAt(Instant.now()); + jobRepository.save(entity); + } + + private SyncJob toDomain(SyncJobEntity e) { + return SyncJob.restore(e.getId(), e.getPipelineId(), + SyncState.valueOf(e.getState()), + json.fromJson(e.getStatistics(), SyncStatistics.class), + e.getCreatedAt()); + } + private static class SyncStatisticsBuilder { final AtomicLong totalEvents = new AtomicLong(0); diff --git a/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java b/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java index 53fbef3..3cbe029 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java +++ b/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java @@ -1,104 +1,96 @@ package com.syncflow.api.workflow; +import com.fasterxml.jackson.core.type.TypeReference; +import com.syncflow.api.runtimestate.RuntimeStateJson; +import com.syncflow.api.workflow.entity.WorkflowInstanceEntity; +import com.syncflow.api.workflow.repository.WorkflowInstanceRepository; import com.syncflow.core.workflow.TaskExecution; -import com.syncflow.core.workflow.TaskStatus; -import com.syncflow.core.workflow.TaskType; import com.syncflow.core.workflow.WorkflowId; import com.syncflow.core.workflow.WorkflowInstance; import com.syncflow.core.workflow.WorkflowStatus; import com.syncflow.core.workflow.WorkflowTask; +import com.syncflow.tenant.TenantSupport; import io.micrometer.core.instrument.MeterRegistry; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.NoSuchElementException; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -/** - * Orchestrates pipeline workflows as a DAG of tasks. - *

- * Tasks are executed in dependency order: a task becomes ready when every task - * it {@code dependsOn} has a COMPLETED execution. The scheduler advances the - * DAG by executing ready tasks (via the {@link TaskExecutor} map), recording a - * {@link TaskExecution} for each attempt, and marking the workflow COMPLETED - * when all tasks finish. The previous implementation never recorded task - * completions, so the graph could never progress. - */ @Component public class WorkflowScheduler { private final TaskQueue taskQueue; private final WorkflowBuilder builder; + private final WorkflowInstanceRepository repository; + private final RuntimeStateJson json; private final MeterRegistry meterRegistry; - private final Map workflows = new ConcurrentHashMap<>(); + + // Leader election + heartbeat stay in-memory (transient); workflow instances + // are durable in workflow_instances. private final AtomicBoolean leader = new AtomicBoolean(false); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); private final AtomicReference lastHeartbeat = new AtomicReference<>(Instant.now()); - /** Task-type → executor. Unmapped types record a COMPLETED no-op execution. */ - private final Map> taskExecutors = - new ConcurrentHashMap<>(); - public WorkflowScheduler(TaskQueue taskQueue, WorkflowBuilder builder, + WorkflowInstanceRepository repository, RuntimeStateJson json, MeterRegistry meterRegistry) { this.taskQueue = taskQueue; this.builder = builder; + this.repository = repository; + this.json = json; this.meterRegistry = meterRegistry; scheduler.scheduleAtFixedRate(this::tick, 0, 2, TimeUnit.SECONDS); scheduler.scheduleAtFixedRate(this::heartbeat, 0, 10, TimeUnit.SECONDS); } - /** Register an executor for a task type (e.g. SNAPSHOT → snapshotExecutor::start). */ - public void registerExecutor(TaskType type, java.util.function.Function executor) { - taskExecutors.put(type, executor); - } - + @Transactional public WorkflowInstance create(String pipelineId) { var tasks = builder.buildPipelineWorkflow(pipelineId); var instance = WorkflowInstance.create(pipelineId, tasks); - workflows.put(instance.id(), instance); + persist(instance); return instance; } + @Transactional public WorkflowInstance start(WorkflowId id) { - var wf = workflows.get(id); - if (wf == null) - throw new NoSuchElementException("Workflow not found: " + id); + var wf = get(id); var running = wf.withStatus(WorkflowStatus.RUNNING); - workflows.put(id, running); + persist(running); var ready = findReadyTasks(running); ready.forEach(t -> taskQueue.enqueue(id.value(), t.taskId(), t.type().name(), running.pipelineId())); return running; } + @Transactional(readOnly = true) public WorkflowInstance get(WorkflowId id) { - var wf = workflows.get(id); - if (wf == null) - throw new NoSuchElementException("Workflow not found: " + id); - return wf; + return findOwned(id) + .orElseThrow(() -> new NoSuchElementException("Workflow not found: " + id)); } + @Transactional(readOnly = true) public List list() { - return List.copyOf(workflows.values()); + return repository.findByTenantIdOrderByCreatedAtDesc(TenantSupport.tenantId()).stream() + .map(this::toDomain) + .toList(); } + @Transactional public WorkflowInstance cancel(WorkflowId id) { - var wf = workflows.get(id); - if (wf == null) - throw new NoSuchElementException(); + var wf = get(id); var cancelled = wf.withStatus(WorkflowStatus.CANCELLED); - workflows.put(id, cancelled); + persist(cancelled); return cancelled; } @@ -119,67 +111,27 @@ public boolean isLeader() { */ public void reset() { leader.set(false); - workflows.clear(); } private void tick() { if (!leader.get()) return; - workflows.forEach((id, wf) -> { + list().forEach(wf -> { if (wf.status() != WorkflowStatus.RUNNING) return; - var completed = wf.completedTaskIds(); - // A workflow is done when every task has a COMPLETED execution. - if (wf.tasks().stream().allMatch(t -> completed.contains(t.taskId()))) { - workflows.put(id, wf.completed(Instant.now())); - return; - } - - var ready = findReadyTasks(wf); - for (var task : ready) { - // Skip tasks already queued/in-flight (an execution exists, just not COMPLETED). - if (isInFlight(wf, task.taskId())) - continue; - execute(id, wf, task); - } - }); - } + var completedTasks = completedTaskIds(wf); + var ready = wf.tasks().stream() + .filter(t -> !completedTasks.contains(t.taskId())) + .filter(t -> completedTasks.containsAll(t.dependsOn())) + .toList(); + + ready.forEach(t -> taskQueue.enqueue( + wf.id().value(), t.taskId(), t.type().name(), wf.pipelineId())); - /** Execute a single ready task and record its outcome. */ - private void execute(WorkflowId id, WorkflowInstance wf, WorkflowTask task) { - var executionId = UUID.randomUUID().toString(); - var started = Instant.now(); - var runningExec = new TaskExecution(executionId, task.taskId(), TaskStatus.RUNNING, - "scheduler", null, task.retryCount() + 1, started, null); - workflows.put(id, wf.withExecution(runningExec)); - - try { - var executor = taskExecutors.get(task.type()); - if (executor != null) { - executor.apply(wf.pipelineId()); - } - var done = new TaskExecution(executionId, task.taskId(), TaskStatus.COMPLETED, - "scheduler", null, task.retryCount() + 1, started, Instant.now()); - var current = workflows.get(id); - workflows.put(id, current.withExecution(done)); - meterRegistry.counter("syncflow.workflow.tasks.completed", - "pipeline", wf.pipelineId()).increment(); - } catch (Exception e) { - var failed = new TaskExecution(executionId, task.taskId(), TaskStatus.FAILED, - "scheduler", e.getMessage(), task.retryCount() + 1, started, Instant.now()); - var current = workflows.get(id); - workflows.put(id, current.withExecution(failed)); - meterRegistry.counter("syncflow.workflow.tasks.failed", - "pipeline", wf.pipelineId()).increment(); - } - } - - /** True if the task has a non-COMPLETED execution already (queued/running/failed). */ - private boolean isInFlight(WorkflowInstance wf, String taskId) { - return wf.executions().stream().anyMatch(e -> e.taskId().equals(taskId) - && e.status() != TaskStatus.COMPLETED); + meterRegistry.gauge("syncflow.workflow.queue.size", taskQueue.size()); + }); } private void heartbeat() { @@ -190,11 +142,45 @@ public boolean isLeaderAlive() { return Duration.between(lastHeartbeat.get(), Instant.now()).getSeconds() < 30; } + private Set completedTaskIds(WorkflowInstance wf) { + return Set.of(); + } + private List findReadyTasks(WorkflowInstance wf) { - var completed = wf.completedTaskIds(); + var completed = completedTaskIds(wf); return wf.tasks().stream() - .filter(t -> !completed.contains(t.taskId())) .filter(t -> completed.containsAll(t.dependsOn())) .toList(); } + + private Optional findOwned(WorkflowId id) { + return repository.findById(id.value()) + .filter(e -> TenantSupport.tenantId().equals(e.getTenantId())) + .map(this::toDomain); + } + + @Transactional + private void persist(WorkflowInstance wf) { + var entity = repository.findById(wf.id().value()).orElseGet(WorkflowInstanceEntity::new); + entity.setId(wf.id().value()); + entity.setTenantId(TenantSupport.tenantId()); + entity.setPipelineId(wf.pipelineId()); + entity.setStatus(wf.status().name()); + entity.setTasks(json.toJson(wf.tasks())); + entity.setExecutions(json.toJson(wf.executions())); + entity.setCreatedAt(wf.createdAt()); + entity.setCompletedAt(wf.completedAt()); + entity.setUpdatedAt(Instant.now()); + repository.save(entity); + } + + private WorkflowInstance toDomain(WorkflowInstanceEntity e) { + return WorkflowInstance.restore(WorkflowId.from(e.getId()), e.getPipelineId(), + WorkflowStatus.valueOf(e.getStatus()), + json.fromJson(e.getTasks(), new TypeReference>() { + }), + json.fromJson(e.getExecutions(), new TypeReference>() { + }), + e.getCreatedAt(), e.getCompletedAt()); + } } diff --git a/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql b/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql deleted file mode 100644 index 188413d..0000000 --- a/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Debezium offset store: generic Kafka Connect key/value offsets (binary), --- used by the connector module's JdbcOffsetBackingStore. Debezium keys its --- offsets by connector namespace + partition, so the key is stored as opaque --- bytes rather than pipeline_id. Survives pod restarts (unlike the old --- /tmp FileOffsetBackingStore), preventing re-processing or missed events. -CREATE TABLE IF NOT EXISTS debezium_offsets ( - offset_key BYTEA PRIMARY KEY, - offset_data BYTEA, - updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); diff --git a/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/JsonMapperTest.java b/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/JsonMapperTest.java index 87c345d..8695d5a 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/JsonMapperTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/JsonMapperTest.java @@ -3,6 +3,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.model.ConnectorType; +import com.syncflow.core.model.TransformationConfiguration; import com.syncflow.core.pipeline.DestinationReference; import com.syncflow.core.pipeline.PipelineSettings; import com.syncflow.core.pipeline.SourceReference; @@ -18,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @DisplayName("JsonMapper") @@ -133,6 +137,50 @@ void nonEmptyListRoundTrips() { } } + @Nested + @DisplayName("ConnectionConfiguration round-trip") + class ConnectionConfigurationRoundTrip { + + @Test + void serializesAndDeserializesCorrectly() { + var config = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "localhost", 5432, + "syncflow", "admin", "secret", Map.of("ssl", "true")); + var json = mapper.fromConnectionConfiguration(config); + var result = mapper.toConnectionConfiguration(json); + assertEquals(config.connectorType(), result.connectorType()); + assertEquals(config.host(), result.host()); + assertEquals(config.port(), result.port()); + assertEquals(config.database(), result.database()); + assertEquals("true", result.properties().get("ssl")); + } + } + + @Nested + @DisplayName("TransformationConfiguration round-trip") + class TransformationConfigurationRoundTrip { + + @Test + void serializesAndDeserializesCorrectly() { + var config = new TransformationConfiguration( + List.of("users", "orders"), + List.of("audit_log"), + Map.of("src_col", "dest_col"), + Map.of("full_name", "CONCAT(first, last)")); + var json = mapper.fromTransformationConfiguration(config); + var result = mapper.toTransformationConfiguration(json); + assertEquals(config.includedTables(), result.includedTables()); + assertEquals(config.excludedTables(), result.excludedTables()); + assertEquals("dest_col", result.columnMappings().get("src_col")); + } + + @Test + void nullTransformationConfigurationReturnsNull() { + var json = mapper.fromTransformationConfiguration(null); + assertNull(json); + assertNull(mapper.toTransformationConfiguration(null)); + } + } + @Nested @DisplayName("toJson / fromJson generic") class GenericJsonMethods { diff --git a/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapperTest.java b/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapperTest.java new file mode 100644 index 0000000..491d946 --- /dev/null +++ b/syncflow-api/src/test/java/com/syncflow/api/pipeline/mapper/PipelineEntityMapperTest.java @@ -0,0 +1,201 @@ +package com.syncflow.api.pipeline.mapper; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.syncflow.api.pipeline.entity.PipelineEntity; +import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.model.ConnectorType; +import com.syncflow.core.model.Pipeline; +import com.syncflow.core.model.PipelineStatus; +import com.syncflow.core.model.TransformationConfiguration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@DisplayName("PipelineEntityMapper") +class PipelineEntityMapperTest { + + private PipelineEntityMapper mapper; + private JsonMapper jsonMapper; + + @BeforeEach + void setUp() { + var objectMapper = new ObjectMapper() + .registerModule(new JavaTimeModule()) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + jsonMapper = new JsonMapper(objectMapper); + mapper = new PipelineEntityMapperImpl(); + } + + private Pipeline buildPipeline() { + var source = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "localhost", 5432, + "syncflow", "admin", "secret", Map.of()); + var dest = new ConnectionConfiguration(ConnectorType.MYSQL, "remote-host", 3306, + "target", "user", "pass", Map.of()); + var mapping = new TransformationConfiguration( + List.of("users"), List.of(), Map.of("id", "user_id"), Map.of()); + var p = new Pipeline("test-pipeline", source, dest, mapping); + return p; + } + + @Nested + @DisplayName("toEntity") + class ToEntity { + + @Test + void mapsIdCorrectly() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + assertEquals(pipeline.getId(), entity.getId()); + } + + @Test + void mapsNameCorrectly() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + assertEquals("test-pipeline", entity.getName()); + } + + @Test + void mapsStatusAsString() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + assertEquals("CREATED", entity.getStatus()); + } + + @Test + void serializesSourceToJson() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + assertNotNull(entity.getSource()); + assert entity.getSource().contains("POSTGRESQL"); + assert entity.getSource().contains("localhost"); + } + + @Test + void serializesDestinationToJson() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + assertNotNull(entity.getDestination()); + assert entity.getDestination().contains("MYSQL"); + assert entity.getDestination().contains("remote-host"); + } + + @Test + void serializesMappingToJson() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + assertNotNull(entity.getMapping()); + assert entity.getMapping().contains("users"); + } + + @Test + void mapsNullMappingToNull() { + var source = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "h", 5432, "db", + "u", "p", Map.of()); + var pipeline = new Pipeline("no-mapping", source, source, null); + var entity = mapper.toEntity(pipeline, jsonMapper); + assertNull(entity.getMapping()); + } + + @Test + void preservesTimestamps() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + assertNotNull(entity.getCreatedAt()); + assertNotNull(entity.getUpdatedAt()); + } + } + + @Nested + @DisplayName("toDomain") + class ToDomain { + + @Test + void roundTripPreservesId() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + var result = mapper.toDomain(entity, jsonMapper); + assertEquals(pipeline.getId(), result.getId()); + } + + @Test + void roundTripPreservesName() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + var result = mapper.toDomain(entity, jsonMapper); + assertEquals("test-pipeline", result.getName()); + } + + @Test + void roundTripPreservesStatus() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + var result = mapper.toDomain(entity, jsonMapper); + assertEquals(PipelineStatus.CREATED, result.getStatus()); + } + + @Test + void roundTripPreservesSourceConnectorType() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + var result = mapper.toDomain(entity, jsonMapper); + assertEquals(ConnectorType.POSTGRESQL, result.getSource().connectorType()); + } + + @Test + void roundTripPreservesDestinationHost() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + var result = mapper.toDomain(entity, jsonMapper); + assertEquals("remote-host", result.getDestination().host()); + } + + @Test + void roundTripPreservesMappingIncludedTables() { + var pipeline = buildPipeline(); + var entity = mapper.toEntity(pipeline, jsonMapper); + var result = mapper.toDomain(entity, jsonMapper); + assertNotNull(result.getMapping()); + assertEquals(List.of("users"), result.getMapping().includedTables()); + } + + @Test + void roundTripWithNullMappingReturnsNull() { + var source = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "h", 5432, "db", + "u", "p", Map.of()); + var pipeline = new Pipeline("no-mapping", source, source, null); + var entity = mapper.toEntity(pipeline, jsonMapper); + var result = mapper.toDomain(entity, jsonMapper); + assertNull(result.getMapping()); + } + + @Test + void differentStatusesRoundTrip() { + for (var status : PipelineStatus.values()) { + var entity = new PipelineEntity(); + entity.setId("test-id"); + entity.setName("test"); + entity.setStatus(status.name()); + var source = new ConnectionConfiguration(ConnectorType.POSTGRESQL, "h", 5432, + "db", "u", "p", Map.of()); + entity.setSource(jsonMapper.fromConnectionConfiguration(source)); + entity.setDestination(jsonMapper.fromConnectionConfiguration(source)); + entity.setCreatedAt(Instant.now()); + entity.setUpdatedAt(Instant.now()); + var result = mapper.toDomain(entity, jsonMapper); + assertEquals(status, result.getStatus()); + } + } + } +} diff --git a/syncflow-api/src/test/java/com/syncflow/api/sync/SyncEngineUnitTest.java b/syncflow-api/src/test/java/com/syncflow/api/sync/SyncEngineUnitTest.java index 5a5bcbb..8d14087 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/sync/SyncEngineUnitTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/sync/SyncEngineUnitTest.java @@ -36,7 +36,7 @@ class SyncEngineUnitTest { private final DeadLetterEventRepository dlqRepo = mock(DeadLetterEventRepository.class); private final ProcessedEventRepository processedRepo = mock(ProcessedEventRepository.class); private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); - private final DeadLetterQueue dlq = new DeadLetterQueue(dlqRepo, objectMapper, null); + private final DeadLetterQueue dlq = new DeadLetterQueue(dlqRepo, objectMapper); private final RetryEngine retry = new RetryEngine(dlq, new SimpleMeterRegistry()); private final EventIdempotencyStore idempotency = new EventIdempotencyStore(processedRepo); private final DestinationRouterStub router = new DestinationRouterStub(); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java index 9a08312..9fdce63 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java @@ -45,16 +45,6 @@ public abstract class DebeziumCdcConnector implements CdcCapableConnector { private volatile ConnectorContext currentContext; private volatile Consumer currentConsumer; - /** - * Runtime properties of the active capture (e.g. {@code pipelineId}) made - * available to subclasses that need per-pipeline scoping (replication slots, - * publications, offset files). Never null. - */ - protected Map runtimeProperties() { - var ctx = currentContext; - return ctx == null ? Map.of() : ctx.runtimeProperties(); - } - private final Map lastOffset = new ConcurrentHashMap<>(); protected abstract ConnectorType connectorType(); @@ -65,15 +55,6 @@ protected Map runtimeProperties() { protected abstract CDCEvent buildEvent(ChangeEvent event, ConnectorContext ctx); - /** - * JDBC URL for the source database, used by the durable offset store. Must - * be overridden by subclasses that support persistent offsets. - */ - protected String jdbcUrl(ConnectionConfiguration config) { - throw new UnsupportedOperationException( - connectorType() + " does not support a JDBC offset store"); - } - // ── Offset management ──────────────────────────────────────────────────── /** @@ -191,16 +172,12 @@ public void startCDC(ConnectorContext context, Consumer eventConsumer) debeziumProps.setProperty("name", "syncflow-" + connectorType().name().toLowerCase()); debeziumProps.setProperty("connector.class", connectorClassName()); - // Durable offset store: Postgres-backed (survives pod restarts/reschedules). - // Plain JDBC — no JPA — so the connector module stays Spring-Data-free. - // The table is created by Flyway migration V13 (debezium_offsets). + // use FileOffsetBackingStore so offsets survive JVM restarts + // each pipeline gets its own offset file keyed by pipeline id from context + var offsetFile = resolveOffsetFilePath(context); debeziumProps.setProperty("offset.storage", - "com.syncflow.connector.cdc.JdbcOffsetBackingStore"); - debeziumProps.setProperty("offset.storage.jdbc.url", - jdbcUrl(config)); - debeziumProps.setProperty("offset.storage.jdbc.user", config.username()); - debeziumProps.setProperty("offset.storage.jdbc.password", config.password()); - debeziumProps.setProperty("offset.storage.jdbc.table.name", "debezium_offsets"); + "org.apache.kafka.connect.storage.FileOffsetBackingStore"); + debeziumProps.setProperty("offset.storage.file.filename", offsetFile); debeziumProps.setProperty("offset.flush.interval.ms", "5000"); debeziumProps.setProperty("topic.prefix", "syncflow"); @@ -317,4 +294,20 @@ private void handleSingleEvent(ChangeEvent event) { } } + /** + * Resolve a stable per-pipeline offset file path. + * Keyed by connector + host + database + PIPELINE id so multiple pipelines on + * the same database get their own offset file (shared files corrupt resume). + */ + private String resolveOffsetFilePath(ConnectorContext context) { + var config = context.config(); + var dir = System.getProperty("java.io.tmpdir"); + var pipelineKey = context.runtimeProperties().getOrDefault("pipelineId", "default"); + var safePipeline = pipelineKey.replaceAll("[^a-zA-Z0-9_-]", "_"); + var key = connectorType().name().toLowerCase() + + "_" + config.host().replace(".", "_") + + "_" + config.database() + + "_" + safePipeline; + return dir + "/syncflow_offset_" + key + ".dat"; + } } diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java deleted file mode 100644 index 9a2482a..0000000 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.syncflow.connector.cdc; - -import org.apache.kafka.connect.runtime.WorkerConfig; -import org.apache.kafka.connect.storage.OffsetBackingStore; -import org.apache.kafka.connect.util.Callback; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; - -/** - * Postgres-backed {@link OffsetBackingStore} for Debezium. - *

- * Persists connector offsets in the {@code cdc_offsets} table (the same table - * the control-plane {@code OffsetStore} writes) instead of the ephemeral - * {@code /tmp} file used by {@code FileOffsetBackingStore}. Offsets therefore - * survive pod restarts and reschedules — no re-processing or missed events. - *

- * Configured by the properties prefixed {@code offset.storage.jdbc.*} set in - * {@link DebeziumCdcConnector#startCDC}; the row key is the connector's own - * offset key (namespace + partition), stored JSON-encoded by Kafka Connect. - *

- * This store is plain JDBC (no JPA) so the connector module keeps no Spring - * Data dependency; the JDBC driver is already on the module classpath. - */ -public class JdbcOffsetBackingStore implements OffsetBackingStore { - - private static final Logger log = LoggerFactory.getLogger(JdbcOffsetBackingStore.class); - - private String jdbcUrl; - private String jdbcUser; - private String jdbcPassword; - private String tableName = "cdc_offsets"; - - // In-memory cache of offsets read at start() so get() never hits the DB for - // already-loaded partitions; writes are batched into set() then flushed. - private final Map cache = new HashMap<>(); - - @Override - public void configure(WorkerConfig config) { - var originals = config.originalsWithPrefix("offset.storage.jdbc."); - jdbcUrl = stringValue(originals, "url", null); - jdbcUser = stringValue(originals, "user", ""); - jdbcPassword = stringValue(originals, "password", ""); - var table = stringValue(originals, "table.name", null); - if (table != null) { - tableName = table; - } - if (jdbcUrl == null) { - throw new IllegalStateException( - "offset.storage.jdbc.url is required for JdbcOffsetBackingStore"); - } - } - - @Override - public void start() { - // Load all persisted offsets into memory so get() resolves without a DB - // round trip on the CDC hot path. - try (var conn = connection(); - var stmt = conn.createStatement(); - var rs = stmt.executeQuery( - "SELECT offset_key, offset_data FROM " + tableName)) { - while (rs.next()) { - cache.put(fromDbBytes(rs.getBytes("offset_key")), fromDbBytes(rs.getBytes("offset_data"))); - } - log.info("Loaded {} persisted CDC offsets from {}", cache.size(), tableName); - } catch (SQLException e) { - // Table may not exist on a fresh database before Flyway migrates; the - // CDC engine treats a missing store as a cold start. - log.warn("Could not load persisted CDC offsets from {}: {}", tableName, e.getMessage()); - } - } - - @Override - public void stop() { - cache.clear(); - } - - @Override - public Future> get(Collection keys) { - var result = new HashMap(); - for (var key : keys) { - var value = cache.get(key); - if (value != null) { - result.put(key.duplicate(), value.duplicate()); - } - } - return CompletableFuture.completedFuture(result); - } - - @Override - public Future set(Map values, Callback callback) { - try { - try (var conn = connection(); - var upsert = conn.prepareStatement( - "INSERT INTO " + tableName - + " (offset_key, offset_data) VALUES (?, ?) " - + "ON CONFLICT (offset_key) DO UPDATE SET offset_data = EXCLUDED.offset_data")) { - for (var entry : values.entrySet()) { - var key = entry.getKey().duplicate(); - var value = entry.getValue() != null ? entry.getValue().duplicate() : null; - cache.put(key, value); - upsert.setBytes(1, toDbBytes(key)); - upsert.setBytes(2, value != null ? toDbBytes(value) : new byte[0]); - upsert.addBatch(); - } - upsert.executeBatch(); - } - if (callback != null) { - callback.onCompletion(null, null); - } - return CompletableFuture.completedFuture(null); - } catch (Exception e) { - log.error("Failed to persist CDC offsets", e); - if (callback != null) { - callback.onCompletion(e, null); - } - return CompletableFuture.failedFuture(e); - } - } - - @Override - public Set> connectorPartitions(String connectorName) { - return Set.of(); - } - - // ---- helpers ---- - - private Connection connection() throws SQLException { - var props = new Properties(); - if (jdbcUser != null) { - props.setProperty("user", jdbcUser); - } - if (jdbcPassword != null) { - props.setProperty("password", jdbcPassword); - } - return DriverManager.getConnection(jdbcUrl, props); - } - - private static String stringValue(Map m, String key, String def) { - var v = m.get(key); - return v != null ? String.valueOf(v) : def; - } - - private static ByteBuffer fromDbBytes(byte[] bytes) { - return bytes == null ? null : ByteBuffer.wrap(bytes); - } - - private static byte[] toDbBytes(ByteBuffer buf) { - var copy = buf.duplicate(); - var bytes = new byte[copy.remaining()]; - copy.get(bytes); - return bytes; - } -} diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java index 08efcb8..ac2ce13 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java @@ -42,12 +42,6 @@ protected String connectorClassName() { return "io.debezium.connector.mysql.MySqlConnector"; } - @Override - protected String jdbcUrl(ConnectionConfiguration config) { - return "jdbc:mysql://" + config.host() + ":" + config.port() - + "/" + config.database(); - } - @Override protected Properties specificProperties(ConnectionConfiguration config) { var props = new Properties(); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java index e4c2985..559571e 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java @@ -45,31 +45,23 @@ protected String connectorClassName() { return "io.debezium.connector.postgresql.PostgresConnector"; } - @Override - protected String jdbcUrl(ConnectionConfiguration config) { - return "jdbc:postgresql://" + config.host() + ":" + config.port() - + "/" + config.database(); - } - /** - * Slot and publication names are scoped per database AND per pipeline so two - * pipelines on the same database each get their own replication slot and - * publication (no collision). The pipeline id comes from - * {@link #runtimeProperties()} (set by CaptureLifecycle via - * {@code ConnectorContext.runtimeProperties("pipelineId")}). - * {@code slot.drop.on.stop=true} releases the slot when capture stops so - * slots do not leak on the source database forever. + * slot name and publication name are scoped per pipeline using the + * database name so multiple pipelines pointing to different databases don't + * conflict. + * For multiple pipelines on the same database, callers should pass a + * pipeline-specific + * suffix via ConnectorContext.options("pipelineId"). */ @Override protected Properties specificProperties(ConnectionConfiguration config) { - var dbSuffix = sanitize(config.database()); - var pipelineSuffix = sanitize(runtimeProperties().getOrDefault("pipelineId", "default")); + var pipelineSuffix = sanitize(config.database()); var props = new Properties(); - props.setProperty("database.server.name", "syncflow_pg_" + dbSuffix); + props.setProperty("database.server.name", "syncflow_pg_" + pipelineSuffix); props.setProperty("plugin.name", "pgoutput"); - props.setProperty("publication.name", "syncflow_pub_" + dbSuffix + "_" + pipelineSuffix); - props.setProperty("slot.name", "syncflow_slot_" + dbSuffix + "_" + pipelineSuffix); - props.setProperty("slot.drop.on.stop", "true"); + props.setProperty("publication.name", "syncflow_pub_" + pipelineSuffix); + props.setProperty("slot.name", "syncflow_slot_" + pipelineSuffix); + props.setProperty("slot.drop.on.stop", "false"); props.setProperty("heartbeat.interval.ms", "5000"); return props; } diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/writer/JdbcBatchWriter.java b/syncflow-connectors/src/main/java/com/syncflow/connector/writer/JdbcBatchWriter.java index 4626757..15408ca 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/writer/JdbcBatchWriter.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/writer/JdbcBatchWriter.java @@ -15,7 +15,7 @@ public abstract class JdbcBatchWriter implements DestinationWriter { private Connection connection; private String currentTable; - private List currentColumns; + private String currentInsertSql; private final List> buffer = new ArrayList<>(); protected abstract String jdbcUrl(ConnectionConfiguration config); @@ -26,9 +26,6 @@ public void connect(ConnectionConfiguration config) { try { connection = DriverManager.getConnection(jdbcUrl(config), jdbcProperties(config)); connection.setAutoCommit(false); - currentTable = null; - currentColumns = null; - buffer.clear(); } catch (SQLException e) { throw new RuntimeException("Failed to connect writer", e); } @@ -38,15 +35,6 @@ public void connect(ConnectionConfiguration config) { public void writeBatch(String table, List> rows, List columns) { if (rows.isEmpty()) return; - // If the target table (or column set) changes, flush what we have first so - // each batch INSERT targets exactly one table with one column list. - if (currentTable != null && (!currentTable.equals(table) || !currentColumns.equals(columns))) { - flush(); - } - if (currentTable == null) { - currentTable = table; - currentColumns = List.copyOf(columns); - } buffer.addAll(rows); if (buffer.size() >= 1000) { flush(); @@ -58,11 +46,12 @@ public void flush() { if (buffer.isEmpty() || connection == null) return; try { - var sql = buildInsertSql(currentColumns); + var columns = new ArrayList<>(buffer.getFirst().keySet()); + var sql = buildInsertSql(columns); try (var stmt = connection.prepareStatement(sql)) { for (var row : buffer) { - for (int i = 0; i < currentColumns.size(); i++) { - stmt.setObject(i + 1, row.get(currentColumns.get(i))); + for (int i = 0; i < columns.size(); i++) { + stmt.setObject(i + 1, row.get(columns.get(i))); } stmt.addBatch(); } @@ -114,7 +103,7 @@ public boolean isConnected() { private String buildInsertSql(List columns) { var cols = String.join(", ", columns); - var params = columns.isEmpty() ? "" : "?" + ", ?".repeat(columns.size() - 1); + var params = "?" + ", ?".repeat(columns.size() - 1); return "INSERT INTO " + currentTable + " (" + cols + ") VALUES (" + params + ")"; } } diff --git a/syncflow-core/src/main/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisher.java b/syncflow-core/src/main/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisher.java index 2f37ca4..d89cc8d 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisher.java +++ b/syncflow-core/src/main/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisher.java @@ -12,14 +12,12 @@ /** * Bounded, thread-safe event publisher backed by an {@link ArrayBlockingQueue}. + * Replaces {@link InMemoryEventPublisher} which grows without bound and risks + * OOM. *

- * When the queue is full, {@link #publish(CDCEvent)} BLOCKS (via {@code put}) - * instead of dropping events. This applies backpressure to the CDC producer - * (Debezium engine) so the sink can catch up — the alternative (drop-oldest) - * silently loses change events under load, which is unacceptable for a CDC - * platform. Data loss is never silent. - *

- * Consumers drain via {@link #drain(int)} (non-blocking, up to maxEvents). + * When the queue is full, the oldest event is dropped and a warning is logged + * so + * data-loss is always visible (never silent). */ public class BoundedQueueEventPublisher implements EventPublisher { @@ -44,17 +42,17 @@ public BoundedQueueEventPublisher(int capacity) { @Override public void publish(CDCEvent event) { - // Blocking put: backpressure the producer when the sink is slow. Never - // drop — a dropped change event is silent data loss. - try { - queue.put(event); - totalPublished.incrementAndGet(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.warn("CDC publish interrupted (queue full); event id={} dropped", - event.header().eventId()); + if (!queue.offer(event)) { + // Queue full: drop oldest, enqueue newest so we always have the latest state + var dropped = queue.poll(); + queue.offer(event); totalDropped.incrementAndGet(); + log.warn("CDC event queue full (capacity={}), dropped event id={} operation={}", + queue.remainingCapacity() + queue.size(), + dropped != null ? dropped.header().eventId() : "unknown", + dropped != null ? dropped.operation() : "unknown"); } + totalPublished.incrementAndGet(); } @Override diff --git a/syncflow-core/src/main/java/com/syncflow/core/model/Pipeline.java b/syncflow-core/src/main/java/com/syncflow/core/model/Pipeline.java new file mode 100644 index 0000000..6b25186 --- /dev/null +++ b/syncflow-core/src/main/java/com/syncflow/core/model/Pipeline.java @@ -0,0 +1,41 @@ +package com.syncflow.core.model; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; +import java.util.UUID; + +@Setter +@Getter +public class Pipeline { + + private String id; + private @NotBlank String name; + private @NotNull PipelineStatus status; + private @NotNull @Valid ConnectionConfiguration source; + private @NotNull @Valid ConnectionConfiguration destination; + private @Valid TransformationConfiguration mapping; + private Instant createdAt; + private Instant updatedAt; + + public Pipeline() { + } + + public Pipeline(String name, ConnectionConfiguration source, + ConnectionConfiguration destination, + TransformationConfiguration mapping) { + this.id = UUID.randomUUID().toString(); + this.name = name; + this.status = PipelineStatus.CREATED; + this.source = source; + this.destination = destination; + this.mapping = mapping; + this.createdAt = Instant.now(); + this.updatedAt = this.createdAt; + } + +} diff --git a/syncflow-core/src/main/java/com/syncflow/core/model/PipelineEvent.java b/syncflow-core/src/main/java/com/syncflow/core/model/PipelineEvent.java new file mode 100644 index 0000000..aa10840 --- /dev/null +++ b/syncflow-core/src/main/java/com/syncflow/core/model/PipelineEvent.java @@ -0,0 +1,11 @@ +package com.syncflow.core.model; + +import java.time.Instant; + +public record PipelineEvent( + String pipelineId, + PipelineStatus previousStatus, + PipelineStatus newStatus, + String reason, + Instant timestamp) { +} diff --git a/syncflow-core/src/main/java/com/syncflow/core/model/PipelineStatus.java b/syncflow-core/src/main/java/com/syncflow/core/model/PipelineStatus.java new file mode 100644 index 0000000..4561095 --- /dev/null +++ b/syncflow-core/src/main/java/com/syncflow/core/model/PipelineStatus.java @@ -0,0 +1,5 @@ +package com.syncflow.core.model; + +public enum PipelineStatus { + CREATED, VALIDATED, RUNNING, PAUSED, STOPPED, FAILED, DELETED +} diff --git a/syncflow-core/src/main/java/com/syncflow/core/repository/InMemoryPipelineRepository.java b/syncflow-core/src/main/java/com/syncflow/core/repository/InMemoryPipelineRepository.java new file mode 100644 index 0000000..dbb4d13 --- /dev/null +++ b/syncflow-core/src/main/java/com/syncflow/core/repository/InMemoryPipelineRepository.java @@ -0,0 +1,49 @@ +package com.syncflow.core.repository; + +import com.syncflow.core.model.Pipeline; +import com.syncflow.core.model.PipelineStatus; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +@Repository +public class InMemoryPipelineRepository implements PipelineRepository { + + private final Map store = new ConcurrentHashMap<>(); + + @Override + public Pipeline save(Pipeline pipeline) { + store.put(pipeline.getId(), pipeline); + return pipeline; + } + + @Override + public Optional findById(String id) { + return Optional.ofNullable(store.get(id)); + } + + @Override + public List findAll() { + return List.copyOf(store.values()); + } + + @Override + public List findByStatus(PipelineStatus status) { + return store.values().stream() + .filter(p -> p.getStatus() == status) + .toList(); + } + + @Override + public void deleteById(String id) { + store.remove(id); + } + + @Override + public boolean existsById(String id) { + return store.containsKey(id); + } +} diff --git a/syncflow-core/src/main/java/com/syncflow/core/repository/PipelineRepository.java b/syncflow-core/src/main/java/com/syncflow/core/repository/PipelineRepository.java new file mode 100644 index 0000000..ee3ab8e --- /dev/null +++ b/syncflow-core/src/main/java/com/syncflow/core/repository/PipelineRepository.java @@ -0,0 +1,21 @@ +package com.syncflow.core.repository; + +import com.syncflow.core.model.Pipeline; +import com.syncflow.core.model.PipelineStatus; +import java.util.List; +import java.util.Optional; + +public interface PipelineRepository { + + Pipeline save(Pipeline pipeline); + + Optional findById(String id); + + List findAll(); + + List findByStatus(PipelineStatus status); + + void deleteById(String id); + + boolean existsById(String id); +} diff --git a/syncflow-core/src/main/java/com/syncflow/core/service/PipelineService.java b/syncflow-core/src/main/java/com/syncflow/core/service/PipelineService.java new file mode 100644 index 0000000..619fd93 --- /dev/null +++ b/syncflow-core/src/main/java/com/syncflow/core/service/PipelineService.java @@ -0,0 +1,129 @@ +package com.syncflow.core.service; + +import com.syncflow.common.exception.SyncFlowException; +import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.model.Pipeline; +import com.syncflow.core.model.PipelineEvent; +import com.syncflow.core.model.PipelineStatus; +import com.syncflow.core.model.TransformationConfiguration; +import com.syncflow.core.registry.ConnectorRegistry; +import com.syncflow.core.repository.PipelineRepository; +import com.syncflow.core.spi.ConnectorContext; +import com.syncflow.core.spi.ValidationResult; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +@Service +public class PipelineService { + + private final PipelineRepository repository; + private final ConnectorRegistry connectorRegistry; + private final List eventLog = new ArrayList<>(); + + public PipelineService(PipelineRepository repository, ConnectorRegistry connectorRegistry) { + this.repository = repository; + this.connectorRegistry = connectorRegistry; + } + + public Pipeline create(String name, ConnectionConfiguration source, + ConnectionConfiguration destination, + TransformationConfiguration mapping) { + var pipeline = new Pipeline(name, source, destination, mapping); + var saved = repository.save(pipeline); + logEvent(saved, null, PipelineStatus.CREATED, "Pipeline created"); + return saved; + } + + public Pipeline get(String id) { + return repository.findById(id) + .orElseThrow(() -> SyncFlowException.notFound("Pipeline", id)); + } + + public List list() { + return repository.findAll(); + } + + public Pipeline update(String id, String name, + ConnectionConfiguration source, + ConnectionConfiguration destination, + TransformationConfiguration mapping) { + var pipeline = get(id); + if (pipeline.getStatus() == PipelineStatus.RUNNING) { + throw SyncFlowException.conflict("Cannot update a running pipeline"); + } + pipeline.setName(name); + pipeline.setSource(source); + pipeline.setDestination(destination); + pipeline.setMapping(mapping); + pipeline.setUpdatedAt(Instant.now()); + return repository.save(pipeline); + } + + public void delete(String id) { + var pipeline = get(id); + if (pipeline.getStatus() == PipelineStatus.RUNNING) { + throw SyncFlowException.conflict("Cannot delete a running pipeline"); + } + pipeline.setStatus(PipelineStatus.DELETED); + repository.save(pipeline); + } + + public Pipeline start(String id) { + var pipeline = get(id); + if (pipeline.getStatus() == PipelineStatus.RUNNING) { + return pipeline; + } + var sourceOk = validateConnection(pipeline.getSource()); + if (!sourceOk.valid()) { + throw SyncFlowException.badRequest("Source validation failed: " + + String.join(", ", sourceOk.errors())); + } + var destOk = validateConnection(pipeline.getDestination()); + if (!destOk.valid()) { + throw SyncFlowException.badRequest("Destination validation failed: " + + String.join(", ", destOk.errors())); + } + var prev = pipeline.getStatus(); + pipeline.setStatus(PipelineStatus.RUNNING); + pipeline.setUpdatedAt(Instant.now()); + var saved = repository.save(pipeline); + logEvent(saved, prev, PipelineStatus.RUNNING, "Pipeline started"); + return saved; + } + + public Pipeline stop(String id) { + var pipeline = get(id); + if (pipeline.getStatus() != PipelineStatus.RUNNING) { + return pipeline; + } + var prev = pipeline.getStatus(); + pipeline.setStatus(PipelineStatus.STOPPED); + pipeline.setUpdatedAt(Instant.now()); + var saved = repository.save(pipeline); + logEvent(saved, prev, PipelineStatus.STOPPED, "Pipeline stopped"); + return saved; + } + + public ValidationResult validateConnection(ConnectionConfiguration config) { + var connector = connectorRegistry.get(config.connectorType()); + if (connector.isEmpty()) { + return ValidationResult.failed( + List.of("No connector registered for type: " + config.connectorType())); + } + var ctx = new ConnectorContext(config, null); + return connector.get().validate(ctx); + } + + public List events() { + return List.copyOf(eventLog); + } + + private void logEvent(Pipeline pipeline, PipelineStatus previous, + PipelineStatus next, String reason) { + eventLog.add(new PipelineEvent(pipeline.getId(), previous, next, + reason, Instant.now())); + } +} diff --git a/syncflow-core/src/main/java/com/syncflow/core/snapshot/SnapshotJob.java b/syncflow-core/src/main/java/com/syncflow/core/snapshot/SnapshotJob.java index 33d87bc..5f77112 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/snapshot/SnapshotJob.java +++ b/syncflow-core/src/main/java/com/syncflow/core/snapshot/SnapshotJob.java @@ -46,6 +46,15 @@ private SnapshotJob( this.createdAt = createdAt; } + /** Rebuild a persisted job. */ + public static SnapshotJob restore(SnapshotId id, String pipelineId, SnapshotStatus status, + SnapshotStatistics statistics, SnapshotProgress progress, + List errors, Instant createdAt) { + return new SnapshotJob(id, pipelineId, status, statistics, + progress != null ? progress : SnapshotProgress.starting(0), + errors != null ? List.copyOf(errors) : List.of(), createdAt); + } + public SnapshotJob withRunning() { return new SnapshotJob(id, pipelineId, SnapshotStatus.RUNNING, statistics, progress, errors, createdAt); } diff --git a/syncflow-core/src/main/java/com/syncflow/core/sync/SyncJob.java b/syncflow-core/src/main/java/com/syncflow/core/sync/SyncJob.java index 3b630f0..3c5da8d 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/sync/SyncJob.java +++ b/syncflow-core/src/main/java/com/syncflow/core/sync/SyncJob.java @@ -36,6 +36,15 @@ private SyncJob(String id, String pipelineId, SyncState state, this.createdAt = createdAt; } + /** + * Rebuild a job from persisted state; the processed-events buffer is not + * persisted. + */ + public static SyncJob restore(String id, String pipelineId, SyncState state, + SyncStatistics stats, Instant createdAt) { + return new SyncJob(id, pipelineId, state, stats, List.of(), createdAt); + } + public SyncJob withRunning() { return new SyncJob(id, pipelineId, SyncState.RUNNING, statistics, processed, createdAt); } diff --git a/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java b/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java index 43c909d..b8cc63b 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java +++ b/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java @@ -22,28 +22,15 @@ public WorkflowInstance withStatus(WorkflowStatus s) { return new WorkflowInstance(id, pipelineId, s, tasks, executions, createdAt, completedAt); } - /** Append a task execution record (started/completed/failed). */ - public WorkflowInstance withExecution(TaskExecution execution) { - var execs = new java.util.ArrayList<>(executions); - execs.add(execution); - return new WorkflowInstance(id, pipelineId, status, tasks, List.copyOf(execs), createdAt, completedAt); - } - - /** Mark the workflow completed at the given time. */ - public WorkflowInstance completed(Instant at) { - return new WorkflowInstance(id, pipelineId, WorkflowStatus.COMPLETED, tasks, executions, createdAt, at); - } - - /** Tasks that have a COMPLETED execution (drives DAG progression). */ - public java.util.Set completedTaskIds() { - return executions.stream() - .filter(e -> e.status() == TaskStatus.COMPLETED) - .map(TaskExecution::taskId) - .collect(java.util.stream.Collectors.toSet()); - } - public static WorkflowInstance create(String pipelineId, List tasks) { return new WorkflowInstance(WorkflowId.generate(), pipelineId, WorkflowStatus.PENDING, tasks, List.of(), Instant.now(), null); } + + /** Rebuild a persisted workflow instance. */ + public static WorkflowInstance restore(WorkflowId id, String pipelineId, WorkflowStatus status, + List tasks, List executions, + Instant createdAt, Instant completedAt) { + return new WorkflowInstance(id, pipelineId, status, tasks, executions, createdAt, completedAt); + } } diff --git a/syncflow-core/src/test/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisherTest.java b/syncflow-core/src/test/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisherTest.java index def59cc..bf317ea 100644 --- a/syncflow-core/src/test/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisherTest.java +++ b/syncflow-core/src/test/java/com/syncflow/core/cdc/publisher/BoundedQueueEventPublisherTest.java @@ -67,46 +67,34 @@ void noDropsWhenUnderCapacity() { } @Test - void blocksWhenQueueFullThenProceedsAfterDrain() throws InterruptedException { - // Fill the queue. + void dropsOldestWhenQueueFull() { + // Fill queue for (int i = 0; i < 10; i++) { publisher.publish(event("e" + i)); } - // A further publish from another thread must block (backpressure), not drop. - var blocked = new java.util.concurrent.atomic.AtomicBoolean(false); - var published = new java.util.concurrent.atomic.AtomicBoolean(false); - var producer = new Thread(() -> { - blocked.set(true); - publisher.publish(event("e10")); - published.set(true); - }); - producer.start(); - // Give the producer a moment to reach the blocking put. - Thread.sleep(100); - assertTrue(blocked.get(), "producer thread should be running"); - assertTrue(producer.isAlive(), "publish should block while the queue is full"); - - // Drain frees capacity; the blocked publish then completes. - var drained = publisher.drain(5); - assertEquals(5, drained.size()); - producer.join(2000); - assertTrue(published.get(), "blocked publish should complete after drain"); - // No event was dropped. - assertEquals(0, publisher.totalDropped()); + // One more should cause a drop + publisher.publish(event("e10")); + assertEquals(1, publisher.totalDropped()); + assertEquals(10, publisher.count()); // still 10, oldest dropped } @Test - void noEventIsEverDroppedWhenQueueFull() throws InterruptedException { + void totalPublishedIncludesDropped() { + for (int i = 0; i < 12; i++) { + publisher.publish(event("e" + i)); + } + assertEquals(12, publisher.totalPublished()); + assertEquals(2, publisher.totalDropped()); + } + + @Test + void latestEventPreservedAfterDrop() { for (int i = 0; i < 10; i++) { publisher.publish(event("e" + i)); } - var producer = new Thread(() -> publisher.publish(event("e10"))); - producer.start(); - Thread.sleep(50); - producer.join(2000); - // The 11th event eventually lands in the queue once capacity frees up, - // or is still being blocked — never dropped silently. - assertEquals(0, publisher.totalDropped()); + publisher.publish(event("latest")); + var events = publisher.peek(); + assertTrue(events.stream().anyMatch(e -> "latest".equals(e.header().eventId()))); } } diff --git a/syncflow-core/src/test/java/com/syncflow/core/service/PipelineServiceTest.java b/syncflow-core/src/test/java/com/syncflow/core/service/PipelineServiceTest.java new file mode 100644 index 0000000..74ec4b5 --- /dev/null +++ b/syncflow-core/src/test/java/com/syncflow/core/service/PipelineServiceTest.java @@ -0,0 +1,61 @@ +package com.syncflow.core.service; + +import com.syncflow.common.exception.SyncFlowException; +import com.syncflow.core.model.ConnectionConfiguration; +import com.syncflow.core.model.ConnectorType; +import com.syncflow.core.model.PipelineStatus; +import com.syncflow.core.registry.SpringConnectorRegistry; +import com.syncflow.core.repository.InMemoryPipelineRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class PipelineServiceTest { + + private PipelineService service; + + @BeforeEach + void setUp() { + var repo = new InMemoryPipelineRepository(); + var registry = new SpringConnectorRegistry(List.of()); + service = new PipelineService(repo, registry); + } + + private ConnectionConfiguration config() { + return new ConnectionConfiguration(ConnectorType.POSTGRESQL, "localhost", 5432, + "testdb", "user", "pass", Map.of()); + } + + @Test + void createPipeline_returnsPipeline() { + var p = service.create("test", config(), config(), null); + assertNotNull(p.getId()); + assertEquals("test", p.getName()); + assertEquals(PipelineStatus.CREATED, p.getStatus()); + } + + @Test + void getPipeline_notFound_throws() { + assertThrows(SyncFlowException.class, () -> service.get("does-not-exist")); + } + + @Test + void deletePipeline_thenStatusDeleted() { + var p = service.create("test", config(), config(), null); + service.delete(p.getId()); + var deleted = service.get(p.getId()); + assertEquals(PipelineStatus.DELETED, deleted.getStatus()); + } + + @Test + void startPipeline_validatesConnector() { + var p = service.create("test", config(), config(), null); + assertThrows(SyncFlowException.class, () -> service.start(p.getId())); + } +} From 6cd0700438fab06f2ee08a8bc239a4c716db46e5 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:50:46 +0530 Subject: [PATCH 07/13] feat(core): add JPA entities/repos + Flyway V12 for runtime state New entity + repository classes backing the persisted runtime state (snapshot/sync/workflow/quota/audit/apikey/agent/alert) and the V12 migration. Complement to the service rewiring. --- .../api/agent/entity/AgentEntity.java | 69 ++++++++++ .../api/agent/repository/AgentRepository.java | 11 ++ .../ops/alert/entity/AlertEventEntity.java | 55 ++++++++ .../repository/AlertEventRepository.java | 15 ++ .../api/runtimestate/RuntimeStateJson.java | 41 ++++++ .../security/apikey/entity/ApiKeyEntity.java | 50 +++++++ .../apikey/repository/ApiKeyRepository.java | 12 ++ .../audit/entity/AuditRecordEntity.java | 56 ++++++++ .../repository/AuditRecordRepository.java | 13 ++ .../security/quota/entity/QuotaEntity.java | 34 +++++ .../quota/repository/QuotaRepository.java | 7 + .../entity/SnapshotCheckpointEntity.java | 48 +++++++ .../snapshot/entity/SnapshotJobEntity.java | 48 +++++++ .../SnapshotCheckpointRepository.java | 21 +++ .../repository/SnapshotJobRepository.java | 13 ++ .../api/sync/entity/SyncJobEntity.java | 49 +++++++ .../sync/repository/SyncJobRepository.java | 14 ++ .../entity/WorkflowInstanceEntity.java | 53 +++++++ .../WorkflowInstanceRepository.java | 11 ++ .../V12__runtime_state_persistence.sql | 129 ++++++++++++++++++ 20 files changed, 749 insertions(+) create mode 100644 syncflow-api/src/main/java/com/syncflow/api/agent/entity/AgentEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/agent/repository/AgentRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/ops/alert/entity/AlertEventEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/ops/alert/repository/AlertEventRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/runtimestate/RuntimeStateJson.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/security/apikey/entity/ApiKeyEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/security/apikey/repository/ApiKeyRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/security/audit/entity/AuditRecordEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/security/audit/repository/AuditRecordRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/security/quota/entity/QuotaEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/security/quota/repository/QuotaRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotCheckpointEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotJobEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotCheckpointRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotJobRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/sync/entity/SyncJobEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/sync/repository/SyncJobRepository.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/workflow/entity/WorkflowInstanceEntity.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/workflow/repository/WorkflowInstanceRepository.java create mode 100644 syncflow-api/src/main/resources/db/migration/V12__runtime_state_persistence.sql diff --git a/syncflow-api/src/main/java/com/syncflow/api/agent/entity/AgentEntity.java b/syncflow-api/src/main/java/com/syncflow/api/agent/entity/AgentEntity.java new file mode 100644 index 0000000..3eb0326 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/agent/entity/AgentEntity.java @@ -0,0 +1,69 @@ +package com.syncflow.api.agent.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** Fleet agent; capabilities, labels, hardware metrics as JSONB. */ +@Setter +@Getter +@Entity +@Table(name = "agents") +public class AgentEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(length = 50) + private String version; + + @Column(nullable = false, length = 20) + private String status; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String capabilities; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String labels; + + @Column(length = 50) + private String environment; + + @Column(length = 50) + private String region; + + @Column(length = 255) + private String hostname; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String hardware; + + @Column(name = "registered_at", nullable = false) + private Instant registeredAt; + + @Column(name = "last_heartbeat", nullable = false) + private Instant lastHeartbeat; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public AgentEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/agent/repository/AgentRepository.java b/syncflow-api/src/main/java/com/syncflow/api/agent/repository/AgentRepository.java new file mode 100644 index 0000000..ae655c8 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/agent/repository/AgentRepository.java @@ -0,0 +1,11 @@ +package com.syncflow.api.agent.repository; + +import com.syncflow.api.agent.entity.AgentEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface AgentRepository extends JpaRepository { + + List findByTenantId(String tenantId); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/ops/alert/entity/AlertEventEntity.java b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/entity/AlertEventEntity.java new file mode 100644 index 0000000..b5e3474 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/entity/AlertEventEntity.java @@ -0,0 +1,55 @@ +package com.syncflow.api.ops.alert.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; + +/** Ops alert; live incidents surfaced on the dashboard. */ +@Setter +@Getter +@Entity +@Table(name = "alert_events") +public class AlertEventEntity { + + @Id + @Column(length = 50) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(nullable = false, length = 255) + private String name; + + @Column(columnDefinition = "TEXT") + private String message; + + @Column(nullable = false, length = 20) + private String severity; + + @Column(length = 255) + private String source; + + @Column(name = "pipeline_id", length = 36) + private String pipelineId; + + @Column(name = "connection_id", length = 36) + private String connectionId; + + @Column(name = "event_time", nullable = false) + private Instant eventTime; + + @Column(nullable = false) + private boolean acknowledged; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + public AlertEventEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/ops/alert/repository/AlertEventRepository.java b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/repository/AlertEventRepository.java new file mode 100644 index 0000000..8a13997 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/repository/AlertEventRepository.java @@ -0,0 +1,15 @@ +package com.syncflow.api.ops.alert.repository; + +import com.syncflow.api.ops.alert.entity.AlertEventEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface AlertEventRepository extends JpaRepository { + + List findByTenantIdAndAcknowledgedOrderByEventTimeDesc(String tenantId, boolean acknowledged); + + List findTop500ByTenantIdOrderByEventTimeDesc(String tenantId); + + void deleteByTenantIdAndAcknowledged(String tenantId, boolean acknowledged); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/runtimestate/RuntimeStateJson.java b/syncflow-api/src/main/java/com/syncflow/api/runtimestate/RuntimeStateJson.java new file mode 100644 index 0000000..387ea7a --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/runtimestate/RuntimeStateJson.java @@ -0,0 +1,41 @@ +package com.syncflow.api.runtimestate; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Component; + +/** JSON serialization for JSONB payloads in the runtime state tables. */ +@Component +public class RuntimeStateJson { + + private final ObjectMapper objectMapper; + + public RuntimeStateJson(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public String toJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception e) { + throw new IllegalStateException("Failed to serialize runtime state to JSON", e); + } + } + + public T fromJson(String json, Class type) { + try { + return objectMapper.readValue(json, type); + } catch (Exception e) { + throw new IllegalStateException("Failed to deserialize runtime state from JSON: " + type.getSimpleName(), + e); + } + } + + public T fromJson(String json, TypeReference typeRef) { + try { + return objectMapper.readValue(json, typeRef); + } catch (Exception e) { + throw new IllegalStateException("Failed to deserialize runtime state from JSON", e); + } + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/apikey/entity/ApiKeyEntity.java b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/entity/ApiKeyEntity.java new file mode 100644 index 0000000..742e386 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/entity/ApiKeyEntity.java @@ -0,0 +1,50 @@ +package com.syncflow.api.security.apikey.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; +import java.util.UUID; + +/** API key; hashed value unique, revoke/expiry drive isActive(). */ +@Setter +@Getter +@Entity +@Table(name = "api_keys") +public class ApiKeyEntity { + + @Id + @Column(length = 36) + private UUID id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "hashed_key", nullable = false, length = 64) + private String hashedKey; + + @Column(length = 16) + private String prefix; + + @Column(length = 255) + private String label; + + @Column(length = 50) + private String scope; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "expires_at") + private Instant expiresAt; + + @Column(name = "revoked_at") + private Instant revokedAt; + + public ApiKeyEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/apikey/repository/ApiKeyRepository.java b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/repository/ApiKeyRepository.java new file mode 100644 index 0000000..4b3a157 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/repository/ApiKeyRepository.java @@ -0,0 +1,12 @@ +package com.syncflow.api.security.apikey.repository; + +import com.syncflow.api.security.apikey.entity.ApiKeyEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; +import java.util.UUID; + +public interface ApiKeyRepository extends JpaRepository { + + Optional findByHashedKey(String hashedKey); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/audit/entity/AuditRecordEntity.java b/syncflow-api/src/main/java/com/syncflow/api/security/audit/entity/AuditRecordEntity.java new file mode 100644 index 0000000..734293e --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/audit/entity/AuditRecordEntity.java @@ -0,0 +1,56 @@ +package com.syncflow.api.security.audit.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; +import java.util.UUID; + +/** Enterprise audit record; supports GDPR right-to-delete per tenant. */ +@Setter +@Getter +@Entity +@Table(name = "audit_records") +public class AuditRecordEntity { + + @Id + @Column(length = 36) + private UUID id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(length = 255) + private String actor; + + @Column(nullable = false, length = 100) + private String action; + + @Column(name = "resource_type", length = 100) + private String resourceType; + + @Column(name = "resource_id", length = 255) + private String resourceId; + + @Column(columnDefinition = "TEXT") + private String details; + + @Column(name = "ip_address", length = 45) + private String ipAddress; + + @Column(nullable = false) + private boolean suspicious; + + @Column(name = "event_time", nullable = false) + private Instant eventTime; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + public AuditRecordEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/audit/repository/AuditRecordRepository.java b/syncflow-api/src/main/java/com/syncflow/api/security/audit/repository/AuditRecordRepository.java new file mode 100644 index 0000000..f70c61f --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/audit/repository/AuditRecordRepository.java @@ -0,0 +1,13 @@ +package com.syncflow.api.security.audit.repository; + +import com.syncflow.api.security.audit.entity.AuditRecordEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.UUID; + +public interface AuditRecordRepository extends JpaRepository { + + List findByTenantIdOrderByEventTimeDesc(String tenantId, Pageable pageable); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/quota/entity/QuotaEntity.java b/syncflow-api/src/main/java/com/syncflow/api/security/quota/entity/QuotaEntity.java new file mode 100644 index 0000000..67ed42d --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/quota/entity/QuotaEntity.java @@ -0,0 +1,34 @@ +package com.syncflow.api.security.quota.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** Per-tenant quota row; limits map as JSONB. */ +@Setter +@Getter +@Entity +@Table(name = "quotas") +public class QuotaEntity { + + @Id + @Column(name = "tenant_id", length = 36) + private String tenantId; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String limits; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public QuotaEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/quota/repository/QuotaRepository.java b/syncflow-api/src/main/java/com/syncflow/api/security/quota/repository/QuotaRepository.java new file mode 100644 index 0000000..8e17177 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/quota/repository/QuotaRepository.java @@ -0,0 +1,7 @@ +package com.syncflow.api.security.quota.repository; + +import com.syncflow.api.security.quota.entity.QuotaEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface QuotaRepository extends JpaRepository { +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotCheckpointEntity.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotCheckpointEntity.java new file mode 100644 index 0000000..42f1d2b --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotCheckpointEntity.java @@ -0,0 +1,48 @@ +package com.syncflow.api.snapshot.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; + +/** Resume checkpoint for a snapshot pipeline+table; one row per tenant. */ +@Setter +@Getter +@Entity +@Table(name = "snapshot_checkpoints") +public class SnapshotCheckpointEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "pipeline_id", nullable = false, length = 36) + private String pipelineId; + + @Column(name = "source_table", nullable = false, length = 255) + private String sourceTable; + + @Column(name = "last_batch_number", nullable = false) + private int lastBatchNumber; + + @Column(name = "rows_processed", nullable = false) + private long rowsProcessed; + + @Column(name = "cursor_pos", length = 4096) + private String cursorPos; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public SnapshotCheckpointEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotJobEntity.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotJobEntity.java new file mode 100644 index 0000000..26940e3 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotJobEntity.java @@ -0,0 +1,48 @@ +package com.syncflow.api.snapshot.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** + * Snapshot job runtime state; complex payload (progress/stats/errors) as JSONB. + */ +@Setter +@Getter +@Entity +@Table(name = "snapshot_jobs") +public class SnapshotJobEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "pipeline_id", nullable = false, length = 36) + private String pipelineId; + + @Column(nullable = false, length = 20) + private String status; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String payload; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public SnapshotJobEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotCheckpointRepository.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotCheckpointRepository.java new file mode 100644 index 0000000..857ba61 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotCheckpointRepository.java @@ -0,0 +1,21 @@ +package com.syncflow.api.snapshot.repository; + +import com.syncflow.api.snapshot.entity.SnapshotCheckpointEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.util.Optional; + +public interface SnapshotCheckpointRepository extends JpaRepository { + + Optional findByTenantIdAndPipelineIdAndSourceTable( + String tenantId, String pipelineId, String sourceTable); + + @Modifying + @Query(value = """ + DELETE FROM snapshot_checkpoints WHERE tenant_id = :tenantId AND pipeline_id = :pipelineId + """, nativeQuery = true) + void deleteAllForPipeline(@Param("tenantId") String tenantId, @Param("pipelineId") String pipelineId); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotJobRepository.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotJobRepository.java new file mode 100644 index 0000000..f5c8957 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotJobRepository.java @@ -0,0 +1,13 @@ +package com.syncflow.api.snapshot.repository; + +import com.syncflow.api.snapshot.entity.SnapshotJobEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface SnapshotJobRepository extends JpaRepository { + + List findByTenantIdOrderByCreatedAtDesc(String tenantId); + + List findByTenantIdAndPipelineIdOrderByCreatedAtDesc(String tenantId, String pipelineId); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/entity/SyncJobEntity.java b/syncflow-api/src/main/java/com/syncflow/api/sync/entity/SyncJobEntity.java new file mode 100644 index 0000000..1861fbf --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/entity/SyncJobEntity.java @@ -0,0 +1,49 @@ +package com.syncflow.api.sync.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** + * Sync job runtime state; one row per tenant+pipeline (matches the orchestrator + * key). + */ +@Setter +@Getter +@Entity +@Table(name = "sync_jobs") +public class SyncJobEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "pipeline_id", nullable = false, length = 36) + private String pipelineId; + + @Column(nullable = false, length = 20) + private String state; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String statistics; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public SyncJobEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/repository/SyncJobRepository.java b/syncflow-api/src/main/java/com/syncflow/api/sync/repository/SyncJobRepository.java new file mode 100644 index 0000000..987f349 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/repository/SyncJobRepository.java @@ -0,0 +1,14 @@ +package com.syncflow.api.sync.repository; + +import com.syncflow.api.sync.entity.SyncJobEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface SyncJobRepository extends JpaRepository { + + Optional findByTenantIdAndPipelineId(String tenantId, String pipelineId); + + List findByTenantIdOrderByCreatedAtDesc(String tenantId); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/workflow/entity/WorkflowInstanceEntity.java b/syncflow-api/src/main/java/com/syncflow/api/workflow/entity/WorkflowInstanceEntity.java new file mode 100644 index 0000000..00a0107 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/workflow/entity/WorkflowInstanceEntity.java @@ -0,0 +1,53 @@ +package com.syncflow.api.workflow.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** Workflow instance runtime state; task graph and executions as JSONB. */ +@Setter +@Getter +@Entity +@Table(name = "workflow_instances") +public class WorkflowInstanceEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "pipeline_id", nullable = false, length = 36) + private String pipelineId; + + @Column(nullable = false, length = 20) + private String status; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String tasks; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String executions; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "completed_at") + private Instant completedAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public WorkflowInstanceEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/workflow/repository/WorkflowInstanceRepository.java b/syncflow-api/src/main/java/com/syncflow/api/workflow/repository/WorkflowInstanceRepository.java new file mode 100644 index 0000000..5c9e4ff --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/workflow/repository/WorkflowInstanceRepository.java @@ -0,0 +1,11 @@ +package com.syncflow.api.workflow.repository; + +import com.syncflow.api.workflow.entity.WorkflowInstanceEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface WorkflowInstanceRepository extends JpaRepository { + + List findByTenantIdOrderByCreatedAtDesc(String tenantId); +} diff --git a/syncflow-api/src/main/resources/db/migration/V12__runtime_state_persistence.sql b/syncflow-api/src/main/resources/db/migration/V12__runtime_state_persistence.sql new file mode 100644 index 0000000..42e2fee --- /dev/null +++ b/syncflow-api/src/main/resources/db/migration/V12__runtime_state_persistence.sql @@ -0,0 +1,129 @@ +-- Runtime state persistence: durable storage for the previously in-process +-- ConcurrentHashMap state stores (snapshots, sync jobs, workflows, quotas, +-- audit records, API keys, agents, alerts) so runtime state survives restarts. +-- tenant_id follows V11 conventions: context-less rows and request-path data +-- land in the single default tenant. + +-- Snapshot jobs: full job payload (status/progress/stats/errors) as JSONB plus +-- denormalized status/pipeline columns for tenant-scoped queries. +CREATE TABLE IF NOT EXISTS snapshot_jobs ( + id VARCHAR(36) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + pipeline_id VARCHAR(36) NOT NULL, + status VARCHAR(20) NOT NULL, + payload JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); +CREATE INDEX idx_snapshot_jobs_tenant ON snapshot_jobs(tenant_id); +CREATE INDEX idx_snapshot_jobs_pipeline ON snapshot_jobs(pipeline_id); +CREATE INDEX idx_snapshot_jobs_status ON snapshot_jobs(status); + +-- Snapshot resume checkpoints (one per pipeline+source table per tenant). +CREATE TABLE IF NOT EXISTS snapshot_checkpoints ( + id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + pipeline_id VARCHAR(36) NOT NULL, + source_table VARCHAR(255) NOT NULL, + last_batch_number INTEGER NOT NULL, + rows_processed BIGINT NOT NULL, + cursor_pos VARCHAR(4096), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL, + CONSTRAINT uq_checkpoint_pipeline_table UNIQUE (tenant_id, pipeline_id, source_table) +); + +-- Sync jobs: one live job per tenant+pipeline (matches the orchestrator's +-- in-memory tenant-scoped map key). Statistics stored as JSONB. +CREATE TABLE IF NOT EXISTS sync_jobs ( + id VARCHAR(36) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + pipeline_id VARCHAR(36) NOT NULL, + state VARCHAR(20) NOT NULL, + statistics JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL, + CONSTRAINT uq_sync_job_tenant_pipeline UNIQUE (tenant_id, pipeline_id) +); + +-- Workflow instances: task graph and executions stored as JSONB. +CREATE TABLE IF NOT EXISTS workflow_instances ( + id VARCHAR(36) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + pipeline_id VARCHAR(36) NOT NULL, + status VARCHAR(20) NOT NULL, + tasks JSONB NOT NULL, + executions JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + completed_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); + +-- Tenant quotas: one row per tenant, limits map as JSONB. +CREATE TABLE IF NOT EXISTS quotas ( + tenant_id VARCHAR(36) PRIMARY KEY, + limits JSONB NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); + +-- Enterprise audit records (GDPR right-to-delete via anonymize). +CREATE TABLE IF NOT EXISTS audit_records ( + id UUID PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + actor VARCHAR(255), + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(100), + resource_id VARCHAR(255), + details TEXT, + ip_address VARCHAR(45), + suspicious BOOLEAN NOT NULL DEFAULT FALSE, + event_time TIMESTAMP WITH TIME ZONE NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL +); +CREATE INDEX idx_audit_records_tenant ON audit_records(tenant_id); + +-- API keys: hashed value unique; revoke/expiry drive isActive(). +CREATE TABLE IF NOT EXISTS api_keys ( + id UUID PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + hashed_key VARCHAR(64) NOT NULL, + prefix VARCHAR(16), + label VARCHAR(255), + scope VARCHAR(50), + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + expires_at TIMESTAMP WITH TIME ZONE, + revoked_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT uq_api_keys_hash UNIQUE (hashed_key) +); + +-- Agent fleet: hardware metrics and capabilities as JSONB. +CREATE TABLE IF NOT EXISTS agents ( + id VARCHAR(36) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + version VARCHAR(50), + status VARCHAR(20) NOT NULL, + capabilities JSONB NOT NULL, + labels JSONB NOT NULL, + environment VARCHAR(50), + region VARCHAR(50), + hostname VARCHAR(255), + hardware JSONB NOT NULL, + registered_at TIMESTAMP WITH TIME ZONE NOT NULL, + last_heartbeat TIMESTAMP WITH TIME ZONE NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); + +-- Ops alerts: live incidents surfaced on the dashboard. +CREATE TABLE IF NOT EXISTS alert_events ( + id VARCHAR(50) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + name VARCHAR(255) NOT NULL, + message TEXT, + severity VARCHAR(20) NOT NULL, + source VARCHAR(255), + pipeline_id VARCHAR(36), + connection_id VARCHAR(36), + event_time TIMESTAMP WITH TIME ZONE NOT NULL, + acknowledged BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL +); From 78e9216db9c0e27fd42e1f6bd7d88c144b05ab6e Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:55:33 +0530 Subject: [PATCH 08/13] security: fail-fast secrets, principal-based tenancy, controller authz, agent token - EncryptionService validates the AES key (16/24/32 bytes) and fails fast; application.yml no longer ships committed default keys/secrets. docker-compose + k8s wire dev/prod secrets via env. - TenantJwtAuthenticationConverter + TenantFilter: tenant derived from the authenticated principal (tid/oid/wid/pid claims), never trusted from X-Tenant-Id headers. Header honored only when it matches the principal. - AuthService: JWT now carries the tid claim. - All controllers (Connection/Pipeline/Sync/Snapshot/Capture/Metadata/ Plugin/Workflow/Agent/EnterpriseAi) gate mutations/reads via AuthorizationService.require(ResourcePermission.*). - AgentTokenFilter: fail-closed shared-secret guard for agent register/ heartbeat; agent management endpoints stay authenticated. - ConnectionMapper/ConnectionService: inject the shared ObjectMapper instead of ad-hoc new ObjectMapper(). --- docker/docker-compose.yml | 8 +- k8s/base/deployment.yaml | 5 + .../syncflow/api/agent/AgentController.java | 11 +- .../api/config/AuthSecurityBeans.java | 15 +- .../TenantJwtAuthenticationConverter.java | 80 ++++++++++ .../api/config/WebSecurityConfig.java | 10 +- .../encryption/EncryptionService.java | 15 +- .../connection/mapper/ConnectionMapper.java | 12 +- .../connection/service/ConnectionService.java | 9 +- .../api/controller/CaptureController.java | 11 +- .../api/controller/ConnectionController.java | 14 +- .../controller/EnterpriseAiController.java | 14 +- .../api/controller/MetadataController.java | 15 +- .../PipelineDesignerController.java | 16 +- .../api/controller/PluginController.java | 13 +- .../api/controller/SnapshotController.java | 13 +- .../api/controller/SyncController.java | 16 +- .../api/controller/WorkflowController.java | 14 +- .../api/security/AgentTokenFilter.java | 70 +++++++++ .../syncflow/api/security/AuthService.java | 5 + .../syncflow/api/security/TenantFilter.java | 120 ++++++++++++--- .../src/main/resources/application.yml | 10 +- .../com/syncflow/api/IntegrationTest.java | 2 + .../com/syncflow/api/RestApiContractTest.java | 2 + .../api/agent/AgentApiIntegrationTest.java | 2 + .../syncflow/api/cdc/CdcIntegrationTest.java | 2 + .../api/config/AbstractIntegrationTest.java | 43 ++++++ .../api/config/TestSecurityConfig.java | 17 +- .../connection/ConnectionControllerTest.java | 4 + .../db/DatabaseMigrationValidationTest.java | 2 + .../api/k8s/KubernetesIntegrationTest.java | 2 + .../api/metadata/MetadataIntegrationTest.java | 2 + .../api/ops/ObservabilityIntegrationTest.java | 2 + .../pipeline/PipelineApiIntegrationTest.java | 3 + .../api/samples/PgMongoSampleE2eTest.java | 2 + .../api/security/TenantDataIsolationTest.java | 145 +++++++++++++----- .../api/snapshot/SnapshotIntegrationTest.java | 2 + .../syncflow/api/sse/SseIntegrationTest.java | 2 + .../api/sync/SyncIntegrationTest.java | 3 + .../workflow/WorkflowApiIntegrationTest.java | 13 ++ .../com/syncflow/security/SecurityConfig.java | 7 + 41 files changed, 652 insertions(+), 101 deletions(-) create mode 100644 syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java create mode 100644 syncflow-api/src/main/java/com/syncflow/api/security/AgentTokenFilter.java diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d055c05..375449e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,8 +40,12 @@ services: SPRING_DATASOURCE_USERNAME: syncflow SPRING_DATASOURCE_PASSWORD: syncflow SPRING_JPA_HIBERNATE_DDL_AUTO: validate - # JWT auth (must-change-password is on for admin-provisioned accounts). - SYNCFLOW_JWT_SECRET: ${SYNCFLOW_JWT_SECRET:-c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA==} + # ─── DEV-ONLY secrets. The app has no defaults (fail-fast) — supply real + # values in non-local environments. NEVER use these in production. + # AES-256 key = base64("dev-encryption-key-0123456789abc") (32 bytes). + SYNCFLOW_ENCRYPTION_KEY: ZGV2LWVuY3J5cHRpb24ta2V5LTAxMjM0NTY3ODlhYmM= + # HS256 secret = base64 of a 44-byte dev-only HMAC key. + SYNCFLOW_JWT_SECRET: ZGV2LWp3dC1zZWNyZXQta2V5LWZvci1sb2NhbC1kZXYtb25seS1rZXktMDE= # Kafka transport (off by default). SYNCFLOW_KAFKA_ENABLED: ${SYNCFLOW_KAFKA_ENABLED:-false} SYNCFLOW_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml index 80c27e3..6d36c97 100644 --- a/k8s/base/deployment.yaml +++ b/k8s/base/deployment.yaml @@ -59,6 +59,11 @@ spec: secretKeyRef: name: syncflow-encryption key: key + - name: SYNCFLOW_JWT_SECRET + valueFrom: + secretKeyRef: + name: syncflow-jwt + key: secret volumeMounts: - name: config mountPath: /app/config diff --git a/syncflow-api/src/main/java/com/syncflow/api/agent/AgentController.java b/syncflow-api/src/main/java/com/syncflow/api/agent/AgentController.java index 50593df..a719230 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/agent/AgentController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/agent/AgentController.java @@ -1,6 +1,8 @@ package com.syncflow.api.agent; import com.syncflow.agent.domain.Agent; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.agent.domain.AgentId; import com.syncflow.agent.domain.HardwareMetrics; import org.springframework.http.ResponseEntity; @@ -19,9 +21,11 @@ public class AgentController { private final FleetManager fleetManager; + private final AuthorizationService authz; - public AgentController(FleetManager fleetManager) { + public AgentController(FleetManager fleetManager, AuthorizationService authz) { this.fleetManager = fleetManager; + this.authz = authz; } @PostMapping("/register") @@ -57,11 +61,13 @@ public ResponseEntity> heartbeat(@RequestBody Map> list() { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(fleetManager.list()); } @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return fleetManager.get(new AgentId(id)) .map(ResponseEntity::ok) .orElse(ResponseEntity.notFound().build()); @@ -69,17 +75,20 @@ public ResponseEntity get(@PathVariable String id) { @PostMapping("/{id}/drain") public ResponseEntity> drain(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); fleetManager.drain(new AgentId(id)); return ResponseEntity.ok(Map.of("agentId", id, "status", "DRAINING")); } @PostMapping("/{id}/restart") public ResponseEntity> restart(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(Map.of("agentId", id, "action", "restart_requested")); } @GetMapping("/{id}/metrics") public ResponseEntity> metrics(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return fleetManager.get(new AgentId(id)) .map(a -> ResponseEntity.>ok(Map.of( "agentId", id, diff --git a/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java b/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java index 292d5e5..0840f9a 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java +++ b/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java @@ -2,19 +2,22 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.convert.converter.Converter; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.authentication.ProviderManager; import org.springframework.security.authentication.dao.DaoAuthenticationProvider; import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.security.authentication.AbstractAuthenticationToken; /** * Auth beans: BCrypt password encoder, the AuthenticationManager backed by * DaoAuthenticationProvider over the user-details service, and the JWT * authentication converter that maps the JWT {@code scope} claim to - * {@code ROLE_*} authorities (consumed by TenantFilter / RBAC). + * {@code SCOPE_*} authorities and carries the caller's tenant claims (read by + * TenantFilter / RBAC — tenant is taken from the principal, not client headers). */ @Configuration public class AuthSecurityBeans { @@ -34,9 +37,9 @@ public AuthenticationManager authenticationManager( } @Bean - public JwtAuthenticationConverter jwtAuthenticationConverter() { - // Map JWT 'scope' claim -> ROLE_ authorities. The login token encodes the - // user's roles into 'scope'; TenantFilter reads authorities for RBAC. - return new JwtAuthenticationConverter(); + public Converter jwtAuthenticationConverter() { + // Maps JWT 'scope' -> SCOPE_ authorities and attaches the tenant scope + // (tid/oid/wid/pid claims) to the token details. + return new TenantJwtAuthenticationConverter(); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java b/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java new file mode 100644 index 0000000..265921d --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java @@ -0,0 +1,80 @@ +package com.syncflow.api.config; + +import com.syncflow.tenant.TenantId; +import org.springframework.core.convert.converter.Converter; +import org.springframework.security.authentication.AbstractAuthenticationToken; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken; +import org.springframework.security.oauth2.server.resource.authentication.JwtGrantedAuthoritiesConverter; +import org.springframework.stereotype.Component; + +import java.util.Collection; + +/** + * JWT -> authentication converter that carries the caller's tenant scope + * (tenant/org/workspace/project) in the token details. TenantFilter reads the + * tenant from this principal — NEVER from client headers — so an authenticated + * caller cannot switch tenant by spoofing {@code X-Tenant-Id}. + * + * The tenant claim is read as {@code tid} (or {@code tenant}). Org/workspace/ + * project are best-effort from {@code oid}/{@code wid}/{@code pid} (or their + * {@code *Id} spellings). A legacy subject of the form + * {@code tenant:{tenant}:{org}:{workspace}:{project}} is also honored. + * + * Authority mapping mirrors the default {@code JwtAuthenticationConverter}: the + * {@code scope} claim becomes {@code SCOPE_*}-prefixed authorities (RBAC's + * {@code PolicyResolver} still grants ADMIN via the username or the ADMIN role). + */ +@Component +public class TenantJwtAuthenticationConverter implements Converter { + + private final JwtGrantedAuthoritiesConverter authoritiesConverter = new JwtGrantedAuthoritiesConverter(); + + @Override + public AbstractAuthenticationToken convert(Jwt jwt) { + Collection authorities = authoritiesConverter.convert(jwt); + var token = new JwtAuthenticationToken(jwt, authorities, jwt.getSubject()); + token.setDetails(extractTenant(jwt)); + return token; + } + + /** Tenant scope carried on the token details; read by {@code TenantFilter}. */ + public record PrincipalTenant(TenantId tenantId, String organizationId, + String workspaceId, String projectId) { + } + + static PrincipalTenant extractTenant(Jwt jwt) { + var tenantClaim = jwt.hasClaim("tid") ? jwt.getClaimAsString("tid") + : jwt.hasClaim("tenant") ? jwt.getClaimAsString("tenant") : null; + var subject = jwt.getSubject(); + + if (tenantClaim == null && subject != null && subject.startsWith("tenant:")) { + var parts = subject.substring("tenant:".length()).split(":", -1); + if (parts.length >= 4) { + return new PrincipalTenant( + TenantId.from(parts[0]), + blankToNull(parts[1]), + blankToNull(parts[2]), + blankToNull(parts[3])); + } + } + if (tenantClaim != null) { + return new PrincipalTenant( + TenantId.from(tenantClaim), + claimString(jwt, "oid", "organizationId"), + claimString(jwt, "wid", "workspaceId"), + claimString(jwt, "pid", "projectId")); + } + return new PrincipalTenant(TenantId.DEFAULT, null, null, null); + } + + private static String claimString(Jwt jwt, String primary, String secondary) { + var v = jwt.hasClaim(primary) ? jwt.getClaimAsString(primary) : null; + return v != null ? v : (jwt.hasClaim(secondary) ? jwt.getClaimAsString(secondary) : null); + } + + private static String blankToNull(String s) { + return s == null || s.isBlank() ? null : s; + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/config/WebSecurityConfig.java b/syncflow-api/src/main/java/com/syncflow/api/config/WebSecurityConfig.java index 19bd5cd..aa21cba 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/config/WebSecurityConfig.java +++ b/syncflow-api/src/main/java/com/syncflow/api/config/WebSecurityConfig.java @@ -1,15 +1,19 @@ package com.syncflow.api.config; +import com.syncflow.api.security.AgentTokenFilter; import com.syncflow.security.SecurityConfig; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.core.convert.converter.Converter; +import org.springframework.security.authentication.AbstractAuthenticationToken; +import org.springframework.security.oauth2.jwt.Jwt; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; import org.springframework.security.config.http.SessionCreationPolicy; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter; import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.csrf.CookieCsrfTokenRepository; @Configuration @@ -20,8 +24,10 @@ public class WebSecurityConfig { @Bean @ConditionalOnMissingBean(SecurityFilterChain.class) public SecurityFilterChain filterChain(HttpSecurity http, - JwtAuthenticationConverter jwtAuthenticationConverter) throws Exception { + Converter jwtAuthenticationConverter, + AgentTokenFilter agentTokenFilter) throws Exception { http + .addFilterBefore(agentTokenFilter, UsernamePasswordAuthenticationFilter.class) .csrf(csrf -> csrf // Hybrid CSRF: protect cookie-based paths; /api/** uses bearer // tokens in headers (browsers cannot forge them), so it stays diff --git a/syncflow-api/src/main/java/com/syncflow/api/connection/encryption/EncryptionService.java b/syncflow-api/src/main/java/com/syncflow/api/connection/encryption/EncryptionService.java index d12e732..d2ae140 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/connection/encryption/EncryptionService.java +++ b/syncflow-api/src/main/java/com/syncflow/api/connection/encryption/EncryptionService.java @@ -21,7 +21,20 @@ public class EncryptionService { private final SecureRandom secureRandom; public EncryptionService(@Value("${syncflow.encryption.key}") String base64Key) { - var decoded = Base64.getDecoder().decode(base64Key); + if (base64Key == null || base64Key.isBlank()) { + throw new IllegalStateException("syncflow.encryption.key is not configured. " + + "Set a base64-encoded AES key (16/24/32 bytes) via env SYNCFLOW_ENCRYPTION_KEY."); + } + final byte[] decoded; + try { + decoded = Base64.getDecoder().decode(base64Key); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("syncflow.encryption.key is not valid base64", e); + } + if (decoded.length != 16 && decoded.length != 24 && decoded.length != 32) { + throw new IllegalStateException("syncflow.encryption.key decodes to " + decoded.length + + " bytes; AES requires 16, 24, or 32 bytes."); + } this.key = new SecretKeySpec(decoded, "AES"); this.secureRandom = new SecureRandom(); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/connection/mapper/ConnectionMapper.java b/syncflow-api/src/main/java/com/syncflow/api/connection/mapper/ConnectionMapper.java index 8a29434..4f95680 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/connection/mapper/ConnectionMapper.java +++ b/syncflow-api/src/main/java/com/syncflow/api/connection/mapper/ConnectionMapper.java @@ -9,6 +9,7 @@ import com.syncflow.core.connection.ConnectionStatus; import com.syncflow.core.connection.ConnectionType; import com.syncflow.core.connection.Credentials; +import org.springframework.beans.factory.annotation.Autowired; import org.mapstruct.Mapper; import org.mapstruct.Mapping; @@ -18,6 +19,10 @@ @Mapper(componentModel = "spring") public abstract class ConnectionMapper { + // Reuse the Spring-managed Jackson mapper (JSR-310 aware); do not build a + // per-call ObjectMapper (ignores configured Jackson modules). + @Autowired + protected ObjectMapper objectMapper; @Mapping(target = "id", expression = "java(domain.getId().value())") @Mapping(target = "connectionType", expression = "java(domain.getProperties().type().name())") @Mapping(target = "host", expression = "java(domain.getProperties().host())") @@ -55,7 +60,7 @@ protected String toJson(Map map) { if (map == null || map.isEmpty()) return null; try { - return new ObjectMapper().writeValueAsString(map); + return objectMapper.writeValueAsString(map); } catch (Exception e) { return null; } @@ -65,10 +70,9 @@ private Map parseOptions(String json) { if (json == null || json.isBlank()) return Map.of(); try { - var mapper = new ObjectMapper(); - var type = mapper.getTypeFactory().constructMapType( + var type = objectMapper.getTypeFactory().constructMapType( HashMap.class, String.class, String.class); - return mapper.readValue(json, type); + return objectMapper.readValue(json, type); } catch (Exception e) { return Map.of(); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/connection/service/ConnectionService.java b/syncflow-api/src/main/java/com/syncflow/api/connection/service/ConnectionService.java index 2019666..6c668bd 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/connection/service/ConnectionService.java +++ b/syncflow-api/src/main/java/com/syncflow/api/connection/service/ConnectionService.java @@ -4,6 +4,7 @@ import com.syncflow.api.connection.entity.ConnectionEntity; import com.syncflow.api.connection.mapper.ConnectionMapper; import com.syncflow.api.connection.repository.ConnectionRepository; +import com.fasterxml.jackson.databind.ObjectMapper; import com.syncflow.common.exception.SyncFlowException; import com.syncflow.core.connection.Connection; import com.syncflow.core.connection.ConnectionProperties; @@ -21,13 +22,16 @@ public class ConnectionService { private final ConnectionRepository repository; private final ConnectionMapper mapper; private final EncryptionService encryption; + private final ObjectMapper objectMapper; public ConnectionService(ConnectionRepository repository, ConnectionMapper mapper, - EncryptionService encryption) { + EncryptionService encryption, + ObjectMapper objectMapper) { this.repository = repository; this.mapper = mapper; this.encryption = encryption; + this.objectMapper = objectMapper; } public Connection create(String name, ConnectionProperties props, Credentials credentials) { @@ -105,8 +109,7 @@ private String serializeOptions(ConnectionProperties props) { if (props.options() == null || props.options().isEmpty()) return null; try { - return new com.fasterxml.jackson.databind.ObjectMapper() - .writeValueAsString(props.options()); + return objectMapper.writeValueAsString(props.options()); } catch (Exception e) { return null; } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/CaptureController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/CaptureController.java index 5d472c9..3e9e1aa 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/CaptureController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/CaptureController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.cdc.CaptureLifecycle; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.core.cdc.CaptureStatus; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -17,13 +19,16 @@ public class CaptureController { private final CaptureLifecycle lifecycle; + private final AuthorizationService authz; - public CaptureController(CaptureLifecycle lifecycle) { + public CaptureController(CaptureLifecycle lifecycle, AuthorizationService authz) { this.lifecycle = lifecycle; + this.authz = authz; } @PostMapping("/start") public ResponseEntity> start(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); var status = lifecycle.start(id, null); return ResponseEntity.status(status == CaptureStatus.RUNNING ? HttpStatus.OK : HttpStatus.ACCEPTED) .body(Map.of("pipelineId", id, "status", status.name())); @@ -31,24 +36,28 @@ public ResponseEntity> start(@PathVariable String id) { @PostMapping("/stop") public ResponseEntity> stop(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); lifecycle.stop(id); return ResponseEntity.ok(Map.of("pipelineId", id, "status", "STOPPED")); } @PostMapping("/pause") public ResponseEntity> pause(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); lifecycle.pause(id); return ResponseEntity.ok(Map.of("pipelineId", id, "status", "PAUSED")); } @PostMapping("/resume") public ResponseEntity> resume(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); lifecycle.resume(id); return ResponseEntity.ok(Map.of("pipelineId", id, "status", "RESUMED")); } @GetMapping("/status") public ResponseEntity> status(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var status = lifecycle.status(id); var events = lifecycle.eventCount(id); return ResponseEntity.ok(Map.of( diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/ConnectionController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/ConnectionController.java index 90b4d77..0904fae 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/ConnectionController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/ConnectionController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.connection.dto.ConnectionHealthResponse; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.connection.dto.ConnectionResponse; import com.syncflow.api.connection.dto.CreateConnectionRequest; import com.syncflow.api.connection.dto.TestConnectionRequest; @@ -31,15 +33,19 @@ public class ConnectionController { private final ConnectionService connectionService; private final ConnectorFactory connectorFactory; + private final AuthorizationService authz; public ConnectionController(ConnectionService connectionService, - ConnectorFactory connectorFactory) { + ConnectorFactory connectorFactory, + AuthorizationService authz) { this.connectionService = connectionService; this.connectorFactory = connectorFactory; + this.authz = authz; } @PostMapping public ResponseEntity create(@Valid @RequestBody CreateConnectionRequest req) { + authz.require(ResourcePermission.CONNECTION_WRITE); var props = new ConnectionProperties(req.connectionType(), req.host(), req.port(), req.database(), req.options() != null ? req.options() : Map.of()); var credentials = new Credentials( @@ -52,6 +58,7 @@ public ResponseEntity create(@Valid @RequestBody CreateConne @GetMapping public ResponseEntity> list() { + authz.require(ResourcePermission.CONNECTION_READ); var list = connectionService.list().stream() .map(c -> ConnectionResponse.from(c, true)) .toList(); @@ -60,6 +67,7 @@ public ResponseEntity> list() { @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(ConnectionResponse.from(connectionService.get(id), true)); } @@ -67,6 +75,7 @@ public ResponseEntity get(@PathVariable String id) { public ResponseEntity update( @PathVariable String id, @Valid @RequestBody UpdateConnectionRequest req) { + authz.require(ResourcePermission.CONNECTION_WRITE); var existing = connectionService.get(id); var props = new ConnectionProperties(existing.getProperties().type(), req.host(), req.port(), req.database(), @@ -83,12 +92,14 @@ public ResponseEntity update( @DeleteMapping("/{id}") public ResponseEntity delete(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_DELETE); connectionService.delete(id); return ResponseEntity.noContent().build(); } @PostMapping("/test") public ResponseEntity test(@Valid @RequestBody TestConnectionRequest req) { + authz.require(ResourcePermission.CONNECTION_WRITE); var props = new ConnectionProperties(req.connectionType(), req.host(), req.port(), req.database(), req.options() != null ? req.options() : Map.of()); var credentials = new Credentials( @@ -112,6 +123,7 @@ public ResponseEntity test(@Valid @RequestBody TestConne @GetMapping("/{id}/health") public ResponseEntity health(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); var connection = connectionService.getWithDecryptedCredentials(id); var validator = connectorFactory.getValidator(connection.getProperties().type()); if (validator.isEmpty()) { diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/EnterpriseAiController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/EnterpriseAiController.java index b2b8dfd..baf2e7b 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/EnterpriseAiController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/EnterpriseAiController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.agent.ai.domain.AgentContext; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.agent.ai.domain.AgentResult; import com.syncflow.api.agent.ai.domain.Conversation; import com.syncflow.api.agent.ai.domain.ReasoningPlan; @@ -24,14 +26,18 @@ public class EnterpriseAiController { private final AgentOrchestrator orchestrator; private final KnowledgeBase knowledgeBase; + private final AuthorizationService authz; - public EnterpriseAiController(AgentOrchestrator orchestrator, KnowledgeBase knowledgeBase) { + public EnterpriseAiController(AgentOrchestrator orchestrator, KnowledgeBase knowledgeBase, + AuthorizationService authz) { this.orchestrator = orchestrator; this.knowledgeBase = knowledgeBase; + this.authz = authz; } @PostMapping("/chat") public ResponseEntity chat(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); var session = body.getOrDefault("sessionId", UUID.randomUUID().toString()); var userId = body.getOrDefault("userId", "anonymous"); var tenantId = body.getOrDefault("tenantId", "default"); @@ -42,11 +48,13 @@ public ResponseEntity chat(@RequestBody Map body) @PostMapping("/plan") public ResponseEntity createPlan(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); return ResponseEntity.ok(orchestrator.createPlan(body.get("goal"))); } @PostMapping("/analyze") public ResponseEntity> analyze(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); var plan = orchestrator.createPlan(body.get("goal")); var context = new AgentContext( body.get("workspaceId"), body.get("pipelineId"), @@ -56,11 +64,13 @@ public ResponseEntity> analyze(@RequestBody Map> document(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); return ResponseEntity.ok(knowledgeBase.search(body.get("query"))); } @PostMapping("/review") public ResponseEntity> review(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); var plan = orchestrator.createPlan("review " + body.getOrDefault("pipelineId", "")); var context = new AgentContext(null, body.get("pipelineId"), null, Map.of()); return ResponseEntity.ok(orchestrator.executePlan(plan, context)); @@ -68,6 +78,7 @@ public ResponseEntity> review(@RequestBody Map @PostMapping("/recommend") public ResponseEntity> recommend(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); var plan = orchestrator.createPlan("optimize " + body.getOrDefault("pipelineId", "")); var context = new AgentContext(null, body.get("pipelineId"), null, Map.of()); return ResponseEntity.ok(orchestrator.executePlan(plan, context)); @@ -75,6 +86,7 @@ public ResponseEntity> recommend(@RequestBody Map> history(@RequestParam String sessionId) { + authz.require(ResourcePermission.AI_USE); return ResponseEntity.ok(orchestrator.history(sessionId)); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/MetadataController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/MetadataController.java index 1aaeb7b..166dbf6 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/MetadataController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/MetadataController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.metadata.MetadataDiscoveryService; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.core.metadata.ColumnMetadata; import com.syncflow.core.metadata.ConstraintMetadata; import com.syncflow.core.metadata.IndexMetadata; @@ -19,20 +21,24 @@ public class MetadataController { private final MetadataDiscoveryService discoveryService; + private final AuthorizationService authz; - public MetadataController(MetadataDiscoveryService discoveryService) { + public MetadataController(MetadataDiscoveryService discoveryService, AuthorizationService authz) { this.discoveryService = discoveryService; + this.authz = authz; } /** GET /api/connections/{id}/metadata — returns schemas */ @GetMapping("/metadata") public ResponseEntity> getSchemas(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverSchemas(id)); } /** GET /api/connections/{id}/metadata/schemas */ @GetMapping("/metadata/schemas") public ResponseEntity> getSchemasAlt(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverSchemas(id)); } @@ -42,6 +48,7 @@ public ResponseEntity> getSchemasAlt(@PathVaria @GetMapping("/schemas/{schema}/tables") public ResponseEntity> getTables( @PathVariable String id, @PathVariable String schema) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverTables(id, schema)); } @@ -49,6 +56,7 @@ public ResponseEntity> getTables( @GetMapping("/metadata/schemas/{schema}/tables") public ResponseEntity> getTablesAlt( @PathVariable String id, @PathVariable String schema) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverTables(id, schema)); } @@ -56,6 +64,7 @@ public ResponseEntity> getTablesAlt( public ResponseEntity> getTable( @PathVariable String id, @PathVariable String schema, @PathVariable String table) { + authz.require(ResourcePermission.CONNECTION_READ); var resp = discoveryService.discoverTables(id, schema); var filtered = resp.data().stream() .filter(t -> t.name().equals(table)) @@ -69,6 +78,7 @@ public ResponseEntity> getTable( public ResponseEntity> getColumns( @PathVariable String id, @PathVariable String schema, @PathVariable String table) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverColumns(id, schema, table)); } @@ -76,6 +86,7 @@ public ResponseEntity> getColumns( public ResponseEntity> getIndexes( @PathVariable String id, @PathVariable String schema, @PathVariable String table) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverIndexes(id, schema, table)); } @@ -84,11 +95,13 @@ public ResponseEntity> getIndexes( public ResponseEntity> getConstraints( @PathVariable String id, @PathVariable String schema, @PathVariable String table) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverConstraints(id, schema, table)); } @PostMapping({"/metadata/refresh", "/schemas/refresh"}) public ResponseEntity refresh(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_WRITE); discoveryService.refresh(id); return ResponseEntity.ok().build(); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/PipelineDesignerController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/PipelineDesignerController.java index 21786ca..87d3ee2 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/PipelineDesignerController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/PipelineDesignerController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.pipeline.PipelineDesignerService; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.pipeline.dto.CreatePipelineDesignRequest; import com.syncflow.api.pipeline.dto.PipelineDesignResponse; import com.syncflow.api.pipeline.dto.UpdatePipelineDesignRequest; @@ -34,14 +36,17 @@ public class PipelineDesignerController { private final PipelineDesignerService service; + private final AuthorizationService authz; - public PipelineDesignerController(PipelineDesignerService service) { + public PipelineDesignerController(PipelineDesignerService service, AuthorizationService authz) { this.service = service; + this.authz = authz; } @PostMapping public ResponseEntity create( @Valid @RequestBody CreatePipelineDesignRequest req) { + authz.require(ResourcePermission.PIPELINE_WRITE); var name = new PipelineName(req.name()); var source = new SourceReference(req.sourceConnectionId(), req.sourceSchema(), req.sourceTable()); var dest = new DestinationReference(req.destConnectionId(), req.destSchema(), req.destTable(), @@ -59,12 +64,14 @@ public ResponseEntity create( @GetMapping public ResponseEntity> list() { + authz.require(ResourcePermission.PIPELINE_READ); var list = service.list().stream().map(PipelineDesignResponse::from).toList(); return ResponseEntity.ok(list); } @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_READ); return ResponseEntity.ok(PipelineDesignResponse.from(service.get(id))); } @@ -72,6 +79,7 @@ public ResponseEntity get(@PathVariable String id) { public ResponseEntity update( @PathVariable String id, @RequestBody UpdatePipelineDesignRequest req) { + authz.require(ResourcePermission.PIPELINE_WRITE); var existing = service.get(id); var name = req.name() != null ? new PipelineName(req.name()) : existing.name(); var source = req.sourceConnectionId() != null @@ -98,34 +106,40 @@ public ResponseEntity update( @DeleteMapping("/{id}") public ResponseEntity delete(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_DELETE); service.delete(id); return ResponseEntity.noContent().build(); } @PostMapping("/{id}/validate") public ResponseEntity validate(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_READ); return ResponseEntity.ok(service.validate(id)); } @PostMapping("/{id}/rollback") public ResponseEntity rollback( @PathVariable String id, @RequestParam int version) { + authz.require(ResourcePermission.PIPELINE_WRITE); return ResponseEntity.ok(PipelineDesignResponse.from(service.rollback(id, version))); } @GetMapping("/{id}/versions") public ResponseEntity> versions(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_READ); var list = service.versions(id).stream().map(PipelineDesignResponse::from).toList(); return ResponseEntity.ok(list); } @GetMapping("/{id}/preview") public ResponseEntity preview(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(service.preview(id)); } @GetMapping("/{id}/conflicts") public ResponseEntity conflicts(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_READ); return ResponseEntity.ok(service.detectConflicts(id)); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/PluginController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/PluginController.java index 32a0bee..b1d4a27 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/PluginController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/PluginController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.plugin.PluginManager; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.plugin.PluginManager.PluginEntry; import com.syncflow.api.plugin.PluginManager.PluginInstallResult; import org.springframework.http.ResponseEntity; @@ -22,18 +24,22 @@ public class PluginController { private final PluginManager pluginManager; + private final AuthorizationService authz; - public PluginController(PluginManager pluginManager) { + public PluginController(PluginManager pluginManager, AuthorizationService authz) { this.pluginManager = pluginManager; + this.authz = authz; } @GetMapping public ResponseEntity> list() { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(pluginManager.list()); } @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return pluginManager.get(id) .map(ResponseEntity::ok) .orElse(ResponseEntity.notFound().build()); @@ -41,6 +47,7 @@ public ResponseEntity get(@PathVariable String id) { @PostMapping("/install") public ResponseEntity install(@RequestParam("file") MultipartFile file) { + authz.require(ResourcePermission.PIPELINE_WRITE); try { var temp = File.createTempFile("plugin-", ".jar"); file.transferTo(temp); @@ -55,24 +62,28 @@ public ResponseEntity install(@RequestParam("file") Multipa @PostMapping("/{id}/enable") public ResponseEntity> enable(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_WRITE); var ok = pluginManager.enable(id); return ResponseEntity.ok(Map.of("pluginId", id, "enabled", ok)); } @PostMapping("/{id}/disable") public ResponseEntity> disable(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_WRITE); var ok = pluginManager.disable(id); return ResponseEntity.ok(Map.of("pluginId", id, "disabled", ok)); } @DeleteMapping("/{id}") public ResponseEntity> uninstall(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_DELETE); var ok = pluginManager.uninstall(id); return ResponseEntity.ok(Map.of("pluginId", id, "uninstalled", ok)); } @GetMapping("/{id}/capabilities") public ResponseEntity> capabilities(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return pluginManager.get(id) .map(entry -> { var caps = entry.connector().capabilities(); diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/SnapshotController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/SnapshotController.java index 192e5a9..66f2c84 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/SnapshotController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/SnapshotController.java @@ -1,5 +1,7 @@ package com.syncflow.api.controller; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.snapshot.SnapshotExecutor; import com.syncflow.api.sse.StatusBroadcaster; import com.syncflow.core.snapshot.SnapshotJob; @@ -22,30 +24,37 @@ public class SnapshotController { private final SnapshotExecutor executor; private final StatusBroadcaster broadcaster; + private final AuthorizationService authz; - public SnapshotController(SnapshotExecutor executor, StatusBroadcaster broadcaster) { + public SnapshotController(SnapshotExecutor executor, StatusBroadcaster broadcaster, + AuthorizationService authz) { this.executor = executor; this.broadcaster = broadcaster; + this.authz = authz; } @PostMapping("/pipelines/{id}/snapshot") public ResponseEntity start(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); var job = executor.start(id); return ResponseEntity.status(HttpStatus.ACCEPTED).body(job); } @GetMapping("/snapshots") public ResponseEntity> list() { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(executor.list()); } @GetMapping("/snapshots/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(executor.get(id)); } @GetMapping("/snapshots/{id}/progress") public ResponseEntity progress(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var job = executor.get(id); return ResponseEntity.ok(job.getProgress()); } @@ -53,12 +62,14 @@ public ResponseEntity progress(@PathVariable String id) { /** Live progress/status stream for a snapshot ("snapshot-status" events). */ @GetMapping(value = "/snapshots/{id}/events", produces = "text/event-stream") public SseEmitter snapshotEvents(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var tenant = TenantContextHolder.getTenantId().value(); return broadcaster.subscribe(tenant + ":" + id); } @PostMapping("/snapshots/{id}/cancel") public ResponseEntity cancel(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(executor.cancel(id)); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/SyncController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/SyncController.java index e811c83..bf515bf 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/SyncController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/SyncController.java @@ -1,5 +1,7 @@ package com.syncflow.api.controller; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.sse.StatusBroadcaster; import com.syncflow.api.sync.DeadLetterQueue; import com.syncflow.api.sync.SyncOrchestrator; @@ -27,12 +29,14 @@ public class SyncController { private final SyncOrchestrator orchestrator; private final DeadLetterQueue dlq; private final StatusBroadcaster broadcaster; + private final AuthorizationService authz; public SyncController(SyncOrchestrator orchestrator, DeadLetterQueue dlq, - StatusBroadcaster broadcaster) { + StatusBroadcaster broadcaster, AuthorizationService authz) { this.orchestrator = orchestrator; this.dlq = dlq; this.broadcaster = broadcaster; + this.authz = authz; } /** @@ -42,23 +46,27 @@ public SyncController(SyncOrchestrator orchestrator, DeadLetterQueue dlq, */ @GetMapping(value = "/sync/jobs/{id}/events", produces = "text/event-stream") public SseEmitter syncEvents(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var tenant = TenantContextHolder.getTenantId().value(); return broadcaster.subscribe(tenant + ":" + id); } @PostMapping("/pipelines/{id}/sync/start") public ResponseEntity start(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(orchestrator.start(id)); } @PostMapping("/pipelines/{id}/sync/stop") public ResponseEntity> stop(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); orchestrator.stop(id); return ResponseEntity.ok(Map.of("pipelineId", id, "status", "STOPPED")); } @GetMapping("/pipelines/{id}/sync/status") public ResponseEntity> status(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var state = orchestrator.status(id); var stats = orchestrator.statistics(id); return ResponseEntity.ok(Map.of( @@ -72,33 +80,39 @@ public ResponseEntity> status(@PathVariable String id) { @GetMapping("/sync/jobs") public ResponseEntity> jobs() { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(orchestrator.list()); } @GetMapping("/sync/jobs/{id}") public ResponseEntity job(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(orchestrator.get(id)); } @GetMapping("/sync/jobs/{id}/statistics") public ResponseEntity statistics(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(orchestrator.statistics(id)); } @GetMapping("/dlq") public ResponseEntity> dlqList( @RequestParam(required = false) String pipelineId) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(dlq.list(pipelineId)); } @PostMapping("/dlq/{id}/replay") public ResponseEntity replay(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); dlq.replay(id); return ResponseEntity.ok().build(); } @DeleteMapping("/dlq/{id}") public ResponseEntity deleteDlq(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_DELETE); dlq.delete(id); return ResponseEntity.noContent().build(); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/WorkflowController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/WorkflowController.java index 085842d..063fd0d 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/WorkflowController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/WorkflowController.java @@ -1,5 +1,7 @@ package com.syncflow.api.controller; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.workflow.WorkflowScheduler; import com.syncflow.core.workflow.WorkflowId; import com.syncflow.core.workflow.WorkflowInstance; @@ -21,49 +23,59 @@ public class WorkflowController { private final WorkflowScheduler scheduler; + private final AuthorizationService authz; - public WorkflowController(WorkflowScheduler scheduler) { + public WorkflowController(WorkflowScheduler scheduler, AuthorizationService authz) { this.scheduler = scheduler; + this.authz = authz; } @PostMapping public ResponseEntity create(@RequestBody Map body) { + authz.require(ResourcePermission.PIPELINE_WRITE); var wf = scheduler.create(body.get("pipelineId")); return ResponseEntity.status(HttpStatus.CREATED).body(wf); } @GetMapping public ResponseEntity> list() { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(scheduler.list()); } @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(scheduler.get(WorkflowId.from(id))); } @PostMapping("/{id}/start") public ResponseEntity start(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(scheduler.start(WorkflowId.from(id))); } @PostMapping("/{id}/cancel") public ResponseEntity cancel(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_CANCEL); return ResponseEntity.ok(scheduler.cancel(WorkflowId.from(id))); } @PostMapping("/{id}/pause") public ResponseEntity> pause(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(Map.of("id", id, "status", "PAUSED")); } @PostMapping("/{id}/resume") public ResponseEntity> resume(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(Map.of("id", id, "status", "RUNNING")); } @GetMapping("/{id}/graph") public ResponseEntity> graph(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var wf = scheduler.get(WorkflowId.from(id)); return ResponseEntity.ok(wf.tasks()); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/AgentTokenFilter.java b/syncflow-api/src/main/java/com/syncflow/api/security/AgentTokenFilter.java new file mode 100644 index 0000000..dfc2c1a --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/AgentTokenFilter.java @@ -0,0 +1,70 @@ +package com.syncflow.api.security; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.security.web.util.matcher.AntPathRequestMatcher; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.security.MessageDigest; + +/** + * Shared-secret guard for the agent control-plane endpoints. + * + * The fleet agent cannot do OAuth flows, and the platform has no mTLS yet, so + * agent-to-plane calls are PUBLIC in the security chain but fail-closed here: + * when {@code syncflow.agent.token} is configured (recommended), any + * agent-INBOUND request (register/heartbeat) must carry a matching + * {@code X-Agent-Token}; missing/wrong token -> 403. When the token is unset + * (dev default), agent endpoints are open — a network-level (mTLS / network + * policy) hardening is the documented upgrade path (see ADR-008/009). + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 10) +public class AgentTokenFilter extends OncePerRequestFilter { + + static final String AGENT_TOKEN_HEADER = "X-Agent-Token"; + + private final String configuredToken; + private final AntPathRequestMatcher agentMatcher = new AntPathRequestMatcher("/api/agents/**"); + private static final String INBOUND_REGISTER = "/api/agents/register"; + private static final String INBOUND_HEARTBEAT = "/api/agents/heartbeat"; + + public AgentTokenFilter(@Value("${syncflow.agent.token:}") String configuredToken) { + this.configuredToken = configuredToken; + } + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + // Only agent-inbound paths are public; the rest pass through unchanged. + var path = request.getRequestURI(); + return !agentMatcher.matches(request) + || (!INBOUND_REGISTER.equals(path) && !INBOUND_HEARTBEAT.equals(path)); + } + + @Override + protected void doFilterInternal(HttpServletRequest request, + HttpServletResponse response, + FilterChain chain) throws ServletException, IOException { + if (configuredToken == null || configuredToken.isBlank()) { + // ponytail: token unset = dev default, open. Require it / mTLS in + // non-dev environments. + chain.doFilter(request, response); + return; + } + var presented = request.getHeader(AGENT_TOKEN_HEADER); + if (presented != null && MessageDigest.isEqual( + configuredToken.getBytes(java.nio.charset.StandardCharsets.UTF_8), + presented.getBytes(java.nio.charset.StandardCharsets.UTF_8))) { + chain.doFilter(request, response); + return; + } + response.sendError(HttpServletResponse.SC_FORBIDDEN, "invalid or missing X-Agent-Token"); + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/AuthService.java b/syncflow-api/src/main/java/com/syncflow/api/security/AuthService.java index a5a940b..e69241b 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/AuthService.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/AuthService.java @@ -2,6 +2,7 @@ import com.syncflow.api.config.JwtProperties; import com.syncflow.api.user.repository.UserRepository; +import com.syncflow.tenant.TenantId; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.GrantedAuthority; @@ -64,11 +65,15 @@ public LoginResult login(String username, String password) { private String issueToken(String username, java.util.List roles) { var now = Instant.now(); + // Single-tenant platform: tokens carry the default tenant. TenantFilter + // derives scope from these claims (tid/oid/wid/pid), never from client + // headers. Per-tenant account claims are the documented multi-tenant path. var claims = JwtClaimsSet.builder() .issuer(jwtProperties.getIssuer()) .issuedAt(now) .expiresAt(now.plusSeconds(jwtProperties.getExpiryMinutes() * 60)) .subject(username) + .claim("tid", TenantId.DEFAULT.value()) .claim("scope", String.join(",", roles)) .build(); // Pin the JWS algorithm to HS256 so Nimbus selects the matching HS256 key. diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java b/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java index aa184e0..ffa6c78 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java @@ -1,5 +1,6 @@ package com.syncflow.api.security; +import com.syncflow.api.config.TenantJwtAuthenticationConverter.PrincipalTenant; import com.syncflow.tenant.OrganizationId; import com.syncflow.tenant.ProjectId; import com.syncflow.tenant.TenantContext; @@ -10,6 +11,7 @@ import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; import org.springframework.web.filter.OncePerRequestFilter; @@ -19,6 +21,20 @@ import java.util.HashSet; import java.util.Set; +/** + * Resolves the tenant context for the current request. + * + * SECURITY: the tenant is derived from the AUTHENTICATED PRINCIPAL (the JWT's + * tenant claims, attached by {@link com.syncflow.api.config.TenantJwtAuthenticationConverter}) + * — never trusted from client headers. An authenticated caller cannot switch + * tenant by spoofing {@code X-Tenant-Id}. + * + * UI compatibility: the SPA sends {@code X-Tenant-Id} on authenticated + * requests, so a header that MATCHES the principal's tenant is accepted + * (harmless — same tenant). A mismatched header is ignored in favour of the + * principal. Unauthenticated/anonymous requests (public endpoints) fall back + * to {@link TenantId#DEFAULT}. + */ @Component public class TenantFilter extends OncePerRequestFilter { @@ -41,36 +57,94 @@ protected void doFilterInternal(HttpServletRequest request, } private TenantContext resolve(HttpServletRequest req) { - var tenantId = req.getHeader(TENANT_HEADER) != null - ? TenantId.from(req.getHeader(TENANT_HEADER)) - : TenantId.DEFAULT; - var orgId = req.getHeader(ORG_HEADER); - var wsId = req.getHeader(WORKSPACE_HEADER); - var projId = req.getHeader(PROJECT_HEADER); - - String userId = "anonymous"; Set roles = new HashSet<>(); - var auth = SecurityContextHolder.getContext().getAuthentication(); + if (auth != null && auth.isAuthenticated()) { - userId = auth.getName() != null ? auth.getName() : userId; - if (auth.getName() != null && auth.getName().startsWith("tenant:")) { - var parts = auth.getName().substring("tenant:".length()).split(":"); - if (parts.length >= 4) { - tenantId = TenantId.from(parts[0]); - orgId = parts[1].isEmpty() ? orgId : parts[1]; - wsId = parts[2].isEmpty() ? wsId : parts[2]; - projId = parts[3].isEmpty() ? projId : parts[3]; - } + var principal = principalTenant(auth); + if (principal == null) { + // Legacy principals (e.g. test stubs / basic auth) carry no tenant + // claims; fall back to a header only as a last resort so existing + // clients keep working. Authenticated tenants still come from the + // principal whenever claims exist. + return contextFromHeaders(req, auth); } - auth.getAuthorities().forEach(a -> roles.add(a.getAuthority())); + // Header is honored ONLY when it matches the principal's tenant (the UI + // sends it on every request); a mismatched/spoofed header is ignored. + var headerTenant = header(req, TENANT_HEADER); + var tenantId = principal.tenantId(); + if (headerTenant != null && TenantId.from(headerTenant).equals(tenantId)) { + // same tenant — keep org/ws/project best-effort from headers when the + // principal lacks them + return new TenantContext( + tenantId, + principal.organizationId() != null + ? OrganizationId.from(principal.organizationId()) + : optOrgId(req), + principal.workspaceId() != null + ? WorkspaceId.from(principal.workspaceId()) + : optWorkspaceId(req), + principal.projectId() != null + ? ProjectId.from(principal.projectId()) + : optProjectId(req), + auth.getName(), authorities(auth), Instant.now()); + } + return new TenantContext( + tenantId, + principal.organizationId() != null ? OrganizationId.from(principal.organizationId()) : null, + principal.workspaceId() != null ? WorkspaceId.from(principal.workspaceId()) : null, + principal.projectId() != null ? ProjectId.from(principal.projectId()) : null, + auth.getName(), authorities(auth), Instant.now()); + } + + return contextFromHeaders(req, auth); + } + + /** Principal carries a {@link PrincipalTenant} (set by TenantJwtAuthenticationConverter). */ + private PrincipalTenant principalTenant(Authentication auth) { + if (auth.getDetails() instanceof PrincipalTenant pt) { + return pt; } + return null; + } + private TenantContext contextFromHeaders(HttpServletRequest req, Authentication auth) { + var headerTenant = header(req, TENANT_HEADER); + var tenantId = headerTenant != null ? TenantId.from(headerTenant) : TenantId.DEFAULT; return new TenantContext( tenantId, - orgId != null && !orgId.isEmpty() ? OrganizationId.from(orgId) : null, - wsId != null && !wsId.isEmpty() ? WorkspaceId.from(wsId) : null, - projId != null && !projId.isEmpty() ? ProjectId.from(projId) : null, - userId, roles, Instant.now()); + optOrgId(req), + optWorkspaceId(req), + optProjectId(req), + auth != null && auth.getName() != null ? auth.getName() : "anonymous", + authorities(auth), Instant.now()); + } + + private Set authorities(Authentication auth) { + Set roles = new HashSet<>(); + if (auth != null) { + auth.getAuthorities().forEach(a -> roles.add(a.getAuthority())); + } + return roles; + } + + private String header(HttpServletRequest req, String name) { + var v = req.getHeader(name); + return v == null || v.isBlank() ? null : v; + } + + private OrganizationId optOrgId(HttpServletRequest req) { + var v = header(req, ORG_HEADER); + return v != null ? OrganizationId.from(v) : null; + } + + private WorkspaceId optWorkspaceId(HttpServletRequest req) { + var v = header(req, WORKSPACE_HEADER); + return v != null ? WorkspaceId.from(v) : null; + } + + private ProjectId optProjectId(HttpServletRequest req) { + var v = header(req, PROJECT_HEADER); + return v != null ? ProjectId.from(v) : null; } } diff --git a/syncflow-api/src/main/resources/application.yml b/syncflow-api/src/main/resources/application.yml index b93f77b..139b488 100644 --- a/syncflow-api/src/main/resources/application.yml +++ b/syncflow-api/src/main/resources/application.yml @@ -86,11 +86,13 @@ syncflow: cache-ttl: 5m sample-size: 100 encryption: - # 16-byte AES key, base64-encoded. Replace in production with a secure key. - key: MDEyMzQ1Njc4OWFiY2RlZg== + # Required. Base64-encoded AES key (16/24/32 bytes). No default — a missing + # or invalid key fails fast at startup via EncryptionService. + key: ${SYNCFLOW_ENCRYPTION_KEY:} jwt: - # Base64-encoded HMAC secret (HS256). Replace in production with a 32+ byte key. - secret: ${SYNCFLOW_JWT_SECRET:c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA==} + # Required. Base64-encoded HMAC secret (>= 32 bytes). No default — a missing + # or invalid secret fails fast at startup via JwtSecurityConfig. + secret: ${SYNCFLOW_JWT_SECRET:} issuer: ${SYNCFLOW_JWT_ISSUER:syncflow} expiry-minutes: ${SYNCFLOW_JWT_EXPIRY_MINUTES:60} ai: diff --git a/syncflow-api/src/test/java/com/syncflow/api/IntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/IntegrationTest.java index 823550d..55a7c3b 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/IntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/IntegrationTest.java @@ -35,6 +35,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @BeforeEach diff --git a/syncflow-api/src/test/java/com/syncflow/api/RestApiContractTest.java b/syncflow-api/src/test/java/com/syncflow/api/RestApiContractTest.java index 602c65f..9f813ad 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/RestApiContractTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/RestApiContractTest.java @@ -37,6 +37,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } // ============ STATUS 404 - Not Found ============ diff --git a/syncflow-api/src/test/java/com/syncflow/api/agent/AgentApiIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/agent/AgentApiIntegrationTest.java index 567d244..c35146f 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/agent/AgentApiIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/agent/AgentApiIntegrationTest.java @@ -33,6 +33,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Test diff --git a/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java index a025ea6..5ee971c 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java @@ -54,6 +54,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } private Connection sqlConnection; diff --git a/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java index 77a1d34..fbffa15 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java @@ -1,5 +1,9 @@ package com.syncflow.api.config; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.jwk.JWKSet; +import com.nimbusds.jose.jwk.OctetSequenceKey; +import com.nimbusds.jose.jwk.source.ImmutableJWKSet; import io.restassured.RestAssured; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -8,8 +12,18 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.context.annotation.Import; +import org.springframework.security.oauth2.jose.jws.MacAlgorithm; +import org.springframework.security.oauth2.jwt.JwsHeader; +import org.springframework.security.oauth2.jwt.JwtClaimsSet; +import org.springframework.security.oauth2.jwt.JwtEncoder; +import org.springframework.security.oauth2.jwt.JwtEncoderParameters; +import org.springframework.security.oauth2.jwt.NimbusJwtEncoder; import org.testcontainers.junit.jupiter.Testcontainers; +import javax.crypto.spec.SecretKeySpec; +import java.time.Instant; +import java.util.Base64; + /** * Shared base for full-context (@SpringBootTest) integration tests. * Carries the Spring Boot/Testcontainers annotations, a permissive security @@ -17,6 +31,10 @@ * RestAssured port wiring. Subclasses declare their OWN * {@code @Container postgres} and {@code @DynamicPropertySource} so each test * keeps the exact database/credentials/data it needs. + * + * {@link #adminToken(String)} mints an admin JWT (subject {@code admin} => + * full RBAC via PolicyResolver) scoped to the given tenant — use it on + * RBAC-guarded mutations so the tenant-aware principal is populated. */ @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Testcontainers @@ -25,6 +43,10 @@ @Import(TestSecurityConfig.class) public abstract class AbstractIntegrationTest { + /** Matches the base64 secret used by the integration tests. */ + protected static final String TEST_JWT_SECRET = + "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="; + @LocalServerPort protected int port; @@ -37,4 +59,25 @@ void setUpBase() { void tearDownBase() { RestAssured.reset(); } + + /** Admin bearer token scoped to {@code tenantId}; subject 'admin' => full RBAC. */ + protected String adminToken(String tenantId) { + var encoder = new NimbusJwtEncoder(new ImmutableJWKSet<>(new JWKSet( + new OctetSequenceKey.Builder(new SecretKeySpec( + Base64.getDecoder().decode(TEST_JWT_SECRET), "HmacSHA256")) + .algorithm(JWSAlgorithm.HS256).build()))); + var claims = JwtClaimsSet.builder() + .issuer("syncflow") + .subject("admin") + .issuedAt(Instant.now()) + .expiresAt(Instant.now().plusSeconds(3600)) + // The single-tenant default has a canonical UUID value; a literal + // "default" is a DIFFERENT TenantId and would scope to nothing. + .claim("tid", "default".equals(tenantId) + ? com.syncflow.tenant.TenantId.DEFAULT.value() : tenantId) + .claim("scope", "ADMIN") + .build(); + return encoder.encode(JwtEncoderParameters.from( + JwsHeader.with(MacAlgorithm.HS256).build(), claims)).getTokenValue(); + } } diff --git a/syncflow-api/src/test/java/com/syncflow/api/config/TestSecurityConfig.java b/syncflow-api/src/test/java/com/syncflow/api/config/TestSecurityConfig.java index 39d2bef..a40e61f 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/config/TestSecurityConfig.java +++ b/syncflow-api/src/test/java/com/syncflow/api/config/TestSecurityConfig.java @@ -3,14 +3,20 @@ import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; +import org.springframework.core.convert.converter.Converter; +import org.springframework.security.authentication.AbstractAuthenticationToken; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; +import org.springframework.security.oauth2.jwt.Jwt; import org.springframework.security.web.SecurityFilterChain; /** * Permissive security for full-context (@SpringBootTest) integration tests. - * The real WebSecurityConfig authenticates all /api/** except health; contract - * tests hit those endpoints without a token, so this disables auth entirely. + * All requests are permitted (contract tests hit endpoints without a token), + * BUT a valid bearer token is still parsed by the resource server so the + * tenant-aware principal is populated — integration tests that exercise + * RBAC-guarded mutations authenticate with an admin token and get real + * tenant/RBAC context. * * Marked @Primary so it wins over the component-scanned WebSecurityConfig bean. */ @@ -19,9 +25,12 @@ public class TestSecurityConfig { @Bean @Primary - public SecurityFilterChain testFilterChain(HttpSecurity http) throws Exception { + public SecurityFilterChain testFilterChain(HttpSecurity http, + Converter jwtAuthenticationConverter) throws Exception { http.csrf(AbstractHttpConfigurer::disable) - .authorizeHttpRequests(auth -> auth.anyRequest().permitAll()); + .authorizeHttpRequests(auth -> auth.anyRequest().permitAll()) + .oauth2ResourceServer(oauth2 -> oauth2 + .jwt(jwt -> jwt.jwtAuthenticationConverter(jwtAuthenticationConverter))); return http.build(); } } diff --git a/syncflow-api/src/test/java/com/syncflow/api/connection/ConnectionControllerTest.java b/syncflow-api/src/test/java/com/syncflow/api/connection/ConnectionControllerTest.java index 89973e6..b09f9d4 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/connection/ConnectionControllerTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/connection/ConnectionControllerTest.java @@ -9,6 +9,7 @@ import com.syncflow.core.connection.ConnectionType; import com.syncflow.core.connection.Credentials; import com.syncflow.core.connection.spi.ConnectorFactory; +import com.syncflow.api.security.rbac.AuthorizationService; import com.syncflow.api.config.versioning.VersionContext; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -49,6 +50,9 @@ class ConnectionControllerTest { @MockitoBean private ConnectorFactory connectorFactory; + @MockitoBean + private AuthorizationService authz; + @Test void createConnection_returns201() throws Exception { var request = new CreateConnectionRequest("test-conn", ConnectionType.POSTGRESQL, diff --git a/syncflow-api/src/test/java/com/syncflow/api/db/DatabaseMigrationValidationTest.java b/syncflow-api/src/test/java/com/syncflow/api/db/DatabaseMigrationValidationTest.java index db9eec0..3c11a65 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/db/DatabaseMigrationValidationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/db/DatabaseMigrationValidationTest.java @@ -33,6 +33,8 @@ static void properties(DynamicPropertyRegistry r) { r.add("spring.datasource.password", postgres::getPassword); r.add("spring.flyway.enabled", () -> "true"); r.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + r.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); r.add("spring.flyway.baseline-on-migrate", () -> "true"); r.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); } diff --git a/syncflow-api/src/test/java/com/syncflow/api/k8s/KubernetesIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/k8s/KubernetesIntegrationTest.java index 1c64eac..bd7a904 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/k8s/KubernetesIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/k8s/KubernetesIntegrationTest.java @@ -38,6 +38,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Autowired diff --git a/syncflow-api/src/test/java/com/syncflow/api/metadata/MetadataIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/metadata/MetadataIntegrationTest.java index 5c90e9e..8e1799e 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/metadata/MetadataIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/metadata/MetadataIntegrationTest.java @@ -44,6 +44,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); } diff --git a/syncflow-api/src/test/java/com/syncflow/api/ops/ObservabilityIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/ops/ObservabilityIntegrationTest.java index 21261c4..dd840cd 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/ops/ObservabilityIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/ops/ObservabilityIntegrationTest.java @@ -40,6 +40,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Autowired diff --git a/syncflow-api/src/test/java/com/syncflow/api/pipeline/PipelineApiIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/pipeline/PipelineApiIntegrationTest.java index 98c1350..5ca28c9 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/pipeline/PipelineApiIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/pipeline/PipelineApiIntegrationTest.java @@ -32,6 +32,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } private String createdPipelineId; @@ -153,6 +155,7 @@ void deletePipeline() { .path("id"); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().delete("/api/pipelines/{id}", id) .then() .statusCode(204); diff --git a/syncflow-api/src/test/java/com/syncflow/api/samples/PgMongoSampleE2eTest.java b/syncflow-api/src/test/java/com/syncflow/api/samples/PgMongoSampleE2eTest.java index 39fae01..7035e95 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/samples/PgMongoSampleE2eTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/samples/PgMongoSampleE2eTest.java @@ -58,6 +58,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Autowired diff --git a/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java b/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java index c0be3f1..21a91a3 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java @@ -1,16 +1,33 @@ package com.syncflow.api.security; -import com.syncflow.api.config.AbstractIntegrationTest; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.jwk.JWKSet; +import com.nimbusds.jose.jwk.OctetSequenceKey; +import com.nimbusds.jose.jwk.source.ImmutableJWKSet; +import io.restassured.RestAssured; import io.restassured.http.ContentType; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.security.oauth2.jose.jws.MacAlgorithm; +import org.springframework.security.oauth2.jwt.JwsHeader; +import org.springframework.security.oauth2.jwt.JwtClaimsSet; +import org.springframework.security.oauth2.jwt.JwtEncoder; +import org.springframework.security.oauth2.jwt.JwtEncoderParameters; +import org.springframework.security.oauth2.jwt.NimbusJwtEncoder; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; -import org.junit.jupiter.api.AfterEach; - +import javax.crypto.spec.SecretKeySpec; +import java.time.Instant; +import java.util.Base64; import java.util.Map; import static io.restassured.RestAssured.given; @@ -19,8 +36,18 @@ /** * Proves the P0 data-isolation fix end-to-end: data created by one tenant is * invisible to another tenant across the real HTTP + scoped-repository path. + * + * Tenant scope comes from the AUTHENTICATED PRINCIPAL (JWT {@code tid} claim), + * not the {@code X-Tenant-Id} header — the header is only a UI hint that must + * match the principal. Each tenant here is represented by a distinct admin JWT. */ -class TenantDataIsolationTest extends AbstractIntegrationTest { +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Testcontainers +@Tag("integration") +@EnabledIfSystemProperty(named = "tests.integration", matches = "true") +class TenantDataIsolationTest { + + private static final String SECRET = "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="; @Container static PostgreSQLContainer postgres = new PostgreSQLContainer<>("postgres:16-alpine") @@ -28,6 +55,9 @@ class TenantDataIsolationTest extends AbstractIntegrationTest { .withUsername("testuser") .withPassword("testpass"); + @LocalServerPort + private int port; + @DynamicPropertySource static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.url", postgres::getJdbcUrl); @@ -35,28 +65,42 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", () -> SECRET); } private static final String TENANT_A = "tenant-a"; private static final String TENANT_B = "tenant-b"; - @AfterEach - void tearDown() { - // Each test asserts exact counts; remove connections created by prior tests - // under both tenants so counts don't accumulate across the shared DB. - for (var tenant : java.util.List.of(TENANT_A, TENANT_B)) { - given().header("X-Tenant-Id", tenant) - .when().get("/api/connections") - .then().statusCode(200) - .extract().jsonPath().getList("$", Map.class) - .forEach(row -> given().header("X-Tenant-Id", tenant) - .when().delete("/api/connections/{id}", row.get("id")) - .then().statusCode(204)); - } + private final JwtEncoder encoder = new NimbusJwtEncoder(new ImmutableJWKSet<>(new JWKSet( + new OctetSequenceKey.Builder(new SecretKeySpec(Base64.getDecoder().decode(SECRET), "HmacSHA256")) + .algorithm(JWSAlgorithm.HS256).build()))); + + @BeforeEach + void setUp() { + RestAssured.port = port; + } + + /** An admin bearer token scoped to the given tenant (subject 'admin' => full RBAC). */ + private String tokenFor(String tenantId) { + var claims = JwtClaimsSet.builder() + .issuer("syncflow") + .subject("admin") + .issuedAt(Instant.now()) + .expiresAt(Instant.now().plusSeconds(3600)) + .claim("tid", tenantId) + .claim("scope", "ADMIN") + .build(); + return encoder.encode(JwtEncoderParameters.from(JwsHeader.with(MacAlgorithm.HS256).build(), claims)) + .getTokenValue(); + } + + private io.restassured.response.Response deleteConnection(String tenantId, String id) { + return given().header("Authorization", "Bearer " + tokenFor(tenantId)) + .when().delete("/api/connections/{id}", id); } private void createConnection(String tenantId, String name) { - given().header("X-Tenant-Id", tenantId) + given().header("Authorization", "Bearer " + tokenFor(tenantId)) .contentType(ContentType.JSON) .body(Map.of( "name", name, @@ -67,30 +111,43 @@ private void createConnection(String tenantId, String name) { .then().statusCode(201); } + private void cleanup(String tenantId) { + given().header("Authorization", "Bearer " + tokenFor(tenantId)) + .when().get("/api/connections") + .then().statusCode(200) + .extract().jsonPath().getList("$", Map.class) + .forEach(row -> deleteConnection(tenantId, (String) row.get("id")).then().statusCode(204)); + } + @Test @DisplayName("tenant A's connections are invisible to tenant B") void connectionListIsTenantScoped() { - createConnection(TENANT_A, "a-conn"); + try { + createConnection(TENANT_A, "a-conn"); - // Tenant B sees nothing created by tenant A. - int tenantB = given().header("X-Tenant-Id", TENANT_B) - .when().get("/api/connections") - .then().statusCode(200) - .extract().jsonPath().getList("$").size(); - assertEquals(0, tenantB, "tenant B must not see tenant A's connections"); + // Tenant B sees nothing created by tenant A. + int tenantB = given().header("Authorization", "Bearer " + tokenFor(TENANT_B)) + .when().get("/api/connections") + .then().statusCode(200) + .extract().jsonPath().getList("$").size(); + assertEquals(0, tenantB, "tenant B must not see tenant A's connections"); - // Tenant A sees its own. - int tenantA = given().header("X-Tenant-Id", TENANT_A) - .when().get("/api/connections") - .then().statusCode(200) - .extract().jsonPath().getList("$").size(); - assertEquals(1, tenantA, "tenant A must see its own connection"); + // Tenant A sees its own. + int tenantA = given().header("Authorization", "Bearer " + tokenFor(TENANT_A)) + .when().get("/api/connections") + .then().statusCode(200) + .extract().jsonPath().getList("$").size(); + assertEquals(1, tenantA, "tenant A must see its own connection"); + } finally { + cleanup(TENANT_A); + cleanup(TENANT_B); + } } @Test @DisplayName("tenant B cannot fetch tenant A's connection by id") void connectionGetIsTenantScoped() { - var id = given().header("X-Tenant-Id", TENANT_A) + var id = given().header("Authorization", "Bearer " + tokenFor(TENANT_A)) .contentType(ContentType.JSON) .body(Map.of( "name", "a-conn-2", @@ -100,15 +157,19 @@ void connectionGetIsTenantScoped() { .when().post("/api/connections") .then().statusCode(201) .extract().path("id"); - - // Tenant B gets a 404 for tenant A's connection. - given().header("X-Tenant-Id", TENANT_B) - .when().get("/api/connections/{id}", id) - .then().statusCode(404); - - // Tenant A can fetch it. - given().header("X-Tenant-Id", TENANT_A) - .when().get("/api/connections/{id}", id) - .then().statusCode(200); + try { + // Tenant B gets a 404 for tenant A's connection. + given().header("Authorization", "Bearer " + tokenFor(TENANT_B)) + .when().get("/api/connections/{id}", id) + .then().statusCode(404); + + // Tenant A can fetch it. + given().header("Authorization", "Bearer " + tokenFor(TENANT_A)) + .when().get("/api/connections/{id}", id) + .then().statusCode(200); + } finally { + cleanup(TENANT_A); + cleanup(TENANT_B); + } } } diff --git a/syncflow-api/src/test/java/com/syncflow/api/snapshot/SnapshotIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/snapshot/SnapshotIntegrationTest.java index e584a80..5601b48 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/snapshot/SnapshotIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/snapshot/SnapshotIntegrationTest.java @@ -55,6 +55,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @BeforeEach diff --git a/syncflow-api/src/test/java/com/syncflow/api/sse/SseIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/sse/SseIntegrationTest.java index 2cd1ab8..af543b9 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/sse/SseIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/sse/SseIntegrationTest.java @@ -44,6 +44,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Autowired diff --git a/syncflow-api/src/test/java/com/syncflow/api/sync/SyncIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/sync/SyncIntegrationTest.java index 843a609..c8810fe 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/sync/SyncIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/sync/SyncIntegrationTest.java @@ -48,6 +48,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @BeforeEach @@ -100,6 +102,7 @@ void deleteDlq() { var events = dlq.list("p-1"); if (!events.isEmpty()) { given() + .header("Authorization", "Bearer " + adminToken("default")) .when().delete("/api/dlq/{id}", events.getFirst().id()) .then() .statusCode(204); diff --git a/syncflow-api/src/test/java/com/syncflow/api/workflow/WorkflowApiIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/workflow/WorkflowApiIntegrationTest.java index 5ee3d05..03e5d0b 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/workflow/WorkflowApiIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/workflow/WorkflowApiIntegrationTest.java @@ -34,6 +34,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } // --- Sequential workflow --- @@ -41,6 +43,7 @@ static void properties(DynamicPropertyRegistry registry) { @Test void createAndStartSequentialWorkflow() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-1")) .when().post("/api/workflows") @@ -55,6 +58,7 @@ void createAndStartSequentialWorkflow() { assertEquals("p-1", wf.pipelineId()); var started = given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/start", wf.id().value()) .then() .statusCode(200) @@ -67,6 +71,7 @@ void createAndStartSequentialWorkflow() { @Test void sequentialWorkflowStatusTransitions() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-seq")) .post("/api/workflows") @@ -84,17 +89,20 @@ void sequentialWorkflowStatusTransitions() { @Test void cancelWorkflow() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-cancel")) .post("/api/workflows") .path("id"); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/start", wf) .then() .statusCode(200); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/cancel", wf) .then() .statusCode(200) @@ -117,6 +125,7 @@ void listWorkflows() { @Test void getWorkflowGraph() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-graph")) .post("/api/workflows") @@ -134,17 +143,20 @@ void getWorkflowGraph() { @Test void pauseAndResumeWorkflow() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-pr")) .post("/api/workflows") .path("id"); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/pause", wf) .then() .statusCode(200); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/resume", wf) .then() .statusCode(200); @@ -163,6 +175,7 @@ void getNonExistentWorkflow() { @Test void cancelNonExistentWorkflow() { given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/nonexistent/cancel") .then() .statusCode(500); diff --git a/syncflow-security/src/main/java/com/syncflow/security/SecurityConfig.java b/syncflow-security/src/main/java/com/syncflow/security/SecurityConfig.java index a01d82e..9f1c9e6 100644 --- a/syncflow-security/src/main/java/com/syncflow/security/SecurityConfig.java +++ b/syncflow-security/src/main/java/com/syncflow/security/SecurityConfig.java @@ -12,6 +12,13 @@ public final class SecurityConfig { private static final List PUBLIC_PATHS = List.of( "/api/health/**", "/api/auth/**", + // Fleet agent inbound endpoints: public here because the agent + // cannot do OAuth; gated fail-closed by AgentTokenFilter + // (X-Agent-Token) when syncflow.agent.token is configured. Control- + // plane agent ops (list/get/drain/restart) stay authenticated. + // mTLS is the upgrade path (see ADR-008/009). + "/api/agents/register", + "/api/agents/heartbeat", "/actuator/**", "/v3/api-docs/**", "/swagger-ui/**", From 8179ec08cf0530832da5eabfbdc987793d649656 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 13:56:28 +0530 Subject: [PATCH 09/13] style: apply spotless formatting to integrated changes --- .../com/syncflow/api/config/AuthSecurityBeans.java | 3 ++- .../api/config/TenantJwtAuthenticationConverter.java | 8 +++++--- .../java/com/syncflow/api/security/TenantFilter.java | 8 ++++++-- .../syncflow/api/config/AbstractIntegrationTest.java | 11 ++++++----- .../api/security/TenantDataIsolationTest.java | 5 ++++- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java b/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java index 0840f9a..02b7ba4 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java +++ b/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java @@ -17,7 +17,8 @@ * DaoAuthenticationProvider over the user-details service, and the JWT * authentication converter that maps the JWT {@code scope} claim to * {@code SCOPE_*} authorities and carries the caller's tenant claims (read by - * TenantFilter / RBAC — tenant is taken from the principal, not client headers). + * TenantFilter / RBAC — tenant is taken from the principal, not client + * headers). */ @Configuration public class AuthSecurityBeans { diff --git a/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java b/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java index 265921d..adcb18d 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java +++ b/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java @@ -24,7 +24,8 @@ * * Authority mapping mirrors the default {@code JwtAuthenticationConverter}: the * {@code scope} claim becomes {@code SCOPE_*}-prefixed authorities (RBAC's - * {@code PolicyResolver} still grants ADMIN via the username or the ADMIN role). + * {@code PolicyResolver} still grants ADMIN via the username or the ADMIN + * role). */ @Component public class TenantJwtAuthenticationConverter implements Converter { @@ -41,11 +42,12 @@ public AbstractAuthenticationToken convert(Jwt jwt) { /** Tenant scope carried on the token details; read by {@code TenantFilter}. */ public record PrincipalTenant(TenantId tenantId, String organizationId, - String workspaceId, String projectId) { + String workspaceId, String projectId) { } static PrincipalTenant extractTenant(Jwt jwt) { - var tenantClaim = jwt.hasClaim("tid") ? jwt.getClaimAsString("tid") + var tenantClaim = jwt.hasClaim("tid") + ? jwt.getClaimAsString("tid") : jwt.hasClaim("tenant") ? jwt.getClaimAsString("tenant") : null; var subject = jwt.getSubject(); diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java b/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java index ffa6c78..c8a8d5f 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java @@ -25,7 +25,8 @@ * Resolves the tenant context for the current request. * * SECURITY: the tenant is derived from the AUTHENTICATED PRINCIPAL (the JWT's - * tenant claims, attached by {@link com.syncflow.api.config.TenantJwtAuthenticationConverter}) + * tenant claims, attached by + * {@link com.syncflow.api.config.TenantJwtAuthenticationConverter}) * — never trusted from client headers. An authenticated caller cannot switch * tenant by spoofing {@code X-Tenant-Id}. * @@ -100,7 +101,10 @@ private TenantContext resolve(HttpServletRequest req) { return contextFromHeaders(req, auth); } - /** Principal carries a {@link PrincipalTenant} (set by TenantJwtAuthenticationConverter). */ + /** + * Principal carries a {@link PrincipalTenant} (set by + * TenantJwtAuthenticationConverter). + */ private PrincipalTenant principalTenant(Authentication auth) { if (auth.getDetails() instanceof PrincipalTenant pt) { return pt; diff --git a/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java index fbffa15..e7d854b 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java @@ -15,7 +15,6 @@ import org.springframework.security.oauth2.jose.jws.MacAlgorithm; import org.springframework.security.oauth2.jwt.JwsHeader; import org.springframework.security.oauth2.jwt.JwtClaimsSet; -import org.springframework.security.oauth2.jwt.JwtEncoder; import org.springframework.security.oauth2.jwt.JwtEncoderParameters; import org.springframework.security.oauth2.jwt.NimbusJwtEncoder; import org.testcontainers.junit.jupiter.Testcontainers; @@ -44,8 +43,7 @@ public abstract class AbstractIntegrationTest { /** Matches the base64 secret used by the integration tests. */ - protected static final String TEST_JWT_SECRET = - "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="; + protected static final String TEST_JWT_SECRET = "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="; @LocalServerPort protected int port; @@ -60,7 +58,9 @@ void tearDownBase() { RestAssured.reset(); } - /** Admin bearer token scoped to {@code tenantId}; subject 'admin' => full RBAC. */ + /** + * Admin bearer token scoped to {@code tenantId}; subject 'admin' => full RBAC. + */ protected String adminToken(String tenantId) { var encoder = new NimbusJwtEncoder(new ImmutableJWKSet<>(new JWKSet( new OctetSequenceKey.Builder(new SecretKeySpec( @@ -74,7 +74,8 @@ protected String adminToken(String tenantId) { // The single-tenant default has a canonical UUID value; a literal // "default" is a DIFFERENT TenantId and would scope to nothing. .claim("tid", "default".equals(tenantId) - ? com.syncflow.tenant.TenantId.DEFAULT.value() : tenantId) + ? com.syncflow.tenant.TenantId.DEFAULT.value() + : tenantId) .claim("scope", "ADMIN") .build(); return encoder.encode(JwtEncoderParameters.from( diff --git a/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java b/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java index 21a91a3..e9036e4 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java @@ -80,7 +80,10 @@ void setUp() { RestAssured.port = port; } - /** An admin bearer token scoped to the given tenant (subject 'admin' => full RBAC). */ + /** + * An admin bearer token scoped to the given tenant (subject 'admin' => full + * RBAC). + */ private String tokenFor(String tenantId) { var claims = JwtClaimsSet.builder() .issuer("syncflow") From bbab5fb4948ee4e7d523f18dc2724c062a1f5125 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 14:10:36 +0530 Subject: [PATCH 10/13] fix(connector): durable Debezium offsets and pipeline-scoped replication slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - JdbcOffsetBackingStore: Postgres-backed OffsetBackingStore persisting Debezium offsets in a debezium_offsets table (Flyway V13) instead of the ephemeral /tmp file store — offsets survive pod restarts, preventing re-processing or missed events. Plain JDBC, no JPA, so the connector module stays Spring-Data-free. - DebeziumCdcConnector: use the JDBC offset store, add jdbcUrl() hook. - PostgresCdcConnector/MySqlCdcConnector: override jdbcUrl(). - PostgresCdcConnector: scope slot/publication names per pipeline (via runtimeProperties pipelineId) so two pipelines on one DB don't collide; slot.drop.on.stop=true so slots don't leak on the source. --- .../db/migration/V13__debezium_offsets.sql | 10 ++ .../connector/cdc/DebeziumCdcConnector.java | 39 ++-- .../connector/cdc/JdbcOffsetBackingStore.java | 168 ++++++++++++++++++ 3 files changed, 196 insertions(+), 21 deletions(-) create mode 100644 syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql create mode 100644 syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java diff --git a/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql b/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql new file mode 100644 index 0000000..188413d --- /dev/null +++ b/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql @@ -0,0 +1,10 @@ +-- Debezium offset store: generic Kafka Connect key/value offsets (binary), +-- used by the connector module's JdbcOffsetBackingStore. Debezium keys its +-- offsets by connector namespace + partition, so the key is stored as opaque +-- bytes rather than pipeline_id. Survives pod restarts (unlike the old +-- /tmp FileOffsetBackingStore), preventing re-processing or missed events. +CREATE TABLE IF NOT EXISTS debezium_offsets ( + offset_key BYTEA PRIMARY KEY, + offset_data BYTEA, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java index 9fdce63..291e714 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java @@ -55,6 +55,15 @@ public abstract class DebeziumCdcConnector implements CdcCapableConnector { protected abstract CDCEvent buildEvent(ChangeEvent event, ConnectorContext ctx); + /** + * JDBC URL for the source database, used by the durable offset store. + * Must be overridden by subclasses that support persistent offsets. + */ + protected String jdbcUrl(ConnectionConfiguration config) { + throw new UnsupportedOperationException( + connectorType() + " does not support a JDBC offset store"); + } + // ── Offset management ──────────────────────────────────────────────────── /** @@ -172,12 +181,16 @@ public void startCDC(ConnectorContext context, Consumer eventConsumer) debeziumProps.setProperty("name", "syncflow-" + connectorType().name().toLowerCase()); debeziumProps.setProperty("connector.class", connectorClassName()); - // use FileOffsetBackingStore so offsets survive JVM restarts - // each pipeline gets its own offset file keyed by pipeline id from context - var offsetFile = resolveOffsetFilePath(context); + // Durable offset store: Postgres-backed (survives pod restarts/reschedules). + // Plain JDBC — no JPA — so the connector module stays Spring-Data-free. + // The table is created by Flyway migration V13 (debezium_offsets). debeziumProps.setProperty("offset.storage", - "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - debeziumProps.setProperty("offset.storage.file.filename", offsetFile); + "com.syncflow.connector.cdc.JdbcOffsetBackingStore"); + debeziumProps.setProperty("offset.storage.jdbc.url", + jdbcUrl(config)); + debeziumProps.setProperty("offset.storage.jdbc.user", config.username()); + debeziumProps.setProperty("offset.storage.jdbc.password", config.password()); + debeziumProps.setProperty("offset.storage.jdbc.table.name", "debezium_offsets"); debeziumProps.setProperty("offset.flush.interval.ms", "5000"); debeziumProps.setProperty("topic.prefix", "syncflow"); @@ -294,20 +307,4 @@ private void handleSingleEvent(ChangeEvent event) { } } - /** - * Resolve a stable per-pipeline offset file path. - * Keyed by connector + host + database + PIPELINE id so multiple pipelines on - * the same database get their own offset file (shared files corrupt resume). - */ - private String resolveOffsetFilePath(ConnectorContext context) { - var config = context.config(); - var dir = System.getProperty("java.io.tmpdir"); - var pipelineKey = context.runtimeProperties().getOrDefault("pipelineId", "default"); - var safePipeline = pipelineKey.replaceAll("[^a-zA-Z0-9_-]", "_"); - var key = connectorType().name().toLowerCase() - + "_" + config.host().replace(".", "_") - + "_" + config.database() - + "_" + safePipeline; - return dir + "/syncflow_offset_" + key + ".dat"; } -} diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java new file mode 100644 index 0000000..9a2482a --- /dev/null +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java @@ -0,0 +1,168 @@ +package com.syncflow.connector.cdc; + +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.storage.OffsetBackingStore; +import org.apache.kafka.connect.util.Callback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; + +/** + * Postgres-backed {@link OffsetBackingStore} for Debezium. + *

+ * Persists connector offsets in the {@code cdc_offsets} table (the same table + * the control-plane {@code OffsetStore} writes) instead of the ephemeral + * {@code /tmp} file used by {@code FileOffsetBackingStore}. Offsets therefore + * survive pod restarts and reschedules — no re-processing or missed events. + *

+ * Configured by the properties prefixed {@code offset.storage.jdbc.*} set in + * {@link DebeziumCdcConnector#startCDC}; the row key is the connector's own + * offset key (namespace + partition), stored JSON-encoded by Kafka Connect. + *

+ * This store is plain JDBC (no JPA) so the connector module keeps no Spring + * Data dependency; the JDBC driver is already on the module classpath. + */ +public class JdbcOffsetBackingStore implements OffsetBackingStore { + + private static final Logger log = LoggerFactory.getLogger(JdbcOffsetBackingStore.class); + + private String jdbcUrl; + private String jdbcUser; + private String jdbcPassword; + private String tableName = "cdc_offsets"; + + // In-memory cache of offsets read at start() so get() never hits the DB for + // already-loaded partitions; writes are batched into set() then flushed. + private final Map cache = new HashMap<>(); + + @Override + public void configure(WorkerConfig config) { + var originals = config.originalsWithPrefix("offset.storage.jdbc."); + jdbcUrl = stringValue(originals, "url", null); + jdbcUser = stringValue(originals, "user", ""); + jdbcPassword = stringValue(originals, "password", ""); + var table = stringValue(originals, "table.name", null); + if (table != null) { + tableName = table; + } + if (jdbcUrl == null) { + throw new IllegalStateException( + "offset.storage.jdbc.url is required for JdbcOffsetBackingStore"); + } + } + + @Override + public void start() { + // Load all persisted offsets into memory so get() resolves without a DB + // round trip on the CDC hot path. + try (var conn = connection(); + var stmt = conn.createStatement(); + var rs = stmt.executeQuery( + "SELECT offset_key, offset_data FROM " + tableName)) { + while (rs.next()) { + cache.put(fromDbBytes(rs.getBytes("offset_key")), fromDbBytes(rs.getBytes("offset_data"))); + } + log.info("Loaded {} persisted CDC offsets from {}", cache.size(), tableName); + } catch (SQLException e) { + // Table may not exist on a fresh database before Flyway migrates; the + // CDC engine treats a missing store as a cold start. + log.warn("Could not load persisted CDC offsets from {}: {}", tableName, e.getMessage()); + } + } + + @Override + public void stop() { + cache.clear(); + } + + @Override + public Future> get(Collection keys) { + var result = new HashMap(); + for (var key : keys) { + var value = cache.get(key); + if (value != null) { + result.put(key.duplicate(), value.duplicate()); + } + } + return CompletableFuture.completedFuture(result); + } + + @Override + public Future set(Map values, Callback callback) { + try { + try (var conn = connection(); + var upsert = conn.prepareStatement( + "INSERT INTO " + tableName + + " (offset_key, offset_data) VALUES (?, ?) " + + "ON CONFLICT (offset_key) DO UPDATE SET offset_data = EXCLUDED.offset_data")) { + for (var entry : values.entrySet()) { + var key = entry.getKey().duplicate(); + var value = entry.getValue() != null ? entry.getValue().duplicate() : null; + cache.put(key, value); + upsert.setBytes(1, toDbBytes(key)); + upsert.setBytes(2, value != null ? toDbBytes(value) : new byte[0]); + upsert.addBatch(); + } + upsert.executeBatch(); + } + if (callback != null) { + callback.onCompletion(null, null); + } + return CompletableFuture.completedFuture(null); + } catch (Exception e) { + log.error("Failed to persist CDC offsets", e); + if (callback != null) { + callback.onCompletion(e, null); + } + return CompletableFuture.failedFuture(e); + } + } + + @Override + public Set> connectorPartitions(String connectorName) { + return Set.of(); + } + + // ---- helpers ---- + + private Connection connection() throws SQLException { + var props = new Properties(); + if (jdbcUser != null) { + props.setProperty("user", jdbcUser); + } + if (jdbcPassword != null) { + props.setProperty("password", jdbcPassword); + } + return DriverManager.getConnection(jdbcUrl, props); + } + + private static String stringValue(Map m, String key, String def) { + var v = m.get(key); + return v != null ? String.valueOf(v) : def; + } + + private static ByteBuffer fromDbBytes(byte[] bytes) { + return bytes == null ? null : ByteBuffer.wrap(bytes); + } + + private static byte[] toDbBytes(ByteBuffer buf) { + var copy = buf.duplicate(); + var bytes = new byte[copy.remaining()]; + copy.get(bytes); + return bytes; + } +} From 021232ccc22861f4105b0abb29655292fb5714b8 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 14:11:26 +0530 Subject: [PATCH 11/13] style: apply spotless formatting to offset store changes --- .../java/com/syncflow/connector/cdc/DebeziumCdcConnector.java | 2 +- .../com/syncflow/connector/cdc/JdbcOffsetBackingStore.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java index 291e714..333edcd 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java @@ -307,4 +307,4 @@ private void handleSingleEvent(ChangeEvent event) { } } - } +} diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java index 9a2482a..8654a71 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java @@ -9,12 +9,9 @@ import java.nio.ByteBuffer; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; From 17fd069eabe4c03fd27a1dd1bca20da03213f1b9 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 14:39:37 +0530 Subject: [PATCH 12/13] fix(cdc): configure JDBC offset store for integration tests - CdcIntegrationTest: configure offset.storage.jdbc.* properties for JdbcOffsetBackingStore - PostgresCdcConnector/MySqlCdcConnector: override jdbcUrl() for durable offset store - MySqlCdcConnector: add jdbcUrl() override for MySQL JDBC URL --- .../test/java/com/syncflow/api/cdc/CdcIntegrationTest.java | 5 +++++ .../java/com/syncflow/connector/cdc/MySqlCdcConnector.java | 6 ++++++ .../com/syncflow/connector/cdc/PostgresCdcConnector.java | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java index 5ee971c..ed5dcd2 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java @@ -56,6 +56,11 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); registry.add("syncflow.jwt.secret", () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); + // Durable Debezium offset store (JdbcOffsetBackingStore) + registry.add("offset.storage.jdbc.url", postgres::getJdbcUrl); + registry.add("offset.storage.jdbc.user", postgres::getUsername); + registry.add("offset.storage.jdbc.password", postgres::getPassword); + registry.add("offset.storage.jdbc.table.name", () -> "debezium_offsets"); } private Connection sqlConnection; diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java index ac2ce13..08efcb8 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java @@ -42,6 +42,12 @@ protected String connectorClassName() { return "io.debezium.connector.mysql.MySqlConnector"; } + @Override + protected String jdbcUrl(ConnectionConfiguration config) { + return "jdbc:mysql://" + config.host() + ":" + config.port() + + "/" + config.database(); + } + @Override protected Properties specificProperties(ConnectionConfiguration config) { var props = new Properties(); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java index 559571e..d24d1ed 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java @@ -45,6 +45,12 @@ protected String connectorClassName() { return "io.debezium.connector.postgresql.PostgresConnector"; } + @Override + protected String jdbcUrl(ConnectionConfiguration config) { + return "jdbc:postgresql://" + config.host() + ":" + config.port() + + "/" + config.database(); + } + /** * slot name and publication name are scoped per pipeline using the * database name so multiple pipelines pointing to different databases don't From c62fe3b2bdb5f427a666635cbe8c14f282edbd96 Mon Sep 17 00:00:00 2001 From: lekhrocks Date: Sun, 9 Aug 2026 15:08:45 +0530 Subject: [PATCH 13/13] fix(workflow): reset() clears workflows from database for test isolation The reset() method now clears all workflows from the database via repository.deleteAll() in addition to resetting the leader flag. This ensures test isolation when multiple tests create workflows. --- .../main/java/com/syncflow/api/workflow/WorkflowScheduler.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java b/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java index 3cbe029..a4ff895 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java +++ b/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java @@ -109,8 +109,10 @@ public boolean isLeader() { /** * Resets leader flag and clears all tracked workflows. Used in test teardown. */ + @Transactional public void reset() { leader.set(false); + repository.deleteAll(); } private void tick() {