diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d055c05..375449e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,8 +40,12 @@ services: SPRING_DATASOURCE_USERNAME: syncflow SPRING_DATASOURCE_PASSWORD: syncflow SPRING_JPA_HIBERNATE_DDL_AUTO: validate - # JWT auth (must-change-password is on for admin-provisioned accounts). - SYNCFLOW_JWT_SECRET: ${SYNCFLOW_JWT_SECRET:-c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA==} + # ─── DEV-ONLY secrets. The app has no defaults (fail-fast) — supply real + # values in non-local environments. NEVER use these in production. + # AES-256 key = base64("dev-encryption-key-0123456789abc") (32 bytes). + SYNCFLOW_ENCRYPTION_KEY: ZGV2LWVuY3J5cHRpb24ta2V5LTAxMjM0NTY3ODlhYmM= + # HS256 secret = base64 of a 44-byte dev-only HMAC key. + SYNCFLOW_JWT_SECRET: ZGV2LWp3dC1zZWNyZXQta2V5LWZvci1sb2NhbC1kZXYtb25seS1rZXktMDE= # Kafka transport (off by default). SYNCFLOW_KAFKA_ENABLED: ${SYNCFLOW_KAFKA_ENABLED:-false} SYNCFLOW_KAFKA_BOOTSTRAP_SERVERS: kafka:9092 diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml index 80c27e3..6d36c97 100644 --- a/k8s/base/deployment.yaml +++ b/k8s/base/deployment.yaml @@ -59,6 +59,11 @@ spec: secretKeyRef: name: syncflow-encryption key: key + - name: SYNCFLOW_JWT_SECRET + valueFrom: + secretKeyRef: + name: syncflow-jwt + key: secret volumeMounts: - name: config mountPath: /app/config diff --git a/syncflow-api/src/main/java/com/syncflow/api/agent/AgentController.java b/syncflow-api/src/main/java/com/syncflow/api/agent/AgentController.java index 50593df..a719230 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/agent/AgentController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/agent/AgentController.java @@ -1,6 +1,8 @@ package com.syncflow.api.agent; import com.syncflow.agent.domain.Agent; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.agent.domain.AgentId; import com.syncflow.agent.domain.HardwareMetrics; import org.springframework.http.ResponseEntity; @@ -19,9 +21,11 @@ public class AgentController { private final FleetManager fleetManager; + private final AuthorizationService authz; - public AgentController(FleetManager fleetManager) { + public AgentController(FleetManager fleetManager, AuthorizationService authz) { this.fleetManager = fleetManager; + this.authz = authz; } @PostMapping("/register") @@ -57,11 +61,13 @@ public ResponseEntity> heartbeat(@RequestBody Map> list() { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(fleetManager.list()); } @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return fleetManager.get(new AgentId(id)) .map(ResponseEntity::ok) .orElse(ResponseEntity.notFound().build()); @@ -69,17 +75,20 @@ public ResponseEntity get(@PathVariable String id) { @PostMapping("/{id}/drain") public ResponseEntity> drain(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); fleetManager.drain(new AgentId(id)); return ResponseEntity.ok(Map.of("agentId", id, "status", "DRAINING")); } @PostMapping("/{id}/restart") public ResponseEntity> restart(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(Map.of("agentId", id, "action", "restart_requested")); } @GetMapping("/{id}/metrics") public ResponseEntity> metrics(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return fleetManager.get(new AgentId(id)) .map(a -> ResponseEntity.>ok(Map.of( "agentId", id, diff --git a/syncflow-api/src/main/java/com/syncflow/api/agent/FleetManager.java b/syncflow-api/src/main/java/com/syncflow/api/agent/FleetManager.java index 044c828..706fd6a 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/agent/FleetManager.java +++ b/syncflow-api/src/main/java/com/syncflow/api/agent/FleetManager.java @@ -1,11 +1,18 @@ package com.syncflow.api.agent; +import com.fasterxml.jackson.core.type.TypeReference; import com.syncflow.agent.domain.Agent; import com.syncflow.agent.domain.AgentId; import com.syncflow.agent.domain.AgentStatus; import com.syncflow.agent.domain.HardwareMetrics; +import com.syncflow.api.agent.entity.AgentEntity; +import com.syncflow.api.agent.repository.AgentRepository; import com.syncflow.api.ops.metrics.MetricsRegistry; +import com.syncflow.api.runtimestate.RuntimeStateJson; +import com.syncflow.tenant.TenantSupport; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Duration; import java.time.Instant; @@ -19,14 +26,28 @@ @Component public class FleetManager { - private final Map agents = new ConcurrentHashMap<>(); + private final AgentRepository repository; + private final RuntimeStateJson json; private final MetricsRegistry metrics; + + // Fast-path cache; durable source of truth is the agents table. + private final Map agents = new ConcurrentHashMap<>(); private final AtomicLong agentCounter = new AtomicLong(0); private final Map onlineByRegion = new ConcurrentHashMap<>(); private static final Duration HEARTBEAT_TIMEOUT = Duration.ofSeconds(60); + @Autowired + public FleetManager(AgentRepository repository, RuntimeStateJson json, MetricsRegistry metrics) { + this.repository = repository; + this.json = json; + this.metrics = metrics; + } + + /** Unit-test seam: in-memory fleet without a repository. */ public FleetManager(MetricsRegistry metrics) { + this.repository = null; + this.json = null; this.metrics = metrics; } @@ -45,6 +66,7 @@ private void decOnline(String region) { adder.decrement(); } + @Transactional public Agent register(String version, List capabilities, Map labels, String environment, String region, String hostname) { @@ -52,38 +74,53 @@ public Agent register(String version, List capabilities, agents.put(agent.id(), agent); agentCounter.incrementAndGet(); incOnline(region); + persist(agent); return agent; } + @Transactional public Optional heartbeat(AgentId id, HardwareMetrics hw) { return Optional.ofNullable(agents.computeIfPresent(id, (k, agent) -> { var updated = agent.withHeartbeat(hw); pruneOffline(); + persist(updated); return updated; })); } + @Transactional public void markOffline(AgentId id) { agents.computeIfPresent(id, (k, a) -> { if (a.status() == AgentStatus.ONLINE) decOnline(a.region()); - return a.withStatus(AgentStatus.OFFLINE); + var updated = a.withStatus(AgentStatus.OFFLINE); + persist(updated); + return updated; }); } + @Transactional public void drain(AgentId id) { agents.computeIfPresent(id, (k, a) -> { if (a.status() == AgentStatus.ONLINE) decOnline(a.region()); - return a.withStatus(AgentStatus.DRAINING); + var updated = a.withStatus(AgentStatus.DRAINING); + persist(updated); + return updated; }); } + @Transactional(readOnly = true) public Optional get(AgentId id) { return Optional.ofNullable(agents.get(id)); } + @Transactional(readOnly = true) public List list() { + if (repository != null) + return repository.findByTenantId(TenantSupport.tenantId()).stream() + .map(this::toDomain) + .toList(); return List.copyOf(agents.values()); } @@ -104,7 +141,42 @@ private void pruneOffline() { && a.lastHeartbeat().isBefore(threshold)) .forEach(a -> { decOnline(a.region()); - agents.put(a.id(), a.withStatus(AgentStatus.UNREACHABLE)); + var updated = a.withStatus(AgentStatus.UNREACHABLE); + agents.put(a.id(), updated); + persist(updated); }); } + + private void persist(Agent agent) { + if (repository == null) + return; // unit-test seam + var entity = repository.findById(agent.id().value()).orElseGet(AgentEntity::new); + entity.setId(agent.id().value()); + entity.setTenantId(TenantSupport.tenantId()); + entity.setVersion(agent.version()); + entity.setStatus(agent.status().name()); + entity.setCapabilities(json.toJson(agent.capabilities())); + entity.setLabels(json.toJson(agent.labels())); + entity.setEnvironment(agent.environment()); + entity.setRegion(agent.region()); + entity.setHostname(agent.hostname()); + entity.setHardware(json.toJson(agent.hardware())); + entity.setRegisteredAt(agent.registeredAt()); + entity.setLastHeartbeat(agent.lastHeartbeat()); + entity.setCreatedAt(agent.registeredAt()); + entity.setUpdatedAt(Instant.now()); + repository.save(entity); + } + + private Agent toDomain(AgentEntity e) { + return new Agent(new AgentId(e.getId()), e.getVersion(), + AgentStatus.valueOf(e.getStatus()), + json.fromJson(e.getCapabilities(), new TypeReference>() { + }), + json.fromJson(e.getLabels(), new TypeReference>() { + }), + e.getEnvironment(), e.getRegion(), e.getHostname(), + json.fromJson(e.getHardware(), HardwareMetrics.class), + e.getRegisteredAt(), e.getLastHeartbeat()); + } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/agent/entity/AgentEntity.java b/syncflow-api/src/main/java/com/syncflow/api/agent/entity/AgentEntity.java new file mode 100644 index 0000000..3eb0326 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/agent/entity/AgentEntity.java @@ -0,0 +1,69 @@ +package com.syncflow.api.agent.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** Fleet agent; capabilities, labels, hardware metrics as JSONB. */ +@Setter +@Getter +@Entity +@Table(name = "agents") +public class AgentEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(length = 50) + private String version; + + @Column(nullable = false, length = 20) + private String status; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String capabilities; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String labels; + + @Column(length = 50) + private String environment; + + @Column(length = 50) + private String region; + + @Column(length = 255) + private String hostname; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String hardware; + + @Column(name = "registered_at", nullable = false) + private Instant registeredAt; + + @Column(name = "last_heartbeat", nullable = false) + private Instant lastHeartbeat; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public AgentEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/agent/repository/AgentRepository.java b/syncflow-api/src/main/java/com/syncflow/api/agent/repository/AgentRepository.java new file mode 100644 index 0000000..ae655c8 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/agent/repository/AgentRepository.java @@ -0,0 +1,11 @@ +package com.syncflow.api.agent.repository; + +import com.syncflow.api.agent.entity.AgentEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface AgentRepository extends JpaRepository { + + List findByTenantId(String tenantId); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java b/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java index 292d5e5..02b7ba4 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java +++ b/syncflow-api/src/main/java/com/syncflow/api/config/AuthSecurityBeans.java @@ -2,19 +2,23 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.convert.converter.Converter; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.authentication.ProviderManager; import org.springframework.security.authentication.dao.DaoAuthenticationProvider; import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.security.authentication.AbstractAuthenticationToken; /** * Auth beans: BCrypt password encoder, the AuthenticationManager backed by * DaoAuthenticationProvider over the user-details service, and the JWT * authentication converter that maps the JWT {@code scope} claim to - * {@code ROLE_*} authorities (consumed by TenantFilter / RBAC). + * {@code SCOPE_*} authorities and carries the caller's tenant claims (read by + * TenantFilter / RBAC — tenant is taken from the principal, not client + * headers). */ @Configuration public class AuthSecurityBeans { @@ -34,9 +38,9 @@ public AuthenticationManager authenticationManager( } @Bean - public JwtAuthenticationConverter jwtAuthenticationConverter() { - // Map JWT 'scope' claim -> ROLE_ authorities. The login token encodes the - // user's roles into 'scope'; TenantFilter reads authorities for RBAC. - return new JwtAuthenticationConverter(); + public Converter jwtAuthenticationConverter() { + // Maps JWT 'scope' -> SCOPE_ authorities and attaches the tenant scope + // (tid/oid/wid/pid claims) to the token details. + return new TenantJwtAuthenticationConverter(); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java b/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java new file mode 100644 index 0000000..adcb18d --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/config/TenantJwtAuthenticationConverter.java @@ -0,0 +1,82 @@ +package com.syncflow.api.config; + +import com.syncflow.tenant.TenantId; +import org.springframework.core.convert.converter.Converter; +import org.springframework.security.authentication.AbstractAuthenticationToken; +import org.springframework.security.core.GrantedAuthority; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken; +import org.springframework.security.oauth2.server.resource.authentication.JwtGrantedAuthoritiesConverter; +import org.springframework.stereotype.Component; + +import java.util.Collection; + +/** + * JWT -> authentication converter that carries the caller's tenant scope + * (tenant/org/workspace/project) in the token details. TenantFilter reads the + * tenant from this principal — NEVER from client headers — so an authenticated + * caller cannot switch tenant by spoofing {@code X-Tenant-Id}. + * + * The tenant claim is read as {@code tid} (or {@code tenant}). Org/workspace/ + * project are best-effort from {@code oid}/{@code wid}/{@code pid} (or their + * {@code *Id} spellings). A legacy subject of the form + * {@code tenant:{tenant}:{org}:{workspace}:{project}} is also honored. + * + * Authority mapping mirrors the default {@code JwtAuthenticationConverter}: the + * {@code scope} claim becomes {@code SCOPE_*}-prefixed authorities (RBAC's + * {@code PolicyResolver} still grants ADMIN via the username or the ADMIN + * role). + */ +@Component +public class TenantJwtAuthenticationConverter implements Converter { + + private final JwtGrantedAuthoritiesConverter authoritiesConverter = new JwtGrantedAuthoritiesConverter(); + + @Override + public AbstractAuthenticationToken convert(Jwt jwt) { + Collection authorities = authoritiesConverter.convert(jwt); + var token = new JwtAuthenticationToken(jwt, authorities, jwt.getSubject()); + token.setDetails(extractTenant(jwt)); + return token; + } + + /** Tenant scope carried on the token details; read by {@code TenantFilter}. */ + public record PrincipalTenant(TenantId tenantId, String organizationId, + String workspaceId, String projectId) { + } + + static PrincipalTenant extractTenant(Jwt jwt) { + var tenantClaim = jwt.hasClaim("tid") + ? jwt.getClaimAsString("tid") + : jwt.hasClaim("tenant") ? jwt.getClaimAsString("tenant") : null; + var subject = jwt.getSubject(); + + if (tenantClaim == null && subject != null && subject.startsWith("tenant:")) { + var parts = subject.substring("tenant:".length()).split(":", -1); + if (parts.length >= 4) { + return new PrincipalTenant( + TenantId.from(parts[0]), + blankToNull(parts[1]), + blankToNull(parts[2]), + blankToNull(parts[3])); + } + } + if (tenantClaim != null) { + return new PrincipalTenant( + TenantId.from(tenantClaim), + claimString(jwt, "oid", "organizationId"), + claimString(jwt, "wid", "workspaceId"), + claimString(jwt, "pid", "projectId")); + } + return new PrincipalTenant(TenantId.DEFAULT, null, null, null); + } + + private static String claimString(Jwt jwt, String primary, String secondary) { + var v = jwt.hasClaim(primary) ? jwt.getClaimAsString(primary) : null; + return v != null ? v : (jwt.hasClaim(secondary) ? jwt.getClaimAsString(secondary) : null); + } + + private static String blankToNull(String s) { + return s == null || s.isBlank() ? null : s; + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/config/WebSecurityConfig.java b/syncflow-api/src/main/java/com/syncflow/api/config/WebSecurityConfig.java index 19bd5cd..aa21cba 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/config/WebSecurityConfig.java +++ b/syncflow-api/src/main/java/com/syncflow/api/config/WebSecurityConfig.java @@ -1,15 +1,19 @@ package com.syncflow.api.config; +import com.syncflow.api.security.AgentTokenFilter; import com.syncflow.security.SecurityConfig; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.core.convert.converter.Converter; +import org.springframework.security.authentication.AbstractAuthenticationToken; +import org.springframework.security.oauth2.jwt.Jwt; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; import org.springframework.security.config.http.SessionCreationPolicy; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter; import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.csrf.CookieCsrfTokenRepository; @Configuration @@ -20,8 +24,10 @@ public class WebSecurityConfig { @Bean @ConditionalOnMissingBean(SecurityFilterChain.class) public SecurityFilterChain filterChain(HttpSecurity http, - JwtAuthenticationConverter jwtAuthenticationConverter) throws Exception { + Converter jwtAuthenticationConverter, + AgentTokenFilter agentTokenFilter) throws Exception { http + .addFilterBefore(agentTokenFilter, UsernamePasswordAuthenticationFilter.class) .csrf(csrf -> csrf // Hybrid CSRF: protect cookie-based paths; /api/** uses bearer // tokens in headers (browsers cannot forge them), so it stays diff --git a/syncflow-api/src/main/java/com/syncflow/api/connection/encryption/EncryptionService.java b/syncflow-api/src/main/java/com/syncflow/api/connection/encryption/EncryptionService.java index d12e732..d2ae140 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/connection/encryption/EncryptionService.java +++ b/syncflow-api/src/main/java/com/syncflow/api/connection/encryption/EncryptionService.java @@ -21,7 +21,20 @@ public class EncryptionService { private final SecureRandom secureRandom; public EncryptionService(@Value("${syncflow.encryption.key}") String base64Key) { - var decoded = Base64.getDecoder().decode(base64Key); + if (base64Key == null || base64Key.isBlank()) { + throw new IllegalStateException("syncflow.encryption.key is not configured. " + + "Set a base64-encoded AES key (16/24/32 bytes) via env SYNCFLOW_ENCRYPTION_KEY."); + } + final byte[] decoded; + try { + decoded = Base64.getDecoder().decode(base64Key); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("syncflow.encryption.key is not valid base64", e); + } + if (decoded.length != 16 && decoded.length != 24 && decoded.length != 32) { + throw new IllegalStateException("syncflow.encryption.key decodes to " + decoded.length + + " bytes; AES requires 16, 24, or 32 bytes."); + } this.key = new SecretKeySpec(decoded, "AES"); this.secureRandom = new SecureRandom(); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/connection/mapper/ConnectionMapper.java b/syncflow-api/src/main/java/com/syncflow/api/connection/mapper/ConnectionMapper.java index 8a29434..4f95680 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/connection/mapper/ConnectionMapper.java +++ b/syncflow-api/src/main/java/com/syncflow/api/connection/mapper/ConnectionMapper.java @@ -9,6 +9,7 @@ import com.syncflow.core.connection.ConnectionStatus; import com.syncflow.core.connection.ConnectionType; import com.syncflow.core.connection.Credentials; +import org.springframework.beans.factory.annotation.Autowired; import org.mapstruct.Mapper; import org.mapstruct.Mapping; @@ -18,6 +19,10 @@ @Mapper(componentModel = "spring") public abstract class ConnectionMapper { + // Reuse the Spring-managed Jackson mapper (JSR-310 aware); do not build a + // per-call ObjectMapper (ignores configured Jackson modules). + @Autowired + protected ObjectMapper objectMapper; @Mapping(target = "id", expression = "java(domain.getId().value())") @Mapping(target = "connectionType", expression = "java(domain.getProperties().type().name())") @Mapping(target = "host", expression = "java(domain.getProperties().host())") @@ -55,7 +60,7 @@ protected String toJson(Map map) { if (map == null || map.isEmpty()) return null; try { - return new ObjectMapper().writeValueAsString(map); + return objectMapper.writeValueAsString(map); } catch (Exception e) { return null; } @@ -65,10 +70,9 @@ private Map parseOptions(String json) { if (json == null || json.isBlank()) return Map.of(); try { - var mapper = new ObjectMapper(); - var type = mapper.getTypeFactory().constructMapType( + var type = objectMapper.getTypeFactory().constructMapType( HashMap.class, String.class, String.class); - return mapper.readValue(json, type); + return objectMapper.readValue(json, type); } catch (Exception e) { return Map.of(); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/connection/service/ConnectionService.java b/syncflow-api/src/main/java/com/syncflow/api/connection/service/ConnectionService.java index 2019666..6c668bd 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/connection/service/ConnectionService.java +++ b/syncflow-api/src/main/java/com/syncflow/api/connection/service/ConnectionService.java @@ -4,6 +4,7 @@ import com.syncflow.api.connection.entity.ConnectionEntity; import com.syncflow.api.connection.mapper.ConnectionMapper; import com.syncflow.api.connection.repository.ConnectionRepository; +import com.fasterxml.jackson.databind.ObjectMapper; import com.syncflow.common.exception.SyncFlowException; import com.syncflow.core.connection.Connection; import com.syncflow.core.connection.ConnectionProperties; @@ -21,13 +22,16 @@ public class ConnectionService { private final ConnectionRepository repository; private final ConnectionMapper mapper; private final EncryptionService encryption; + private final ObjectMapper objectMapper; public ConnectionService(ConnectionRepository repository, ConnectionMapper mapper, - EncryptionService encryption) { + EncryptionService encryption, + ObjectMapper objectMapper) { this.repository = repository; this.mapper = mapper; this.encryption = encryption; + this.objectMapper = objectMapper; } public Connection create(String name, ConnectionProperties props, Credentials credentials) { @@ -105,8 +109,7 @@ private String serializeOptions(ConnectionProperties props) { if (props.options() == null || props.options().isEmpty()) return null; try { - return new com.fasterxml.jackson.databind.ObjectMapper() - .writeValueAsString(props.options()); + return objectMapper.writeValueAsString(props.options()); } catch (Exception e) { return null; } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/CaptureController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/CaptureController.java index 5d472c9..3e9e1aa 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/CaptureController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/CaptureController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.cdc.CaptureLifecycle; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.core.cdc.CaptureStatus; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -17,13 +19,16 @@ public class CaptureController { private final CaptureLifecycle lifecycle; + private final AuthorizationService authz; - public CaptureController(CaptureLifecycle lifecycle) { + public CaptureController(CaptureLifecycle lifecycle, AuthorizationService authz) { this.lifecycle = lifecycle; + this.authz = authz; } @PostMapping("/start") public ResponseEntity> start(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); var status = lifecycle.start(id, null); return ResponseEntity.status(status == CaptureStatus.RUNNING ? HttpStatus.OK : HttpStatus.ACCEPTED) .body(Map.of("pipelineId", id, "status", status.name())); @@ -31,24 +36,28 @@ public ResponseEntity> start(@PathVariable String id) { @PostMapping("/stop") public ResponseEntity> stop(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); lifecycle.stop(id); return ResponseEntity.ok(Map.of("pipelineId", id, "status", "STOPPED")); } @PostMapping("/pause") public ResponseEntity> pause(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); lifecycle.pause(id); return ResponseEntity.ok(Map.of("pipelineId", id, "status", "PAUSED")); } @PostMapping("/resume") public ResponseEntity> resume(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); lifecycle.resume(id); return ResponseEntity.ok(Map.of("pipelineId", id, "status", "RESUMED")); } @GetMapping("/status") public ResponseEntity> status(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var status = lifecycle.status(id); var events = lifecycle.eventCount(id); return ResponseEntity.ok(Map.of( diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/ConnectionController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/ConnectionController.java index 90b4d77..0904fae 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/ConnectionController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/ConnectionController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.connection.dto.ConnectionHealthResponse; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.connection.dto.ConnectionResponse; import com.syncflow.api.connection.dto.CreateConnectionRequest; import com.syncflow.api.connection.dto.TestConnectionRequest; @@ -31,15 +33,19 @@ public class ConnectionController { private final ConnectionService connectionService; private final ConnectorFactory connectorFactory; + private final AuthorizationService authz; public ConnectionController(ConnectionService connectionService, - ConnectorFactory connectorFactory) { + ConnectorFactory connectorFactory, + AuthorizationService authz) { this.connectionService = connectionService; this.connectorFactory = connectorFactory; + this.authz = authz; } @PostMapping public ResponseEntity create(@Valid @RequestBody CreateConnectionRequest req) { + authz.require(ResourcePermission.CONNECTION_WRITE); var props = new ConnectionProperties(req.connectionType(), req.host(), req.port(), req.database(), req.options() != null ? req.options() : Map.of()); var credentials = new Credentials( @@ -52,6 +58,7 @@ public ResponseEntity create(@Valid @RequestBody CreateConne @GetMapping public ResponseEntity> list() { + authz.require(ResourcePermission.CONNECTION_READ); var list = connectionService.list().stream() .map(c -> ConnectionResponse.from(c, true)) .toList(); @@ -60,6 +67,7 @@ public ResponseEntity> list() { @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(ConnectionResponse.from(connectionService.get(id), true)); } @@ -67,6 +75,7 @@ public ResponseEntity get(@PathVariable String id) { public ResponseEntity update( @PathVariable String id, @Valid @RequestBody UpdateConnectionRequest req) { + authz.require(ResourcePermission.CONNECTION_WRITE); var existing = connectionService.get(id); var props = new ConnectionProperties(existing.getProperties().type(), req.host(), req.port(), req.database(), @@ -83,12 +92,14 @@ public ResponseEntity update( @DeleteMapping("/{id}") public ResponseEntity delete(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_DELETE); connectionService.delete(id); return ResponseEntity.noContent().build(); } @PostMapping("/test") public ResponseEntity test(@Valid @RequestBody TestConnectionRequest req) { + authz.require(ResourcePermission.CONNECTION_WRITE); var props = new ConnectionProperties(req.connectionType(), req.host(), req.port(), req.database(), req.options() != null ? req.options() : Map.of()); var credentials = new Credentials( @@ -112,6 +123,7 @@ public ResponseEntity test(@Valid @RequestBody TestConne @GetMapping("/{id}/health") public ResponseEntity health(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); var connection = connectionService.getWithDecryptedCredentials(id); var validator = connectorFactory.getValidator(connection.getProperties().type()); if (validator.isEmpty()) { diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/EnterpriseAiController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/EnterpriseAiController.java index b2b8dfd..baf2e7b 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/EnterpriseAiController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/EnterpriseAiController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.agent.ai.domain.AgentContext; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.agent.ai.domain.AgentResult; import com.syncflow.api.agent.ai.domain.Conversation; import com.syncflow.api.agent.ai.domain.ReasoningPlan; @@ -24,14 +26,18 @@ public class EnterpriseAiController { private final AgentOrchestrator orchestrator; private final KnowledgeBase knowledgeBase; + private final AuthorizationService authz; - public EnterpriseAiController(AgentOrchestrator orchestrator, KnowledgeBase knowledgeBase) { + public EnterpriseAiController(AgentOrchestrator orchestrator, KnowledgeBase knowledgeBase, + AuthorizationService authz) { this.orchestrator = orchestrator; this.knowledgeBase = knowledgeBase; + this.authz = authz; } @PostMapping("/chat") public ResponseEntity chat(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); var session = body.getOrDefault("sessionId", UUID.randomUUID().toString()); var userId = body.getOrDefault("userId", "anonymous"); var tenantId = body.getOrDefault("tenantId", "default"); @@ -42,11 +48,13 @@ public ResponseEntity chat(@RequestBody Map body) @PostMapping("/plan") public ResponseEntity createPlan(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); return ResponseEntity.ok(orchestrator.createPlan(body.get("goal"))); } @PostMapping("/analyze") public ResponseEntity> analyze(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); var plan = orchestrator.createPlan(body.get("goal")); var context = new AgentContext( body.get("workspaceId"), body.get("pipelineId"), @@ -56,11 +64,13 @@ public ResponseEntity> analyze(@RequestBody Map> document(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); return ResponseEntity.ok(knowledgeBase.search(body.get("query"))); } @PostMapping("/review") public ResponseEntity> review(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); var plan = orchestrator.createPlan("review " + body.getOrDefault("pipelineId", "")); var context = new AgentContext(null, body.get("pipelineId"), null, Map.of()); return ResponseEntity.ok(orchestrator.executePlan(plan, context)); @@ -68,6 +78,7 @@ public ResponseEntity> review(@RequestBody Map @PostMapping("/recommend") public ResponseEntity> recommend(@RequestBody Map body) { + authz.require(ResourcePermission.AI_USE); var plan = orchestrator.createPlan("optimize " + body.getOrDefault("pipelineId", "")); var context = new AgentContext(null, body.get("pipelineId"), null, Map.of()); return ResponseEntity.ok(orchestrator.executePlan(plan, context)); @@ -75,6 +86,7 @@ public ResponseEntity> recommend(@RequestBody Map> history(@RequestParam String sessionId) { + authz.require(ResourcePermission.AI_USE); return ResponseEntity.ok(orchestrator.history(sessionId)); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/MetadataController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/MetadataController.java index 1aaeb7b..166dbf6 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/MetadataController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/MetadataController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.metadata.MetadataDiscoveryService; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.core.metadata.ColumnMetadata; import com.syncflow.core.metadata.ConstraintMetadata; import com.syncflow.core.metadata.IndexMetadata; @@ -19,20 +21,24 @@ public class MetadataController { private final MetadataDiscoveryService discoveryService; + private final AuthorizationService authz; - public MetadataController(MetadataDiscoveryService discoveryService) { + public MetadataController(MetadataDiscoveryService discoveryService, AuthorizationService authz) { this.discoveryService = discoveryService; + this.authz = authz; } /** GET /api/connections/{id}/metadata — returns schemas */ @GetMapping("/metadata") public ResponseEntity> getSchemas(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverSchemas(id)); } /** GET /api/connections/{id}/metadata/schemas */ @GetMapping("/metadata/schemas") public ResponseEntity> getSchemasAlt(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverSchemas(id)); } @@ -42,6 +48,7 @@ public ResponseEntity> getSchemasAlt(@PathVaria @GetMapping("/schemas/{schema}/tables") public ResponseEntity> getTables( @PathVariable String id, @PathVariable String schema) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverTables(id, schema)); } @@ -49,6 +56,7 @@ public ResponseEntity> getTables( @GetMapping("/metadata/schemas/{schema}/tables") public ResponseEntity> getTablesAlt( @PathVariable String id, @PathVariable String schema) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverTables(id, schema)); } @@ -56,6 +64,7 @@ public ResponseEntity> getTablesAlt( public ResponseEntity> getTable( @PathVariable String id, @PathVariable String schema, @PathVariable String table) { + authz.require(ResourcePermission.CONNECTION_READ); var resp = discoveryService.discoverTables(id, schema); var filtered = resp.data().stream() .filter(t -> t.name().equals(table)) @@ -69,6 +78,7 @@ public ResponseEntity> getTable( public ResponseEntity> getColumns( @PathVariable String id, @PathVariable String schema, @PathVariable String table) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverColumns(id, schema, table)); } @@ -76,6 +86,7 @@ public ResponseEntity> getColumns( public ResponseEntity> getIndexes( @PathVariable String id, @PathVariable String schema, @PathVariable String table) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverIndexes(id, schema, table)); } @@ -84,11 +95,13 @@ public ResponseEntity> getIndexes( public ResponseEntity> getConstraints( @PathVariable String id, @PathVariable String schema, @PathVariable String table) { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(discoveryService.discoverConstraints(id, schema, table)); } @PostMapping({"/metadata/refresh", "/schemas/refresh"}) public ResponseEntity refresh(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_WRITE); discoveryService.refresh(id); return ResponseEntity.ok().build(); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/PipelineDesignerController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/PipelineDesignerController.java index 21786ca..87d3ee2 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/PipelineDesignerController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/PipelineDesignerController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.pipeline.PipelineDesignerService; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.pipeline.dto.CreatePipelineDesignRequest; import com.syncflow.api.pipeline.dto.PipelineDesignResponse; import com.syncflow.api.pipeline.dto.UpdatePipelineDesignRequest; @@ -34,14 +36,17 @@ public class PipelineDesignerController { private final PipelineDesignerService service; + private final AuthorizationService authz; - public PipelineDesignerController(PipelineDesignerService service) { + public PipelineDesignerController(PipelineDesignerService service, AuthorizationService authz) { this.service = service; + this.authz = authz; } @PostMapping public ResponseEntity create( @Valid @RequestBody CreatePipelineDesignRequest req) { + authz.require(ResourcePermission.PIPELINE_WRITE); var name = new PipelineName(req.name()); var source = new SourceReference(req.sourceConnectionId(), req.sourceSchema(), req.sourceTable()); var dest = new DestinationReference(req.destConnectionId(), req.destSchema(), req.destTable(), @@ -59,12 +64,14 @@ public ResponseEntity create( @GetMapping public ResponseEntity> list() { + authz.require(ResourcePermission.PIPELINE_READ); var list = service.list().stream().map(PipelineDesignResponse::from).toList(); return ResponseEntity.ok(list); } @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_READ); return ResponseEntity.ok(PipelineDesignResponse.from(service.get(id))); } @@ -72,6 +79,7 @@ public ResponseEntity get(@PathVariable String id) { public ResponseEntity update( @PathVariable String id, @RequestBody UpdatePipelineDesignRequest req) { + authz.require(ResourcePermission.PIPELINE_WRITE); var existing = service.get(id); var name = req.name() != null ? new PipelineName(req.name()) : existing.name(); var source = req.sourceConnectionId() != null @@ -98,34 +106,40 @@ public ResponseEntity update( @DeleteMapping("/{id}") public ResponseEntity delete(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_DELETE); service.delete(id); return ResponseEntity.noContent().build(); } @PostMapping("/{id}/validate") public ResponseEntity validate(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_READ); return ResponseEntity.ok(service.validate(id)); } @PostMapping("/{id}/rollback") public ResponseEntity rollback( @PathVariable String id, @RequestParam int version) { + authz.require(ResourcePermission.PIPELINE_WRITE); return ResponseEntity.ok(PipelineDesignResponse.from(service.rollback(id, version))); } @GetMapping("/{id}/versions") public ResponseEntity> versions(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_READ); var list = service.versions(id).stream().map(PipelineDesignResponse::from).toList(); return ResponseEntity.ok(list); } @GetMapping("/{id}/preview") public ResponseEntity preview(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(service.preview(id)); } @GetMapping("/{id}/conflicts") public ResponseEntity conflicts(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_READ); return ResponseEntity.ok(service.detectConflicts(id)); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/PluginController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/PluginController.java index 32a0bee..b1d4a27 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/PluginController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/PluginController.java @@ -1,6 +1,8 @@ package com.syncflow.api.controller; import com.syncflow.api.plugin.PluginManager; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.plugin.PluginManager.PluginEntry; import com.syncflow.api.plugin.PluginManager.PluginInstallResult; import org.springframework.http.ResponseEntity; @@ -22,18 +24,22 @@ public class PluginController { private final PluginManager pluginManager; + private final AuthorizationService authz; - public PluginController(PluginManager pluginManager) { + public PluginController(PluginManager pluginManager, AuthorizationService authz) { this.pluginManager = pluginManager; + this.authz = authz; } @GetMapping public ResponseEntity> list() { + authz.require(ResourcePermission.CONNECTION_READ); return ResponseEntity.ok(pluginManager.list()); } @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return pluginManager.get(id) .map(ResponseEntity::ok) .orElse(ResponseEntity.notFound().build()); @@ -41,6 +47,7 @@ public ResponseEntity get(@PathVariable String id) { @PostMapping("/install") public ResponseEntity install(@RequestParam("file") MultipartFile file) { + authz.require(ResourcePermission.PIPELINE_WRITE); try { var temp = File.createTempFile("plugin-", ".jar"); file.transferTo(temp); @@ -55,24 +62,28 @@ public ResponseEntity install(@RequestParam("file") Multipa @PostMapping("/{id}/enable") public ResponseEntity> enable(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_WRITE); var ok = pluginManager.enable(id); return ResponseEntity.ok(Map.of("pluginId", id, "enabled", ok)); } @PostMapping("/{id}/disable") public ResponseEntity> disable(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_WRITE); var ok = pluginManager.disable(id); return ResponseEntity.ok(Map.of("pluginId", id, "disabled", ok)); } @DeleteMapping("/{id}") public ResponseEntity> uninstall(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_DELETE); var ok = pluginManager.uninstall(id); return ResponseEntity.ok(Map.of("pluginId", id, "uninstalled", ok)); } @GetMapping("/{id}/capabilities") public ResponseEntity> capabilities(@PathVariable String id) { + authz.require(ResourcePermission.CONNECTION_READ); return pluginManager.get(id) .map(entry -> { var caps = entry.connector().capabilities(); diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/SnapshotController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/SnapshotController.java index 192e5a9..66f2c84 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/SnapshotController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/SnapshotController.java @@ -1,5 +1,7 @@ package com.syncflow.api.controller; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.snapshot.SnapshotExecutor; import com.syncflow.api.sse.StatusBroadcaster; import com.syncflow.core.snapshot.SnapshotJob; @@ -22,30 +24,37 @@ public class SnapshotController { private final SnapshotExecutor executor; private final StatusBroadcaster broadcaster; + private final AuthorizationService authz; - public SnapshotController(SnapshotExecutor executor, StatusBroadcaster broadcaster) { + public SnapshotController(SnapshotExecutor executor, StatusBroadcaster broadcaster, + AuthorizationService authz) { this.executor = executor; this.broadcaster = broadcaster; + this.authz = authz; } @PostMapping("/pipelines/{id}/snapshot") public ResponseEntity start(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); var job = executor.start(id); return ResponseEntity.status(HttpStatus.ACCEPTED).body(job); } @GetMapping("/snapshots") public ResponseEntity> list() { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(executor.list()); } @GetMapping("/snapshots/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(executor.get(id)); } @GetMapping("/snapshots/{id}/progress") public ResponseEntity progress(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var job = executor.get(id); return ResponseEntity.ok(job.getProgress()); } @@ -53,12 +62,14 @@ public ResponseEntity progress(@PathVariable String id) { /** Live progress/status stream for a snapshot ("snapshot-status" events). */ @GetMapping(value = "/snapshots/{id}/events", produces = "text/event-stream") public SseEmitter snapshotEvents(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var tenant = TenantContextHolder.getTenantId().value(); return broadcaster.subscribe(tenant + ":" + id); } @PostMapping("/snapshots/{id}/cancel") public ResponseEntity cancel(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(executor.cancel(id)); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/SyncController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/SyncController.java index e811c83..bf515bf 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/SyncController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/SyncController.java @@ -1,5 +1,7 @@ package com.syncflow.api.controller; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.sse.StatusBroadcaster; import com.syncflow.api.sync.DeadLetterQueue; import com.syncflow.api.sync.SyncOrchestrator; @@ -27,12 +29,14 @@ public class SyncController { private final SyncOrchestrator orchestrator; private final DeadLetterQueue dlq; private final StatusBroadcaster broadcaster; + private final AuthorizationService authz; public SyncController(SyncOrchestrator orchestrator, DeadLetterQueue dlq, - StatusBroadcaster broadcaster) { + StatusBroadcaster broadcaster, AuthorizationService authz) { this.orchestrator = orchestrator; this.dlq = dlq; this.broadcaster = broadcaster; + this.authz = authz; } /** @@ -42,23 +46,27 @@ public SyncController(SyncOrchestrator orchestrator, DeadLetterQueue dlq, */ @GetMapping(value = "/sync/jobs/{id}/events", produces = "text/event-stream") public SseEmitter syncEvents(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var tenant = TenantContextHolder.getTenantId().value(); return broadcaster.subscribe(tenant + ":" + id); } @PostMapping("/pipelines/{id}/sync/start") public ResponseEntity start(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(orchestrator.start(id)); } @PostMapping("/pipelines/{id}/sync/stop") public ResponseEntity> stop(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); orchestrator.stop(id); return ResponseEntity.ok(Map.of("pipelineId", id, "status", "STOPPED")); } @GetMapping("/pipelines/{id}/sync/status") public ResponseEntity> status(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var state = orchestrator.status(id); var stats = orchestrator.statistics(id); return ResponseEntity.ok(Map.of( @@ -72,33 +80,39 @@ public ResponseEntity> status(@PathVariable String id) { @GetMapping("/sync/jobs") public ResponseEntity> jobs() { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(orchestrator.list()); } @GetMapping("/sync/jobs/{id}") public ResponseEntity job(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(orchestrator.get(id)); } @GetMapping("/sync/jobs/{id}/statistics") public ResponseEntity statistics(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(orchestrator.statistics(id)); } @GetMapping("/dlq") public ResponseEntity> dlqList( @RequestParam(required = false) String pipelineId) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(dlq.list(pipelineId)); } @PostMapping("/dlq/{id}/replay") public ResponseEntity replay(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); dlq.replay(id); return ResponseEntity.ok().build(); } @DeleteMapping("/dlq/{id}") public ResponseEntity deleteDlq(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_DELETE); dlq.delete(id); return ResponseEntity.noContent().build(); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/controller/WorkflowController.java b/syncflow-api/src/main/java/com/syncflow/api/controller/WorkflowController.java index 085842d..063fd0d 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/controller/WorkflowController.java +++ b/syncflow-api/src/main/java/com/syncflow/api/controller/WorkflowController.java @@ -1,5 +1,7 @@ package com.syncflow.api.controller; +import com.syncflow.api.security.rbac.AuthorizationService; +import com.syncflow.api.security.rbac.ResourcePermission; import com.syncflow.api.workflow.WorkflowScheduler; import com.syncflow.core.workflow.WorkflowId; import com.syncflow.core.workflow.WorkflowInstance; @@ -21,49 +23,59 @@ public class WorkflowController { private final WorkflowScheduler scheduler; + private final AuthorizationService authz; - public WorkflowController(WorkflowScheduler scheduler) { + public WorkflowController(WorkflowScheduler scheduler, AuthorizationService authz) { this.scheduler = scheduler; + this.authz = authz; } @PostMapping public ResponseEntity create(@RequestBody Map body) { + authz.require(ResourcePermission.PIPELINE_WRITE); var wf = scheduler.create(body.get("pipelineId")); return ResponseEntity.status(HttpStatus.CREATED).body(wf); } @GetMapping public ResponseEntity> list() { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(scheduler.list()); } @GetMapping("/{id}") public ResponseEntity get(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); return ResponseEntity.ok(scheduler.get(WorkflowId.from(id))); } @PostMapping("/{id}/start") public ResponseEntity start(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(scheduler.start(WorkflowId.from(id))); } @PostMapping("/{id}/cancel") public ResponseEntity cancel(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_CANCEL); return ResponseEntity.ok(scheduler.cancel(WorkflowId.from(id))); } @PostMapping("/{id}/pause") public ResponseEntity> pause(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(Map.of("id", id, "status", "PAUSED")); } @PostMapping("/{id}/resume") public ResponseEntity> resume(@PathVariable String id) { + authz.require(ResourcePermission.PIPELINE_EXECUTE); return ResponseEntity.ok(Map.of("id", id, "status", "RUNNING")); } @GetMapping("/{id}/graph") public ResponseEntity> graph(@PathVariable String id) { + authz.require(ResourcePermission.EXECUTION_READ); var wf = scheduler.get(WorkflowId.from(id)); return ResponseEntity.ok(wf.tasks()); } diff --git a/syncflow-api/src/main/java/com/syncflow/api/ops/alert/AlertEngine.java b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/AlertEngine.java index 42c4a5d..de56c66 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/ops/alert/AlertEngine.java +++ b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/AlertEngine.java @@ -1,57 +1,112 @@ package com.syncflow.api.ops.alert; +import com.syncflow.api.ops.alert.entity.AlertEventEntity; +import com.syncflow.api.ops.alert.repository.AlertEventRepository; +import com.syncflow.tenant.TenantSupport; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; -import java.util.Comparator; +import java.time.Instant; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; @Component public class AlertEngine { - private final Map alerts = new ConcurrentHashMap<>(); + private final AlertEventRepository repository; private final AtomicLong counter = new AtomicLong(0); + @Autowired + public AlertEngine(AlertEventRepository repository) { + this.repository = repository; + } + + /** Unit-test seam: in-memory engine without a repository. */ + public AlertEngine() { + this.repository = null; + } + + @Transactional public AlertEvent raise(String name, String message, AlertSeverity severity, String source) { return raise(name, message, severity, source, null, null); } + @Transactional public AlertEvent raise(String name, String message, AlertSeverity severity, String source, String pipelineId, String connectionId) { var id = "alert-" + counter.incrementAndGet(); var event = new AlertEvent(id, name, message, severity, source, - pipelineId, connectionId, java.time.Instant.now(), false); - alerts.put(id, event); + pipelineId, connectionId, Instant.now(), false); + if (repository != null) + repository.save(toEntity(event)); return event; } + @Transactional public void acknowledge(String id) { - alerts.computeIfPresent(id, (k, v) -> new AlertEvent(v.id(), v.name(), v.message(), v.severity(), - v.source(), v.pipelineId(), v.connectionId(), - v.timestamp(), true)); + if (repository == null) + return; + repository.findById(id).ifPresent(e -> { + e.setAcknowledged(true); + repository.save(e); + }); } + @Transactional(readOnly = true) public List active() { - return alerts.values().stream() - .filter(a -> !a.acknowledged()) - .sorted(Comparator.comparing(AlertEvent::timestamp).reversed()) + if (repository == null) + return List.of(); + return repository + .findByTenantIdAndAcknowledgedOrderByEventTimeDesc(TenantSupport.tenantId(), false) + .stream() + .map(this::toDomain) .toList(); } + @Transactional(readOnly = true) public List all() { - return alerts.values().stream() - .sorted(Comparator.comparing(AlertEvent::timestamp).reversed()) - .limit(500) + if (repository == null) + return List.of(); + return repository.findTop500ByTenantIdOrderByEventTimeDesc(TenantSupport.tenantId()) + .stream() + .map(this::toDomain) .toList(); } + @Transactional(readOnly = true) public long count() { - return alerts.size(); + if (repository == null) + return 0; + return repository.count(); } + @Transactional public void clearAcknowledged() { - alerts.values().removeIf(AlertEvent::acknowledged); + if (repository == null) + return; + repository.deleteByTenantIdAndAcknowledged(TenantSupport.tenantId(), true); + } + + private AlertEventEntity toEntity(AlertEvent v) { + var e = new AlertEventEntity(); + e.setId(v.id()); + e.setTenantId(TenantSupport.tenantId()); + e.setName(v.name()); + e.setMessage(v.message()); + e.setSeverity(v.severity().name()); + e.setSource(v.source()); + e.setPipelineId(v.pipelineId()); + e.setConnectionId(v.connectionId()); + e.setEventTime(v.timestamp()); + e.setAcknowledged(v.acknowledged()); + e.setCreatedAt(v.timestamp()); + return e; + } + + private AlertEvent toDomain(AlertEventEntity e) { + return new AlertEvent(e.getId(), e.getName(), e.getMessage(), + AlertSeverity.valueOf(e.getSeverity()), e.getSource(), + e.getPipelineId(), e.getConnectionId(), e.getEventTime(), e.isAcknowledged()); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/ops/alert/entity/AlertEventEntity.java b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/entity/AlertEventEntity.java new file mode 100644 index 0000000..b5e3474 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/entity/AlertEventEntity.java @@ -0,0 +1,55 @@ +package com.syncflow.api.ops.alert.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; + +/** Ops alert; live incidents surfaced on the dashboard. */ +@Setter +@Getter +@Entity +@Table(name = "alert_events") +public class AlertEventEntity { + + @Id + @Column(length = 50) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(nullable = false, length = 255) + private String name; + + @Column(columnDefinition = "TEXT") + private String message; + + @Column(nullable = false, length = 20) + private String severity; + + @Column(length = 255) + private String source; + + @Column(name = "pipeline_id", length = 36) + private String pipelineId; + + @Column(name = "connection_id", length = 36) + private String connectionId; + + @Column(name = "event_time", nullable = false) + private Instant eventTime; + + @Column(nullable = false) + private boolean acknowledged; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + public AlertEventEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/ops/alert/repository/AlertEventRepository.java b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/repository/AlertEventRepository.java new file mode 100644 index 0000000..8a13997 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/ops/alert/repository/AlertEventRepository.java @@ -0,0 +1,15 @@ +package com.syncflow.api.ops.alert.repository; + +import com.syncflow.api.ops.alert.entity.AlertEventEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface AlertEventRepository extends JpaRepository { + + List findByTenantIdAndAcknowledgedOrderByEventTimeDesc(String tenantId, boolean acknowledged); + + List findTop500ByTenantIdOrderByEventTimeDesc(String tenantId); + + void deleteByTenantIdAndAcknowledged(String tenantId, boolean acknowledged); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/runtimestate/RuntimeStateJson.java b/syncflow-api/src/main/java/com/syncflow/api/runtimestate/RuntimeStateJson.java new file mode 100644 index 0000000..387ea7a --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/runtimestate/RuntimeStateJson.java @@ -0,0 +1,41 @@ +package com.syncflow.api.runtimestate; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Component; + +/** JSON serialization for JSONB payloads in the runtime state tables. */ +@Component +public class RuntimeStateJson { + + private final ObjectMapper objectMapper; + + public RuntimeStateJson(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public String toJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception e) { + throw new IllegalStateException("Failed to serialize runtime state to JSON", e); + } + } + + public T fromJson(String json, Class type) { + try { + return objectMapper.readValue(json, type); + } catch (Exception e) { + throw new IllegalStateException("Failed to deserialize runtime state from JSON: " + type.getSimpleName(), + e); + } + } + + public T fromJson(String json, TypeReference typeRef) { + try { + return objectMapper.readValue(json, typeRef); + } catch (Exception e) { + throw new IllegalStateException("Failed to deserialize runtime state from JSON", e); + } + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/AgentTokenFilter.java b/syncflow-api/src/main/java/com/syncflow/api/security/AgentTokenFilter.java new file mode 100644 index 0000000..dfc2c1a --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/AgentTokenFilter.java @@ -0,0 +1,70 @@ +package com.syncflow.api.security; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.security.web.util.matcher.AntPathRequestMatcher; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import java.io.IOException; +import java.security.MessageDigest; + +/** + * Shared-secret guard for the agent control-plane endpoints. + * + * The fleet agent cannot do OAuth flows, and the platform has no mTLS yet, so + * agent-to-plane calls are PUBLIC in the security chain but fail-closed here: + * when {@code syncflow.agent.token} is configured (recommended), any + * agent-INBOUND request (register/heartbeat) must carry a matching + * {@code X-Agent-Token}; missing/wrong token -> 403. When the token is unset + * (dev default), agent endpoints are open — a network-level (mTLS / network + * policy) hardening is the documented upgrade path (see ADR-008/009). + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 10) +public class AgentTokenFilter extends OncePerRequestFilter { + + static final String AGENT_TOKEN_HEADER = "X-Agent-Token"; + + private final String configuredToken; + private final AntPathRequestMatcher agentMatcher = new AntPathRequestMatcher("/api/agents/**"); + private static final String INBOUND_REGISTER = "/api/agents/register"; + private static final String INBOUND_HEARTBEAT = "/api/agents/heartbeat"; + + public AgentTokenFilter(@Value("${syncflow.agent.token:}") String configuredToken) { + this.configuredToken = configuredToken; + } + + @Override + protected boolean shouldNotFilter(HttpServletRequest request) { + // Only agent-inbound paths are public; the rest pass through unchanged. + var path = request.getRequestURI(); + return !agentMatcher.matches(request) + || (!INBOUND_REGISTER.equals(path) && !INBOUND_HEARTBEAT.equals(path)); + } + + @Override + protected void doFilterInternal(HttpServletRequest request, + HttpServletResponse response, + FilterChain chain) throws ServletException, IOException { + if (configuredToken == null || configuredToken.isBlank()) { + // ponytail: token unset = dev default, open. Require it / mTLS in + // non-dev environments. + chain.doFilter(request, response); + return; + } + var presented = request.getHeader(AGENT_TOKEN_HEADER); + if (presented != null && MessageDigest.isEqual( + configuredToken.getBytes(java.nio.charset.StandardCharsets.UTF_8), + presented.getBytes(java.nio.charset.StandardCharsets.UTF_8))) { + chain.doFilter(request, response); + return; + } + response.sendError(HttpServletResponse.SC_FORBIDDEN, "invalid or missing X-Agent-Token"); + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/AuthService.java b/syncflow-api/src/main/java/com/syncflow/api/security/AuthService.java index a5a940b..e69241b 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/AuthService.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/AuthService.java @@ -2,6 +2,7 @@ import com.syncflow.api.config.JwtProperties; import com.syncflow.api.user.repository.UserRepository; +import com.syncflow.tenant.TenantId; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.GrantedAuthority; @@ -64,11 +65,15 @@ public LoginResult login(String username, String password) { private String issueToken(String username, java.util.List roles) { var now = Instant.now(); + // Single-tenant platform: tokens carry the default tenant. TenantFilter + // derives scope from these claims (tid/oid/wid/pid), never from client + // headers. Per-tenant account claims are the documented multi-tenant path. var claims = JwtClaimsSet.builder() .issuer(jwtProperties.getIssuer()) .issuedAt(now) .expiresAt(now.plusSeconds(jwtProperties.getExpiryMinutes() * 60)) .subject(username) + .claim("tid", TenantId.DEFAULT.value()) .claim("scope", String.join(",", roles)) .build(); // Pin the JWS algorithm to HS256 so Nimbus selects the matching HS256 key. diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java b/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java index aa184e0..c8a8d5f 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/TenantFilter.java @@ -1,5 +1,6 @@ package com.syncflow.api.security; +import com.syncflow.api.config.TenantJwtAuthenticationConverter.PrincipalTenant; import com.syncflow.tenant.OrganizationId; import com.syncflow.tenant.ProjectId; import com.syncflow.tenant.TenantContext; @@ -10,6 +11,7 @@ import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; import org.springframework.web.filter.OncePerRequestFilter; @@ -19,6 +21,21 @@ import java.util.HashSet; import java.util.Set; +/** + * Resolves the tenant context for the current request. + * + * SECURITY: the tenant is derived from the AUTHENTICATED PRINCIPAL (the JWT's + * tenant claims, attached by + * {@link com.syncflow.api.config.TenantJwtAuthenticationConverter}) + * — never trusted from client headers. An authenticated caller cannot switch + * tenant by spoofing {@code X-Tenant-Id}. + * + * UI compatibility: the SPA sends {@code X-Tenant-Id} on authenticated + * requests, so a header that MATCHES the principal's tenant is accepted + * (harmless — same tenant). A mismatched header is ignored in favour of the + * principal. Unauthenticated/anonymous requests (public endpoints) fall back + * to {@link TenantId#DEFAULT}. + */ @Component public class TenantFilter extends OncePerRequestFilter { @@ -41,36 +58,97 @@ protected void doFilterInternal(HttpServletRequest request, } private TenantContext resolve(HttpServletRequest req) { - var tenantId = req.getHeader(TENANT_HEADER) != null - ? TenantId.from(req.getHeader(TENANT_HEADER)) - : TenantId.DEFAULT; - var orgId = req.getHeader(ORG_HEADER); - var wsId = req.getHeader(WORKSPACE_HEADER); - var projId = req.getHeader(PROJECT_HEADER); - - String userId = "anonymous"; Set roles = new HashSet<>(); - var auth = SecurityContextHolder.getContext().getAuthentication(); + if (auth != null && auth.isAuthenticated()) { - userId = auth.getName() != null ? auth.getName() : userId; - if (auth.getName() != null && auth.getName().startsWith("tenant:")) { - var parts = auth.getName().substring("tenant:".length()).split(":"); - if (parts.length >= 4) { - tenantId = TenantId.from(parts[0]); - orgId = parts[1].isEmpty() ? orgId : parts[1]; - wsId = parts[2].isEmpty() ? wsId : parts[2]; - projId = parts[3].isEmpty() ? projId : parts[3]; - } + var principal = principalTenant(auth); + if (principal == null) { + // Legacy principals (e.g. test stubs / basic auth) carry no tenant + // claims; fall back to a header only as a last resort so existing + // clients keep working. Authenticated tenants still come from the + // principal whenever claims exist. + return contextFromHeaders(req, auth); } - auth.getAuthorities().forEach(a -> roles.add(a.getAuthority())); + // Header is honored ONLY when it matches the principal's tenant (the UI + // sends it on every request); a mismatched/spoofed header is ignored. + var headerTenant = header(req, TENANT_HEADER); + var tenantId = principal.tenantId(); + if (headerTenant != null && TenantId.from(headerTenant).equals(tenantId)) { + // same tenant — keep org/ws/project best-effort from headers when the + // principal lacks them + return new TenantContext( + tenantId, + principal.organizationId() != null + ? OrganizationId.from(principal.organizationId()) + : optOrgId(req), + principal.workspaceId() != null + ? WorkspaceId.from(principal.workspaceId()) + : optWorkspaceId(req), + principal.projectId() != null + ? ProjectId.from(principal.projectId()) + : optProjectId(req), + auth.getName(), authorities(auth), Instant.now()); + } + return new TenantContext( + tenantId, + principal.organizationId() != null ? OrganizationId.from(principal.organizationId()) : null, + principal.workspaceId() != null ? WorkspaceId.from(principal.workspaceId()) : null, + principal.projectId() != null ? ProjectId.from(principal.projectId()) : null, + auth.getName(), authorities(auth), Instant.now()); + } + + return contextFromHeaders(req, auth); + } + + /** + * Principal carries a {@link PrincipalTenant} (set by + * TenantJwtAuthenticationConverter). + */ + private PrincipalTenant principalTenant(Authentication auth) { + if (auth.getDetails() instanceof PrincipalTenant pt) { + return pt; } + return null; + } + private TenantContext contextFromHeaders(HttpServletRequest req, Authentication auth) { + var headerTenant = header(req, TENANT_HEADER); + var tenantId = headerTenant != null ? TenantId.from(headerTenant) : TenantId.DEFAULT; return new TenantContext( tenantId, - orgId != null && !orgId.isEmpty() ? OrganizationId.from(orgId) : null, - wsId != null && !wsId.isEmpty() ? WorkspaceId.from(wsId) : null, - projId != null && !projId.isEmpty() ? ProjectId.from(projId) : null, - userId, roles, Instant.now()); + optOrgId(req), + optWorkspaceId(req), + optProjectId(req), + auth != null && auth.getName() != null ? auth.getName() : "anonymous", + authorities(auth), Instant.now()); + } + + private Set authorities(Authentication auth) { + Set roles = new HashSet<>(); + if (auth != null) { + auth.getAuthorities().forEach(a -> roles.add(a.getAuthority())); + } + return roles; + } + + private String header(HttpServletRequest req, String name) { + var v = req.getHeader(name); + return v == null || v.isBlank() ? null : v; + } + + private OrganizationId optOrgId(HttpServletRequest req) { + var v = header(req, ORG_HEADER); + return v != null ? OrganizationId.from(v) : null; + } + + private WorkspaceId optWorkspaceId(HttpServletRequest req) { + var v = header(req, WORKSPACE_HEADER); + return v != null ? WorkspaceId.from(v) : null; + } + + private ProjectId optProjectId(HttpServletRequest req) { + var v = header(req, PROJECT_HEADER); + return v != null ? ProjectId.from(v) : null; } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/apikey/ApiKeyStore.java b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/ApiKeyStore.java index 48817a9..152dda9 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/apikey/ApiKeyStore.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/ApiKeyStore.java @@ -1,7 +1,11 @@ package com.syncflow.api.security.apikey; +import com.syncflow.api.security.apikey.entity.ApiKeyEntity; +import com.syncflow.api.security.apikey.repository.ApiKeyRepository; import com.syncflow.tenant.TenantId; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; import java.security.MessageDigest; import java.time.Instant; @@ -12,8 +16,21 @@ @Repository public class ApiKeyStore { - private final Map store = new ConcurrentHashMap<>(); + private final ApiKeyRepository repository; + // Read-through cache for the auth hot path; source of truth is the DB. + private final Map cache = new ConcurrentHashMap<>(); + @Autowired + public ApiKeyStore(ApiKeyRepository repository) { + this.repository = repository; + } + + /** Unit-test seam: in-memory store without a repository. */ + public ApiKeyStore() { + this.repository = null; + } + + @Transactional public ApiKey issue(TenantId tenantId, String label, String scope, Instant expiresAt) { var raw = UUID.randomUUID().toString().replace("-", "") + UUID.randomUUID().toString().replace("-", ""); @@ -21,23 +38,46 @@ public ApiKey issue(TenantId tenantId, String label, String scope, Instant expir var prefix = raw.substring(0, 6); var key = new ApiKey(UUID.randomUUID(), tenantId, hash, prefix, label, scope, Instant.now(), expiresAt, null); - store.put(hash, key); + if (repository != null) + repository.save(toEntity(key)); + cache.put(hash, key); return key; } + @Transactional(readOnly = true) public ApiKey validate(String rawKey) { var hash = hash(rawKey); - var k = store.get(hash); - if (k != null && k.isActive()) - return k; - return null; + var cached = cache.get(hash); + if (cached != null) + return cached.isActive() ? cached : null; + if (repository == null) + return null; // unit-test seam + return repository.findByHashedKey(hash) + .map(this::toDomain) + .filter(ApiKey::isActive) + .map(k -> { + cache.put(hash, k); + return k; + }) + .orElse(null); } + @Transactional public boolean revoke(UUID id) { - for (var entry : store.entrySet()) { + if (repository != null) { + var entity = repository.findById(id).orElse(null); + if (entity == null) + return false; + entity.setRevokedAt(Instant.now()); + repository.save(entity); + cache.remove(entity.getHashedKey()); + return true; + } + // Unit-test seam: scan the in-memory cache. + for (var entry : cache.entrySet()) { if (entry.getValue().id().equals(id)) { var k = entry.getValue(); - store.put(entry.getKey(), + cache.put(entry.getKey(), new ApiKey(k.id(), k.tenantId(), k.hashedKey(), k.prefix(), k.label(), k.scope(), k.createdAt(), k.expiresAt(), Instant.now())); return true; @@ -58,4 +98,24 @@ private String hash(String raw) { throw new RuntimeException("Hashing failed", e); } } + + private ApiKeyEntity toEntity(ApiKey k) { + var e = new ApiKeyEntity(); + e.setId(k.id()); + e.setTenantId(k.tenantId().value()); + e.setHashedKey(k.hashedKey()); + e.setPrefix(k.prefix()); + e.setLabel(k.label()); + e.setScope(k.scope()); + e.setCreatedAt(k.createdAt()); + e.setExpiresAt(k.expiresAt()); + e.setRevokedAt(k.revokedAt()); + return e; + } + + private ApiKey toDomain(ApiKeyEntity e) { + return new ApiKey(e.getId(), TenantId.from(e.getTenantId()), e.getHashedKey(), + e.getPrefix(), e.getLabel(), e.getScope(), e.getCreatedAt(), + e.getExpiresAt(), e.getRevokedAt()); + } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/apikey/entity/ApiKeyEntity.java b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/entity/ApiKeyEntity.java new file mode 100644 index 0000000..742e386 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/entity/ApiKeyEntity.java @@ -0,0 +1,50 @@ +package com.syncflow.api.security.apikey.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; +import java.util.UUID; + +/** API key; hashed value unique, revoke/expiry drive isActive(). */ +@Setter +@Getter +@Entity +@Table(name = "api_keys") +public class ApiKeyEntity { + + @Id + @Column(length = 36) + private UUID id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "hashed_key", nullable = false, length = 64) + private String hashedKey; + + @Column(length = 16) + private String prefix; + + @Column(length = 255) + private String label; + + @Column(length = 50) + private String scope; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "expires_at") + private Instant expiresAt; + + @Column(name = "revoked_at") + private Instant revokedAt; + + public ApiKeyEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/apikey/repository/ApiKeyRepository.java b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/repository/ApiKeyRepository.java new file mode 100644 index 0000000..4b3a157 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/apikey/repository/ApiKeyRepository.java @@ -0,0 +1,12 @@ +package com.syncflow.api.security.apikey.repository; + +import com.syncflow.api.security.apikey.entity.ApiKeyEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; +import java.util.UUID; + +public interface ApiKeyRepository extends JpaRepository { + + Optional findByHashedKey(String hashedKey); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/audit/EnterpriseAuditStore.java b/syncflow-api/src/main/java/com/syncflow/api/security/audit/EnterpriseAuditStore.java index 20582f8..d156daf 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/audit/EnterpriseAuditStore.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/audit/EnterpriseAuditStore.java @@ -1,45 +1,78 @@ package com.syncflow.api.security.audit; +import com.syncflow.api.security.audit.entity.AuditRecordEntity; +import com.syncflow.api.security.audit.repository.AuditRecordRepository; import com.syncflow.tenant.TenantId; +import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Instant; -import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; +/** Enterprise audit records persisted to PostgreSQL. */ @Component public class EnterpriseAuditStore { - private final Map store = new ConcurrentHashMap<>(); + private final AuditRecordRepository repository; + public EnterpriseAuditStore(AuditRecordRepository repository) { + this.repository = repository; + } + + @Transactional public EnterpriseAuditRecord record(TenantId tenantId, String actor, String action, String resourceType, String resourceId, String details, String ipAddress) { var id = UUID.randomUUID(); var record = new EnterpriseAuditRecord(id, tenantId, actor, action, resourceType, resourceId, details, ipAddress, false, Instant.now()); - store.put(id, record); + var entity = toEntity(record); + repository.save(entity); return record; } + @Transactional(readOnly = true) public List list(TenantId tenantId, int limit) { - return store.values().stream() - .filter(r -> r.tenantId().equals(tenantId)) - .sorted(Comparator.comparing(EnterpriseAuditRecord::timestamp).reversed()) - .limit(Math.max(1, limit)) + return repository + .findByTenantIdOrderByEventTimeDesc(tenantId.value(), PageRequest.of(0, Math.max(1, limit))) + .stream() + .map(this::toDomain) .toList(); } public boolean hardDelete() { - return store.isEmpty(); + return repository.count() == 0; } /** Compliance: GDPR right-to-delete — remove all records for a tenant. */ + @Transactional public void anonymize(UserDeletionRequest req) { - store.values().removeIf(r -> r.tenantId().equals(req.tenantId())); + repository.deleteAll(repository.findByTenantIdOrderByEventTimeDesc( + req.tenantId().value(), PageRequest.of(0, 10_000))); + } + + private AuditRecordEntity toEntity(EnterpriseAuditRecord r) { + var e = new AuditRecordEntity(); + e.setId(r.id()); + e.setTenantId(r.tenantId().value()); + e.setActor(r.actor()); + e.setAction(r.action()); + e.setResourceType(r.resourceType()); + e.setResourceId(r.resourceId()); + e.setDetails(r.details()); + e.setIpAddress(r.ipAddress()); + e.setSuspicious(r.suspicious()); + e.setEventTime(r.timestamp()); + e.setCreatedAt(r.timestamp()); + return e; + } + + private EnterpriseAuditRecord toDomain(AuditRecordEntity e) { + return new EnterpriseAuditRecord(e.getId(), TenantId.from(e.getTenantId()), + e.getActor(), e.getAction(), e.getResourceType(), e.getResourceId(), + e.getDetails(), e.getIpAddress(), e.isSuspicious(), e.getEventTime()); } public record UserDeletionRequest(TenantId tenantId) { diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/audit/entity/AuditRecordEntity.java b/syncflow-api/src/main/java/com/syncflow/api/security/audit/entity/AuditRecordEntity.java new file mode 100644 index 0000000..734293e --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/audit/entity/AuditRecordEntity.java @@ -0,0 +1,56 @@ +package com.syncflow.api.security.audit.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; +import java.util.UUID; + +/** Enterprise audit record; supports GDPR right-to-delete per tenant. */ +@Setter +@Getter +@Entity +@Table(name = "audit_records") +public class AuditRecordEntity { + + @Id + @Column(length = 36) + private UUID id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(length = 255) + private String actor; + + @Column(nullable = false, length = 100) + private String action; + + @Column(name = "resource_type", length = 100) + private String resourceType; + + @Column(name = "resource_id", length = 255) + private String resourceId; + + @Column(columnDefinition = "TEXT") + private String details; + + @Column(name = "ip_address", length = 45) + private String ipAddress; + + @Column(nullable = false) + private boolean suspicious; + + @Column(name = "event_time", nullable = false) + private Instant eventTime; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + public AuditRecordEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/audit/repository/AuditRecordRepository.java b/syncflow-api/src/main/java/com/syncflow/api/security/audit/repository/AuditRecordRepository.java new file mode 100644 index 0000000..f70c61f --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/audit/repository/AuditRecordRepository.java @@ -0,0 +1,13 @@ +package com.syncflow.api.security.audit.repository; + +import com.syncflow.api.security.audit.entity.AuditRecordEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.UUID; + +public interface AuditRecordRepository extends JpaRepository { + + List findByTenantIdOrderByEventTimeDesc(String tenantId, Pageable pageable); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/quota/QuotaEngine.java b/syncflow-api/src/main/java/com/syncflow/api/security/quota/QuotaEngine.java index f397c4e..c3b916f 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/security/quota/QuotaEngine.java +++ b/syncflow-api/src/main/java/com/syncflow/api/security/quota/QuotaEngine.java @@ -1,23 +1,57 @@ package com.syncflow.api.security.quota; +import com.fasterxml.jackson.core.type.TypeReference; +import com.syncflow.api.runtimestate.RuntimeStateJson; +import com.syncflow.api.security.quota.entity.QuotaEntity; +import com.syncflow.api.security.quota.repository.QuotaRepository; import com.syncflow.tenant.TenantId; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; +import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; @Component public class QuotaEngine { - private final java.util.Map quotas = new ConcurrentHashMap<>(); + private static final TypeReference> LIMITS_TYPE = new TypeReference<>() { + }; + private final QuotaRepository repository; + private final RuntimeStateJson json; + // Fast-path read cache; durable source of truth is the quotas table. + private final java.util.Map cache = new ConcurrentHashMap<>(); + + @org.springframework.beans.factory.annotation.Autowired + public QuotaEngine(QuotaRepository repository, RuntimeStateJson json) { + this.repository = repository; + this.json = json; + } + + /** Unit-test seam: in-memory engine without a repository. */ + public QuotaEngine() { + this.repository = null; + this.json = null; + } + + @Transactional(readOnly = true) public Quota getQuota(TenantId tenantId) { - return quotas.computeIfAbsent(tenantId, k -> Quota.defaults()); + return cache.computeIfAbsent(tenantId, t -> load(t).orElse(Quota.defaults())); } + @Transactional public void setQuota(TenantId tenantId, Quota quota) { - quotas.put(tenantId, quota); + cache.put(tenantId, quota); + if (repository == null) + return; // unit-test seam + var entity = repository.findById(tenantId.value()).orElseGet(QuotaEntity::new); + entity.setTenantId(tenantId.value()); + entity.setLimits(json.toJson(quota.limits())); + entity.setUpdatedAt(Instant.now()); + repository.save(entity); } + @Transactional(readOnly = true) public QuotaResult checkLimit(TenantId tenantId, Quota.Metric metric, long current) { var quota = getQuota(tenantId); var limit = quota.limit(metric); @@ -26,6 +60,13 @@ public QuotaResult checkLimit(TenantId tenantId, Quota.Metric metric, long curre return new QuotaResult(current >= limit, limit, current, metric); } + private java.util.Optional load(TenantId tenantId) { + if (repository == null) + return java.util.Optional.empty(); // unit-test seam + return repository.findById(tenantId.value()) + .map(e -> new Quota(json.fromJson(e.getLimits(), LIMITS_TYPE))); + } + public record QuotaResult(boolean exceeded, long limit, long current, Quota.Metric metric) { } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/quota/entity/QuotaEntity.java b/syncflow-api/src/main/java/com/syncflow/api/security/quota/entity/QuotaEntity.java new file mode 100644 index 0000000..67ed42d --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/quota/entity/QuotaEntity.java @@ -0,0 +1,34 @@ +package com.syncflow.api.security.quota.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** Per-tenant quota row; limits map as JSONB. */ +@Setter +@Getter +@Entity +@Table(name = "quotas") +public class QuotaEntity { + + @Id + @Column(name = "tenant_id", length = 36) + private String tenantId; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String limits; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public QuotaEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/security/quota/repository/QuotaRepository.java b/syncflow-api/src/main/java/com/syncflow/api/security/quota/repository/QuotaRepository.java new file mode 100644 index 0000000..8e17177 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/security/quota/repository/QuotaRepository.java @@ -0,0 +1,7 @@ +package com.syncflow.api.security.quota.repository; + +import com.syncflow.api.security.quota.entity.QuotaEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface QuotaRepository extends JpaRepository { +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/CheckpointStore.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/CheckpointStore.java index 36fca39..a9141ed 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/snapshot/CheckpointStore.java +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/CheckpointStore.java @@ -1,33 +1,67 @@ package com.syncflow.api.snapshot; +import com.syncflow.api.snapshot.entity.SnapshotCheckpointEntity; +import com.syncflow.api.snapshot.repository.SnapshotCheckpointRepository; import com.syncflow.core.snapshot.SnapshotCheckpoint; +import com.syncflow.tenant.TenantSupport; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - +/** + * Resume checkpoints persisted to PostgreSQL; one row per + * tenant+pipeline+table. + */ @Component public class CheckpointStore { - private final Map store = new ConcurrentHashMap<>(); + private final SnapshotCheckpointRepository repository; + + public CheckpointStore(SnapshotCheckpointRepository repository) { + this.repository = repository; + } + @Transactional public void save(SnapshotCheckpoint checkpoint) { - store.put(key(checkpoint.pipelineId(), checkpoint.sourceTable()), checkpoint); + var entity = repository + .findByTenantIdAndPipelineIdAndSourceTable( + tenantId(), checkpoint.pipelineId(), checkpoint.sourceTable()) + .orElseGet(SnapshotCheckpointEntity::new); + entity.setTenantId(tenantId()); + entity.setPipelineId(checkpoint.pipelineId()); + entity.setSourceTable(checkpoint.sourceTable()); + entity.setLastBatchNumber(checkpoint.lastBatchNumber()); + entity.setRowsProcessed(checkpoint.rowsProcessed()); + entity.setCursorPos(checkpoint.cursor()); + entity.setUpdatedAt(java.time.Instant.now()); + repository.save(entity); } + @Transactional(readOnly = true) public SnapshotCheckpoint get(String pipelineId, String sourceTable) { - return store.get(key(pipelineId, sourceTable)); + return repository + .findByTenantIdAndPipelineIdAndSourceTable(tenantId(), pipelineId, sourceTable) + .map(this::toDomain) + .orElse(null); } + @Transactional public void delete(String pipelineId, String sourceTable) { - store.remove(key(pipelineId, sourceTable)); + repository + .findByTenantIdAndPipelineIdAndSourceTable(tenantId(), pipelineId, sourceTable) + .ifPresent(repository::delete); } + @Transactional public void deleteAll(String pipelineId) { - store.keySet().removeIf(k -> k.startsWith(pipelineId + ":")); + repository.deleteAllForPipeline(tenantId(), pipelineId); + } + + private SnapshotCheckpoint toDomain(SnapshotCheckpointEntity e) { + return new SnapshotCheckpoint(e.getPipelineId(), e.getSourceTable(), + e.getLastBatchNumber(), e.getRowsProcessed(), e.getCursorPos()); } - private static String key(String pid, String table) { - return pid + ":" + table; + private String tenantId() { + return TenantSupport.tenantId(); } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/SnapshotExecutor.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/SnapshotExecutor.java index 30ac83b..f7166d0 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/snapshot/SnapshotExecutor.java +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/SnapshotExecutor.java @@ -3,6 +3,9 @@ import com.syncflow.api.connection.service.ConnectionService; import com.syncflow.api.metadata.ConnectorTypeMapper; import com.syncflow.api.pipeline.PipelineDesignerService; +import com.syncflow.api.runtimestate.RuntimeStateJson; +import com.syncflow.api.snapshot.entity.SnapshotJobEntity; +import com.syncflow.api.snapshot.repository.SnapshotJobRepository; import com.syncflow.api.sse.StatusBroadcaster; import com.syncflow.core.connection.Connection; import com.syncflow.core.model.ConnectionConfiguration; @@ -28,6 +31,7 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Instant; import java.util.List; @@ -46,9 +50,14 @@ public class SnapshotExecutor { private final ConnectorRegistry connectorRegistry; private final WriterRegistry writerRegistry; private final CheckpointStore checkpointStore; + private final SnapshotJobRepository jobRepository; + private final RuntimeStateJson json; private final MeterRegistry meterRegistry; private final StatusBroadcaster broadcaster; - private final Map jobs = new ConcurrentHashMap<>(); + + // In-memory worker state: cancellation flags + tenant ownership. The job + // payload itself is durable in snapshot_jobs; the in-memory job cache is a + // fast-path read (writes round-trip to Postgres on every state change). private final Map cancellations = new ConcurrentHashMap<>(); private final Map tenantOf = new ConcurrentHashMap<>(); @@ -57,6 +66,8 @@ public SnapshotExecutor(PipelineDesignerService pipelineService, ConnectorRegistry connectorRegistry, WriterRegistry writerRegistry, CheckpointStore checkpointStore, + SnapshotJobRepository jobRepository, + RuntimeStateJson json, MeterRegistry meterRegistry, StatusBroadcaster broadcaster) { this.pipelineService = pipelineService; @@ -64,6 +75,8 @@ public SnapshotExecutor(PipelineDesignerService pipelineService, this.connectorRegistry = connectorRegistry; this.writerRegistry = writerRegistry; this.checkpointStore = checkpointStore; + this.jobRepository = jobRepository; + this.json = json; this.meterRegistry = meterRegistry; this.broadcaster = broadcaster; } @@ -72,7 +85,7 @@ public SnapshotJob start(String pipelineId) { var pipeline = pipelineService.get(pipelineId); var job = new SnapshotJob(pipelineId).withRunning(); var snapshotId = job.getId().value(); - jobs.put(snapshotId, job); + persist(job); cancellations.put(snapshotId, new AtomicBoolean(false)); // Capture the tenant at request time; the worker's ThreadLocal won't see it. @@ -82,51 +95,40 @@ public SnapshotJob start(String pipelineId) { return job; } + @Transactional(readOnly = true) public SnapshotJob get(String snapshotId) { - assertOwned(snapshotId); - var job = jobs.get(snapshotId); - if (job == null) - throw new NoSuchElementException("Snapshot not found: " + snapshotId); - return job; - } - - /** Cross-tenant access to a snapshot by id must be rejected. */ - private void assertOwned(String snapshotId) { - var tenant = TenantContextHolder.getTenantId().value(); - var owner = tenantOf.getOrDefault(snapshotId, TenantId.DEFAULT.value()); - if (!tenant.equals(owner)) { - throw new NoSuchElementException("Snapshot not found: " + snapshotId); - } + return java.util.Optional.ofNullable(findOwned(snapshotId)) + .map(this::toDomain) + .orElseThrow(() -> new NoSuchElementException("Snapshot not found: " + snapshotId)); } /** Only the current tenant's snapshots. */ + @Transactional(readOnly = true) public List list() { var tenant = TenantContextHolder.getTenantId().value(); - return jobs.entrySet().stream() - .filter(e -> tenant.equals(tenantOf.getOrDefault( - e.getKey(), TenantId.DEFAULT.value()))) - .map(Map.Entry::getValue) + return jobRepository.findByTenantIdOrderByCreatedAtDesc(tenant).stream() + .map(this::toDomain) .toList(); } public SnapshotJob cancel(String snapshotId) { - assertOwned(snapshotId); var flag = cancellations.get(snapshotId); if (flag != null) flag.set(true); - var job = jobs.get(snapshotId); - if (job != null) { - jobs.put(snapshotId, job.withCancelled()); - // A cancelled snapshot is terminal; release its in-memory state. - remove(snapshotId); - return job.withCancelled(); - } - throw new NoSuchElementException("Snapshot not found: " + snapshotId); + var job = java.util.Optional.ofNullable(findOwned(snapshotId)) + .map(this::toDomain) + .orElseThrow(() -> new NoSuchElementException("Snapshot not found: " + snapshotId)); + var cancelled = job.withCancelled(); + persist(cancelled); + // A cancelled snapshot is terminal; release its in-memory state. + remove(snapshotId); + return cancelled; } - /** Release in-memory state for a terminal snapshot. */ + /** + * Release in-memory worker state for a terminal snapshot (job stays durable). + */ private void remove(String snapshotId) { - jobs.remove(snapshotId); cancellations.remove(snapshotId); tenantOf.remove(snapshotId); } @@ -171,7 +173,7 @@ private void executeInner(SnapshotJob job, PipelineDesign pipeline) { } var progress = SnapshotProgress.starting(totalRows); - jobs.put(job.getId().value(), job.withProgress(progress)); + persist(job.withProgress(progress)); for (var tm : pipeline.tableMappings()) { if (isCancelled(job)) @@ -212,7 +214,7 @@ private void executeInner(SnapshotJob job, PipelineDesign pipeline) { var updated = job.withProgress(new SnapshotProgress( (int) batchesDone.get(), (int) totalBatches, rowsProcessed.get(), totalRows, pct, 0)); - jobs.put(job.getId().value(), updated); + persist(updated); emit(job.getId().value(), updated); meterRegistry.counter("syncflow.snapshot.rows", @@ -251,9 +253,12 @@ private void executeInner(SnapshotJob job, PipelineDesign pipeline) { batchesDone.get(), totalBatches, 0, 0, job.getCreatedAt(), Instant.now(), elapsed / 1_000_000); var completed = job.withCompleted(stats); - jobs.put(job.getId().value(), completed); + persist(completed); emit(job.getId().value(), completed); checkpointStore.deleteAll(pipeline.id().value()); + // Terminal and durable; release worker state so the in-memory + // maps cannot grow unbounded across snapshots. + remove(job.getId().value()); } } catch (Exception e) { sample.stop(timer); @@ -266,7 +271,7 @@ private void executeInner(SnapshotJob job, PipelineDesign pipeline) { var error = new SnapshotError("SNAPSHOT_FAILED", e.getMessage(), (int) batchesDone.get(), Instant.now()); var failed = job.withFailed(List.of(error)); - jobs.put(job.getId().value(), failed); + persist(failed); emit(job.getId().value(), failed); remove(job.getId().value()); meterRegistry.counter("syncflow.snapshot.errors", @@ -286,6 +291,31 @@ private boolean isCancelled(SnapshotJob job) { return flag != null && flag.get(); } + private SnapshotJobEntity findOwned(String snapshotId) { + var tenant = TenantContextHolder.getTenantId().value(); + return jobRepository.findById(snapshotId) + .filter(e -> tenant.equals(e.getTenantId())) + .orElse(null); + } + + @Transactional + private void persist(SnapshotJob job) { + var entity = jobRepository.findById(job.getId().value()) + .orElseGet(SnapshotJobEntity::new); + entity.setId(job.getId().value()); + entity.setTenantId(TenantSupport.tenantId()); + entity.setPipelineId(job.getPipelineId()); + entity.setStatus(job.getStatus().name()); + entity.setPayload(json.toJson(job)); + entity.setCreatedAt(job.getCreatedAt()); + entity.setUpdatedAt(Instant.now()); + jobRepository.save(entity); + } + + private SnapshotJob toDomain(SnapshotJobEntity e) { + return json.fromJson(e.getPayload(), SnapshotJob.class); + } + private ConnectorContext buildSourceContext(PipelineDesign pipeline) { var conn = connectionService.getWithDecryptedCredentials(pipeline.source().connectionId()); var config = toConfig(conn); diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotCheckpointEntity.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotCheckpointEntity.java new file mode 100644 index 0000000..42f1d2b --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotCheckpointEntity.java @@ -0,0 +1,48 @@ +package com.syncflow.api.snapshot.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; + +/** Resume checkpoint for a snapshot pipeline+table; one row per tenant. */ +@Setter +@Getter +@Entity +@Table(name = "snapshot_checkpoints") +public class SnapshotCheckpointEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "pipeline_id", nullable = false, length = 36) + private String pipelineId; + + @Column(name = "source_table", nullable = false, length = 255) + private String sourceTable; + + @Column(name = "last_batch_number", nullable = false) + private int lastBatchNumber; + + @Column(name = "rows_processed", nullable = false) + private long rowsProcessed; + + @Column(name = "cursor_pos", length = 4096) + private String cursorPos; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public SnapshotCheckpointEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotJobEntity.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotJobEntity.java new file mode 100644 index 0000000..26940e3 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/entity/SnapshotJobEntity.java @@ -0,0 +1,48 @@ +package com.syncflow.api.snapshot.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** + * Snapshot job runtime state; complex payload (progress/stats/errors) as JSONB. + */ +@Setter +@Getter +@Entity +@Table(name = "snapshot_jobs") +public class SnapshotJobEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "pipeline_id", nullable = false, length = 36) + private String pipelineId; + + @Column(nullable = false, length = 20) + private String status; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String payload; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public SnapshotJobEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotCheckpointRepository.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotCheckpointRepository.java new file mode 100644 index 0000000..857ba61 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotCheckpointRepository.java @@ -0,0 +1,21 @@ +package com.syncflow.api.snapshot.repository; + +import com.syncflow.api.snapshot.entity.SnapshotCheckpointEntity; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.util.Optional; + +public interface SnapshotCheckpointRepository extends JpaRepository { + + Optional findByTenantIdAndPipelineIdAndSourceTable( + String tenantId, String pipelineId, String sourceTable); + + @Modifying + @Query(value = """ + DELETE FROM snapshot_checkpoints WHERE tenant_id = :tenantId AND pipeline_id = :pipelineId + """, nativeQuery = true) + void deleteAllForPipeline(@Param("tenantId") String tenantId, @Param("pipelineId") String pipelineId); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotJobRepository.java b/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotJobRepository.java new file mode 100644 index 0000000..f5c8957 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/snapshot/repository/SnapshotJobRepository.java @@ -0,0 +1,13 @@ +package com.syncflow.api.snapshot.repository; + +import com.syncflow.api.snapshot.entity.SnapshotJobEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface SnapshotJobRepository extends JpaRepository { + + List findByTenantIdOrderByCreatedAtDesc(String tenantId); + + List findByTenantIdAndPipelineIdOrderByCreatedAtDesc(String tenantId, String pipelineId); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java b/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java index 0fe273c..12f35ba 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/SyncOrchestrator.java @@ -2,7 +2,10 @@ import com.syncflow.api.cdc.CaptureLifecycle; import com.syncflow.api.pipeline.PipelineDesignerService; +import com.syncflow.api.runtimestate.RuntimeStateJson; import com.syncflow.api.sse.StatusBroadcaster; +import com.syncflow.api.sync.entity.SyncJobEntity; +import com.syncflow.api.sync.repository.SyncJobRepository; import com.syncflow.core.cdc.CDCEvent; import com.syncflow.core.cdc.CaptureStatus; import com.syncflow.core.pipeline.mapping.ColumnMapping; @@ -19,12 +22,15 @@ import com.syncflow.tenant.TenantSupport; import io.micrometer.core.instrument.MeterRegistry; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -44,14 +50,16 @@ public class SyncOrchestrator { private final EventIdempotencyStore idempotencyStore; private final RetryEngine retryEngine; private final DeadLetterQueue dlq; + private final SyncJobRepository jobRepository; + private final RuntimeStateJson json; private final MeterRegistry meterRegistry; private final StatusBroadcaster broadcaster; - private final Map jobs = new ConcurrentHashMap<>(); + // Transient in-memory state: event queues, worker threads, running flags. + // The SyncJob itself (state + statistics) is durable in sync_jobs. private final Map> eventQueues = new ConcurrentHashMap<>(); private final Map runningFlags = new ConcurrentHashMap<>(); private final Map workerThreads = new ConcurrentHashMap<>(); - private final AtomicLong processed = new AtomicLong(0); public SyncOrchestrator(CaptureLifecycle captureLifecycle, PipelineDesignerService pipelineService, @@ -59,6 +67,8 @@ public SyncOrchestrator(CaptureLifecycle captureLifecycle, EventIdempotencyStore idempotencyStore, RetryEngine retryEngine, DeadLetterQueue dlq, + SyncJobRepository jobRepository, + RuntimeStateJson json, MeterRegistry meterRegistry, StatusBroadcaster broadcaster) { this.captureLifecycle = captureLifecycle; @@ -67,6 +77,8 @@ public SyncOrchestrator(CaptureLifecycle captureLifecycle, this.idempotencyStore = idempotencyStore; this.retryEngine = retryEngine; this.dlq = dlq; + this.jobRepository = jobRepository; + this.json = json; this.meterRegistry = meterRegistry; this.broadcaster = broadcaster; } @@ -81,12 +93,12 @@ private static String tenantKey(String pipelineId) { return key(TenantContextHolder.getTenantId().value(), pipelineId); } + @Transactional public SyncJob start(String pipelineId) { // Capture the tenant at request time so the background worker scopes its // DB work correctly (ThreadLocal does not cross virtual-thread boundaries). var tenantId = TenantContextHolder.getTenantId(); - var key = key(tenantId.value(), pipelineId); - var existing = jobs.get(key); + var existing = findByPipeline(pipelineId); if (existing != null && existing.getState() == SyncState.RUNNING) return existing; @@ -97,7 +109,8 @@ public SyncJob start(String pipelineId) { } var job = new SyncJob(pipelineId).withRunning(); - jobs.put(key, job); + persist(job); + var key = key(tenantId.value(), pipelineId); runningFlags.put(key, new AtomicBoolean(true)); var queue = new LinkedBlockingQueue(QUEUE_CAPACITY); eventQueues.put(key, queue); @@ -114,21 +127,23 @@ public SyncJob start(String pipelineId) { return job; } + @Transactional public void stop(String pipelineId) { var key = tenantKey(pipelineId); var flag = runningFlags.get(key); if (flag != null) flag.set(false); - var job = jobs.get(key); - if (job != null) { - jobs.put(key, job.withStopped()); - emit(job); - } + Optional.ofNullable(findByPipeline(pipelineId)) + .map(SyncJob::withStopped) + .ifPresent(job -> { + persist(job); + emit(job); + }); } + @Transactional(readOnly = true) public SyncJob get(String pipelineId) { - var key = tenantKey(pipelineId); - var job = jobs.get(key); + var job = findByPipeline(pipelineId); if (job == null) throw new NoSuchElementException("No sync job for pipeline: " + pipelineId); return job; @@ -147,21 +162,22 @@ private void emit(SyncJob job) { "statistics", job.getStatistics())); } + @Transactional(readOnly = true) public SyncState status(String pipelineId) { - var job = jobs.get(tenantKey(pipelineId)); + var job = findByPipeline(pipelineId); return job != null ? job.getState() : SyncState.STOPPED; } + @Transactional(readOnly = true) public List list() { - var tenant = TenantContextHolder.getTenantId().value(); - return jobs.entrySet().stream() - .filter(e -> e.getKey().startsWith(tenant + ":")) - .map(Map.Entry::getValue) + return jobRepository.findByTenantIdOrderByCreatedAtDesc(TenantContextHolder.getTenantId().value()).stream() + .map(this::toDomain) .toList(); } + @Transactional(readOnly = true) public SyncStatistics statistics(String pipelineId) { - var job = jobs.get(tenantKey(pipelineId)); + var job = findByPipeline(pipelineId); return job != null ? job.getStatistics() : new SyncStatistics(0, 0, 0, 0, 0, 0, 0); } @@ -218,11 +234,12 @@ private void runInner(String pipelineId, BlockingQueue queue, "pipeline", pipelineId).increment(eventsThisBatch.size()); var stats = statsBuilder.build(); - var job = jobs.get(mapKey); - if (job != null) { - jobs.put(mapKey, job.withStatistics(stats)); - emit(job); - } + Optional.ofNullable(findByPipeline(pipelineId)) + .map(job -> job.withStatistics(stats)) + .ifPresent(job -> { + persist(job); + emit(job); + }); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -233,11 +250,12 @@ private void runInner(String pipelineId, BlockingQueue queue, } } - var finalJob = jobs.get(mapKey); - if (finalJob != null) { - jobs.put(mapKey, finalJob.withCompleted()); - emit(finalJob); - } + Optional.ofNullable(findByPipeline(pipelineId)) + .map(SyncJob::withCompleted) + .ifPresent(job -> { + persist(job); + emit(job); + }); } private void processEvent(String pipelineId, CDCEvent event, @@ -295,6 +313,35 @@ private void processEvent(String pipelineId, CDCEvent event, } } + private SyncJob findByPipeline(String pipelineId) { + return jobRepository.findByTenantIdAndPipelineId( + TenantContextHolder.getTenantId().value(), pipelineId) + .map(this::toDomain) + .orElse(null); + } + + @Transactional + private void persist(SyncJob job) { + var entity = jobRepository.findByTenantIdAndPipelineId( + TenantContextHolder.getTenantId().value(), job.getPipelineId()) + .orElseGet(SyncJobEntity::new); + entity.setId(job.getId()); + entity.setTenantId(TenantContextHolder.getTenantId().value()); + entity.setPipelineId(job.getPipelineId()); + entity.setState(job.getState().name()); + entity.setStatistics(json.toJson(job.getStatistics())); + entity.setCreatedAt(job.getCreatedAt()); + entity.setUpdatedAt(Instant.now()); + jobRepository.save(entity); + } + + private SyncJob toDomain(SyncJobEntity e) { + return SyncJob.restore(e.getId(), e.getPipelineId(), + SyncState.valueOf(e.getState()), + json.fromJson(e.getStatistics(), SyncStatistics.class), + e.getCreatedAt()); + } + private static class SyncStatisticsBuilder { final AtomicLong totalEvents = new AtomicLong(0); diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/entity/SyncJobEntity.java b/syncflow-api/src/main/java/com/syncflow/api/sync/entity/SyncJobEntity.java new file mode 100644 index 0000000..1861fbf --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/entity/SyncJobEntity.java @@ -0,0 +1,49 @@ +package com.syncflow.api.sync.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** + * Sync job runtime state; one row per tenant+pipeline (matches the orchestrator + * key). + */ +@Setter +@Getter +@Entity +@Table(name = "sync_jobs") +public class SyncJobEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "pipeline_id", nullable = false, length = 36) + private String pipelineId; + + @Column(nullable = false, length = 20) + private String state; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String statistics; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public SyncJobEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/sync/repository/SyncJobRepository.java b/syncflow-api/src/main/java/com/syncflow/api/sync/repository/SyncJobRepository.java new file mode 100644 index 0000000..987f349 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/sync/repository/SyncJobRepository.java @@ -0,0 +1,14 @@ +package com.syncflow.api.sync.repository; + +import com.syncflow.api.sync.entity.SyncJobEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface SyncJobRepository extends JpaRepository { + + Optional findByTenantIdAndPipelineId(String tenantId, String pipelineId); + + List findByTenantIdOrderByCreatedAtDesc(String tenantId); +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java b/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java index d0b8e66..a4ff895 100644 --- a/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java +++ b/syncflow-api/src/main/java/com/syncflow/api/workflow/WorkflowScheduler.java @@ -1,19 +1,25 @@ package com.syncflow.api.workflow; +import com.fasterxml.jackson.core.type.TypeReference; +import com.syncflow.api.runtimestate.RuntimeStateJson; +import com.syncflow.api.workflow.entity.WorkflowInstanceEntity; +import com.syncflow.api.workflow.repository.WorkflowInstanceRepository; +import com.syncflow.core.workflow.TaskExecution; import com.syncflow.core.workflow.WorkflowId; import com.syncflow.core.workflow.WorkflowInstance; import com.syncflow.core.workflow.WorkflowStatus; import com.syncflow.core.workflow.WorkflowTask; +import com.syncflow.tenant.TenantSupport; import io.micrometer.core.instrument.MeterRegistry; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.NoSuchElementException; +import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -25,58 +31,66 @@ public class WorkflowScheduler { private final TaskQueue taskQueue; private final WorkflowBuilder builder; + private final WorkflowInstanceRepository repository; + private final RuntimeStateJson json; private final MeterRegistry meterRegistry; - private final Map workflows = new ConcurrentHashMap<>(); + + // Leader election + heartbeat stay in-memory (transient); workflow instances + // are durable in workflow_instances. private final AtomicBoolean leader = new AtomicBoolean(false); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); private final AtomicReference lastHeartbeat = new AtomicReference<>(Instant.now()); public WorkflowScheduler(TaskQueue taskQueue, WorkflowBuilder builder, + WorkflowInstanceRepository repository, RuntimeStateJson json, MeterRegistry meterRegistry) { this.taskQueue = taskQueue; this.builder = builder; + this.repository = repository; + this.json = json; this.meterRegistry = meterRegistry; scheduler.scheduleAtFixedRate(this::tick, 0, 2, TimeUnit.SECONDS); scheduler.scheduleAtFixedRate(this::heartbeat, 0, 10, TimeUnit.SECONDS); } + @Transactional public WorkflowInstance create(String pipelineId) { var tasks = builder.buildPipelineWorkflow(pipelineId); var instance = WorkflowInstance.create(pipelineId, tasks); - workflows.put(instance.id(), instance); + persist(instance); return instance; } + @Transactional public WorkflowInstance start(WorkflowId id) { - var wf = workflows.get(id); - if (wf == null) - throw new NoSuchElementException("Workflow not found: " + id); + var wf = get(id); var running = wf.withStatus(WorkflowStatus.RUNNING); - workflows.put(id, running); + persist(running); var ready = findReadyTasks(running); ready.forEach(t -> taskQueue.enqueue(id.value(), t.taskId(), t.type().name(), running.pipelineId())); return running; } + @Transactional(readOnly = true) public WorkflowInstance get(WorkflowId id) { - var wf = workflows.get(id); - if (wf == null) - throw new NoSuchElementException("Workflow not found: " + id); - return wf; + return findOwned(id) + .orElseThrow(() -> new NoSuchElementException("Workflow not found: " + id)); } + @Transactional(readOnly = true) public List list() { - return List.copyOf(workflows.values()); + return repository.findByTenantIdOrderByCreatedAtDesc(TenantSupport.tenantId()).stream() + .map(this::toDomain) + .toList(); } + @Transactional public WorkflowInstance cancel(WorkflowId id) { - var wf = workflows.get(id); - if (wf == null) - throw new NoSuchElementException(); + var wf = get(id); var cancelled = wf.withStatus(WorkflowStatus.CANCELLED); - workflows.put(id, cancelled); + persist(cancelled); return cancelled; } @@ -95,16 +109,17 @@ public boolean isLeader() { /** * Resets leader flag and clears all tracked workflows. Used in test teardown. */ + @Transactional public void reset() { leader.set(false); - workflows.clear(); + repository.deleteAll(); } private void tick() { if (!leader.get()) return; - workflows.forEach((id, wf) -> { + list().forEach(wf -> { if (wf.status() != WorkflowStatus.RUNNING) return; @@ -115,7 +130,7 @@ private void tick() { .toList(); ready.forEach(t -> taskQueue.enqueue( - id.value(), t.taskId(), t.type().name(), wf.pipelineId())); + wf.id().value(), t.taskId(), t.type().name(), wf.pipelineId())); meterRegistry.gauge("syncflow.workflow.queue.size", taskQueue.size()); }); @@ -139,4 +154,35 @@ private List findReadyTasks(WorkflowInstance wf) { .filter(t -> completed.containsAll(t.dependsOn())) .toList(); } + + private Optional findOwned(WorkflowId id) { + return repository.findById(id.value()) + .filter(e -> TenantSupport.tenantId().equals(e.getTenantId())) + .map(this::toDomain); + } + + @Transactional + private void persist(WorkflowInstance wf) { + var entity = repository.findById(wf.id().value()).orElseGet(WorkflowInstanceEntity::new); + entity.setId(wf.id().value()); + entity.setTenantId(TenantSupport.tenantId()); + entity.setPipelineId(wf.pipelineId()); + entity.setStatus(wf.status().name()); + entity.setTasks(json.toJson(wf.tasks())); + entity.setExecutions(json.toJson(wf.executions())); + entity.setCreatedAt(wf.createdAt()); + entity.setCompletedAt(wf.completedAt()); + entity.setUpdatedAt(Instant.now()); + repository.save(entity); + } + + private WorkflowInstance toDomain(WorkflowInstanceEntity e) { + return WorkflowInstance.restore(WorkflowId.from(e.getId()), e.getPipelineId(), + WorkflowStatus.valueOf(e.getStatus()), + json.fromJson(e.getTasks(), new TypeReference>() { + }), + json.fromJson(e.getExecutions(), new TypeReference>() { + }), + e.getCreatedAt(), e.getCompletedAt()); + } } diff --git a/syncflow-api/src/main/java/com/syncflow/api/workflow/entity/WorkflowInstanceEntity.java b/syncflow-api/src/main/java/com/syncflow/api/workflow/entity/WorkflowInstanceEntity.java new file mode 100644 index 0000000..00a0107 --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/workflow/entity/WorkflowInstanceEntity.java @@ -0,0 +1,53 @@ +package com.syncflow.api.workflow.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.Instant; + +/** Workflow instance runtime state; task graph and executions as JSONB. */ +@Setter +@Getter +@Entity +@Table(name = "workflow_instances") +public class WorkflowInstanceEntity { + + @Id + @Column(length = 36) + private String id; + + @Column(name = "tenant_id", nullable = false, length = 36) + private String tenantId = "00000000-0000-0000-0000-000000000000"; + + @Column(name = "pipeline_id", nullable = false, length = 36) + private String pipelineId; + + @Column(nullable = false, length = 20) + private String status; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String tasks; + + @Column(nullable = false, columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) + private String executions; + + @Column(name = "created_at", nullable = false) + private Instant createdAt; + + @Column(name = "completed_at") + private Instant completedAt; + + @Column(name = "updated_at", nullable = false) + private Instant updatedAt; + + public WorkflowInstanceEntity() { + } +} diff --git a/syncflow-api/src/main/java/com/syncflow/api/workflow/repository/WorkflowInstanceRepository.java b/syncflow-api/src/main/java/com/syncflow/api/workflow/repository/WorkflowInstanceRepository.java new file mode 100644 index 0000000..5c9e4ff --- /dev/null +++ b/syncflow-api/src/main/java/com/syncflow/api/workflow/repository/WorkflowInstanceRepository.java @@ -0,0 +1,11 @@ +package com.syncflow.api.workflow.repository; + +import com.syncflow.api.workflow.entity.WorkflowInstanceEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface WorkflowInstanceRepository extends JpaRepository { + + List findByTenantIdOrderByCreatedAtDesc(String tenantId); +} diff --git a/syncflow-api/src/main/resources/application.yml b/syncflow-api/src/main/resources/application.yml index b93f77b..139b488 100644 --- a/syncflow-api/src/main/resources/application.yml +++ b/syncflow-api/src/main/resources/application.yml @@ -86,11 +86,13 @@ syncflow: cache-ttl: 5m sample-size: 100 encryption: - # 16-byte AES key, base64-encoded. Replace in production with a secure key. - key: MDEyMzQ1Njc4OWFiY2RlZg== + # Required. Base64-encoded AES key (16/24/32 bytes). No default — a missing + # or invalid key fails fast at startup via EncryptionService. + key: ${SYNCFLOW_ENCRYPTION_KEY:} jwt: - # Base64-encoded HMAC secret (HS256). Replace in production with a 32+ byte key. - secret: ${SYNCFLOW_JWT_SECRET:c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA==} + # Required. Base64-encoded HMAC secret (>= 32 bytes). No default — a missing + # or invalid secret fails fast at startup via JwtSecurityConfig. + secret: ${SYNCFLOW_JWT_SECRET:} issuer: ${SYNCFLOW_JWT_ISSUER:syncflow} expiry-minutes: ${SYNCFLOW_JWT_EXPIRY_MINUTES:60} ai: diff --git a/syncflow-api/src/main/resources/db/migration/V12__runtime_state_persistence.sql b/syncflow-api/src/main/resources/db/migration/V12__runtime_state_persistence.sql new file mode 100644 index 0000000..42e2fee --- /dev/null +++ b/syncflow-api/src/main/resources/db/migration/V12__runtime_state_persistence.sql @@ -0,0 +1,129 @@ +-- Runtime state persistence: durable storage for the previously in-process +-- ConcurrentHashMap state stores (snapshots, sync jobs, workflows, quotas, +-- audit records, API keys, agents, alerts) so runtime state survives restarts. +-- tenant_id follows V11 conventions: context-less rows and request-path data +-- land in the single default tenant. + +-- Snapshot jobs: full job payload (status/progress/stats/errors) as JSONB plus +-- denormalized status/pipeline columns for tenant-scoped queries. +CREATE TABLE IF NOT EXISTS snapshot_jobs ( + id VARCHAR(36) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + pipeline_id VARCHAR(36) NOT NULL, + status VARCHAR(20) NOT NULL, + payload JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); +CREATE INDEX idx_snapshot_jobs_tenant ON snapshot_jobs(tenant_id); +CREATE INDEX idx_snapshot_jobs_pipeline ON snapshot_jobs(pipeline_id); +CREATE INDEX idx_snapshot_jobs_status ON snapshot_jobs(status); + +-- Snapshot resume checkpoints (one per pipeline+source table per tenant). +CREATE TABLE IF NOT EXISTS snapshot_checkpoints ( + id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + pipeline_id VARCHAR(36) NOT NULL, + source_table VARCHAR(255) NOT NULL, + last_batch_number INTEGER NOT NULL, + rows_processed BIGINT NOT NULL, + cursor_pos VARCHAR(4096), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL, + CONSTRAINT uq_checkpoint_pipeline_table UNIQUE (tenant_id, pipeline_id, source_table) +); + +-- Sync jobs: one live job per tenant+pipeline (matches the orchestrator's +-- in-memory tenant-scoped map key). Statistics stored as JSONB. +CREATE TABLE IF NOT EXISTS sync_jobs ( + id VARCHAR(36) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + pipeline_id VARCHAR(36) NOT NULL, + state VARCHAR(20) NOT NULL, + statistics JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL, + CONSTRAINT uq_sync_job_tenant_pipeline UNIQUE (tenant_id, pipeline_id) +); + +-- Workflow instances: task graph and executions stored as JSONB. +CREATE TABLE IF NOT EXISTS workflow_instances ( + id VARCHAR(36) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + pipeline_id VARCHAR(36) NOT NULL, + status VARCHAR(20) NOT NULL, + tasks JSONB NOT NULL, + executions JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + completed_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); + +-- Tenant quotas: one row per tenant, limits map as JSONB. +CREATE TABLE IF NOT EXISTS quotas ( + tenant_id VARCHAR(36) PRIMARY KEY, + limits JSONB NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); + +-- Enterprise audit records (GDPR right-to-delete via anonymize). +CREATE TABLE IF NOT EXISTS audit_records ( + id UUID PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + actor VARCHAR(255), + action VARCHAR(100) NOT NULL, + resource_type VARCHAR(100), + resource_id VARCHAR(255), + details TEXT, + ip_address VARCHAR(45), + suspicious BOOLEAN NOT NULL DEFAULT FALSE, + event_time TIMESTAMP WITH TIME ZONE NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL +); +CREATE INDEX idx_audit_records_tenant ON audit_records(tenant_id); + +-- API keys: hashed value unique; revoke/expiry drive isActive(). +CREATE TABLE IF NOT EXISTS api_keys ( + id UUID PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + hashed_key VARCHAR(64) NOT NULL, + prefix VARCHAR(16), + label VARCHAR(255), + scope VARCHAR(50), + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + expires_at TIMESTAMP WITH TIME ZONE, + revoked_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT uq_api_keys_hash UNIQUE (hashed_key) +); + +-- Agent fleet: hardware metrics and capabilities as JSONB. +CREATE TABLE IF NOT EXISTS agents ( + id VARCHAR(36) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + version VARCHAR(50), + status VARCHAR(20) NOT NULL, + capabilities JSONB NOT NULL, + labels JSONB NOT NULL, + environment VARCHAR(50), + region VARCHAR(50), + hostname VARCHAR(255), + hardware JSONB NOT NULL, + registered_at TIMESTAMP WITH TIME ZONE NOT NULL, + last_heartbeat TIMESTAMP WITH TIME ZONE NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL +); + +-- Ops alerts: live incidents surfaced on the dashboard. +CREATE TABLE IF NOT EXISTS alert_events ( + id VARCHAR(50) PRIMARY KEY, + tenant_id VARCHAR(36) NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + name VARCHAR(255) NOT NULL, + message TEXT, + severity VARCHAR(20) NOT NULL, + source VARCHAR(255), + pipeline_id VARCHAR(36), + connection_id VARCHAR(36), + event_time TIMESTAMP WITH TIME ZONE NOT NULL, + acknowledged BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL +); diff --git a/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql b/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql new file mode 100644 index 0000000..188413d --- /dev/null +++ b/syncflow-api/src/main/resources/db/migration/V13__debezium_offsets.sql @@ -0,0 +1,10 @@ +-- Debezium offset store: generic Kafka Connect key/value offsets (binary), +-- used by the connector module's JdbcOffsetBackingStore. Debezium keys its +-- offsets by connector namespace + partition, so the key is stored as opaque +-- bytes rather than pipeline_id. Survives pod restarts (unlike the old +-- /tmp FileOffsetBackingStore), preventing re-processing or missed events. +CREATE TABLE IF NOT EXISTS debezium_offsets ( + offset_key BYTEA PRIMARY KEY, + offset_data BYTEA, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); diff --git a/syncflow-api/src/test/java/com/syncflow/api/IntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/IntegrationTest.java index 823550d..55a7c3b 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/IntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/IntegrationTest.java @@ -35,6 +35,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @BeforeEach diff --git a/syncflow-api/src/test/java/com/syncflow/api/RestApiContractTest.java b/syncflow-api/src/test/java/com/syncflow/api/RestApiContractTest.java index 602c65f..9f813ad 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/RestApiContractTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/RestApiContractTest.java @@ -37,6 +37,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } // ============ STATUS 404 - Not Found ============ diff --git a/syncflow-api/src/test/java/com/syncflow/api/agent/AgentApiIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/agent/AgentApiIntegrationTest.java index 567d244..c35146f 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/agent/AgentApiIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/agent/AgentApiIntegrationTest.java @@ -33,6 +33,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Test diff --git a/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java index a025ea6..ed5dcd2 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/cdc/CdcIntegrationTest.java @@ -54,6 +54,13 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); + // Durable Debezium offset store (JdbcOffsetBackingStore) + registry.add("offset.storage.jdbc.url", postgres::getJdbcUrl); + registry.add("offset.storage.jdbc.user", postgres::getUsername); + registry.add("offset.storage.jdbc.password", postgres::getPassword); + registry.add("offset.storage.jdbc.table.name", () -> "debezium_offsets"); } private Connection sqlConnection; diff --git a/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java index 77a1d34..e7d854b 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/config/AbstractIntegrationTest.java @@ -1,5 +1,9 @@ package com.syncflow.api.config; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.jwk.JWKSet; +import com.nimbusds.jose.jwk.OctetSequenceKey; +import com.nimbusds.jose.jwk.source.ImmutableJWKSet; import io.restassured.RestAssured; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -8,8 +12,17 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.context.annotation.Import; +import org.springframework.security.oauth2.jose.jws.MacAlgorithm; +import org.springframework.security.oauth2.jwt.JwsHeader; +import org.springframework.security.oauth2.jwt.JwtClaimsSet; +import org.springframework.security.oauth2.jwt.JwtEncoderParameters; +import org.springframework.security.oauth2.jwt.NimbusJwtEncoder; import org.testcontainers.junit.jupiter.Testcontainers; +import javax.crypto.spec.SecretKeySpec; +import java.time.Instant; +import java.util.Base64; + /** * Shared base for full-context (@SpringBootTest) integration tests. * Carries the Spring Boot/Testcontainers annotations, a permissive security @@ -17,6 +30,10 @@ * RestAssured port wiring. Subclasses declare their OWN * {@code @Container postgres} and {@code @DynamicPropertySource} so each test * keeps the exact database/credentials/data it needs. + * + * {@link #adminToken(String)} mints an admin JWT (subject {@code admin} => + * full RBAC via PolicyResolver) scoped to the given tenant — use it on + * RBAC-guarded mutations so the tenant-aware principal is populated. */ @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Testcontainers @@ -25,6 +42,9 @@ @Import(TestSecurityConfig.class) public abstract class AbstractIntegrationTest { + /** Matches the base64 secret used by the integration tests. */ + protected static final String TEST_JWT_SECRET = "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="; + @LocalServerPort protected int port; @@ -37,4 +57,28 @@ void setUpBase() { void tearDownBase() { RestAssured.reset(); } + + /** + * Admin bearer token scoped to {@code tenantId}; subject 'admin' => full RBAC. + */ + protected String adminToken(String tenantId) { + var encoder = new NimbusJwtEncoder(new ImmutableJWKSet<>(new JWKSet( + new OctetSequenceKey.Builder(new SecretKeySpec( + Base64.getDecoder().decode(TEST_JWT_SECRET), "HmacSHA256")) + .algorithm(JWSAlgorithm.HS256).build()))); + var claims = JwtClaimsSet.builder() + .issuer("syncflow") + .subject("admin") + .issuedAt(Instant.now()) + .expiresAt(Instant.now().plusSeconds(3600)) + // The single-tenant default has a canonical UUID value; a literal + // "default" is a DIFFERENT TenantId and would scope to nothing. + .claim("tid", "default".equals(tenantId) + ? com.syncflow.tenant.TenantId.DEFAULT.value() + : tenantId) + .claim("scope", "ADMIN") + .build(); + return encoder.encode(JwtEncoderParameters.from( + JwsHeader.with(MacAlgorithm.HS256).build(), claims)).getTokenValue(); + } } diff --git a/syncflow-api/src/test/java/com/syncflow/api/config/TestSecurityConfig.java b/syncflow-api/src/test/java/com/syncflow/api/config/TestSecurityConfig.java index 39d2bef..a40e61f 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/config/TestSecurityConfig.java +++ b/syncflow-api/src/test/java/com/syncflow/api/config/TestSecurityConfig.java @@ -3,14 +3,20 @@ import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; +import org.springframework.core.convert.converter.Converter; +import org.springframework.security.authentication.AbstractAuthenticationToken; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; +import org.springframework.security.oauth2.jwt.Jwt; import org.springframework.security.web.SecurityFilterChain; /** * Permissive security for full-context (@SpringBootTest) integration tests. - * The real WebSecurityConfig authenticates all /api/** except health; contract - * tests hit those endpoints without a token, so this disables auth entirely. + * All requests are permitted (contract tests hit endpoints without a token), + * BUT a valid bearer token is still parsed by the resource server so the + * tenant-aware principal is populated — integration tests that exercise + * RBAC-guarded mutations authenticate with an admin token and get real + * tenant/RBAC context. * * Marked @Primary so it wins over the component-scanned WebSecurityConfig bean. */ @@ -19,9 +25,12 @@ public class TestSecurityConfig { @Bean @Primary - public SecurityFilterChain testFilterChain(HttpSecurity http) throws Exception { + public SecurityFilterChain testFilterChain(HttpSecurity http, + Converter jwtAuthenticationConverter) throws Exception { http.csrf(AbstractHttpConfigurer::disable) - .authorizeHttpRequests(auth -> auth.anyRequest().permitAll()); + .authorizeHttpRequests(auth -> auth.anyRequest().permitAll()) + .oauth2ResourceServer(oauth2 -> oauth2 + .jwt(jwt -> jwt.jwtAuthenticationConverter(jwtAuthenticationConverter))); return http.build(); } } diff --git a/syncflow-api/src/test/java/com/syncflow/api/connection/ConnectionControllerTest.java b/syncflow-api/src/test/java/com/syncflow/api/connection/ConnectionControllerTest.java index 89973e6..b09f9d4 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/connection/ConnectionControllerTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/connection/ConnectionControllerTest.java @@ -9,6 +9,7 @@ import com.syncflow.core.connection.ConnectionType; import com.syncflow.core.connection.Credentials; import com.syncflow.core.connection.spi.ConnectorFactory; +import com.syncflow.api.security.rbac.AuthorizationService; import com.syncflow.api.config.versioning.VersionContext; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -49,6 +50,9 @@ class ConnectionControllerTest { @MockitoBean private ConnectorFactory connectorFactory; + @MockitoBean + private AuthorizationService authz; + @Test void createConnection_returns201() throws Exception { var request = new CreateConnectionRequest("test-conn", ConnectionType.POSTGRESQL, diff --git a/syncflow-api/src/test/java/com/syncflow/api/db/DatabaseMigrationValidationTest.java b/syncflow-api/src/test/java/com/syncflow/api/db/DatabaseMigrationValidationTest.java index db9eec0..3c11a65 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/db/DatabaseMigrationValidationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/db/DatabaseMigrationValidationTest.java @@ -33,6 +33,8 @@ static void properties(DynamicPropertyRegistry r) { r.add("spring.datasource.password", postgres::getPassword); r.add("spring.flyway.enabled", () -> "true"); r.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + r.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); r.add("spring.flyway.baseline-on-migrate", () -> "true"); r.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); } diff --git a/syncflow-api/src/test/java/com/syncflow/api/k8s/KubernetesIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/k8s/KubernetesIntegrationTest.java index 1c64eac..bd7a904 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/k8s/KubernetesIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/k8s/KubernetesIntegrationTest.java @@ -38,6 +38,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Autowired diff --git a/syncflow-api/src/test/java/com/syncflow/api/metadata/MetadataIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/metadata/MetadataIntegrationTest.java index 5c90e9e..8e1799e 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/metadata/MetadataIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/metadata/MetadataIntegrationTest.java @@ -44,6 +44,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); registry.add("spring.jpa.hibernate.ddl-auto", () -> "validate"); } diff --git a/syncflow-api/src/test/java/com/syncflow/api/ops/ObservabilityIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/ops/ObservabilityIntegrationTest.java index 21261c4..dd840cd 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/ops/ObservabilityIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/ops/ObservabilityIntegrationTest.java @@ -40,6 +40,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Autowired diff --git a/syncflow-api/src/test/java/com/syncflow/api/pipeline/PipelineApiIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/pipeline/PipelineApiIntegrationTest.java index 98c1350..5ca28c9 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/pipeline/PipelineApiIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/pipeline/PipelineApiIntegrationTest.java @@ -32,6 +32,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } private String createdPipelineId; @@ -153,6 +155,7 @@ void deletePipeline() { .path("id"); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().delete("/api/pipelines/{id}", id) .then() .statusCode(204); diff --git a/syncflow-api/src/test/java/com/syncflow/api/samples/PgMongoSampleE2eTest.java b/syncflow-api/src/test/java/com/syncflow/api/samples/PgMongoSampleE2eTest.java index 39fae01..7035e95 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/samples/PgMongoSampleE2eTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/samples/PgMongoSampleE2eTest.java @@ -58,6 +58,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Autowired diff --git a/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java b/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java index c0be3f1..e9036e4 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/security/TenantDataIsolationTest.java @@ -1,16 +1,33 @@ package com.syncflow.api.security; -import com.syncflow.api.config.AbstractIntegrationTest; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.jwk.JWKSet; +import com.nimbusds.jose.jwk.OctetSequenceKey; +import com.nimbusds.jose.jwk.source.ImmutableJWKSet; +import io.restassured.RestAssured; import io.restassured.http.ContentType; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.security.oauth2.jose.jws.MacAlgorithm; +import org.springframework.security.oauth2.jwt.JwsHeader; +import org.springframework.security.oauth2.jwt.JwtClaimsSet; +import org.springframework.security.oauth2.jwt.JwtEncoder; +import org.springframework.security.oauth2.jwt.JwtEncoderParameters; +import org.springframework.security.oauth2.jwt.NimbusJwtEncoder; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; -import org.junit.jupiter.api.AfterEach; - +import javax.crypto.spec.SecretKeySpec; +import java.time.Instant; +import java.util.Base64; import java.util.Map; import static io.restassured.RestAssured.given; @@ -19,8 +36,18 @@ /** * Proves the P0 data-isolation fix end-to-end: data created by one tenant is * invisible to another tenant across the real HTTP + scoped-repository path. + * + * Tenant scope comes from the AUTHENTICATED PRINCIPAL (JWT {@code tid} claim), + * not the {@code X-Tenant-Id} header — the header is only a UI hint that must + * match the principal. Each tenant here is represented by a distinct admin JWT. */ -class TenantDataIsolationTest extends AbstractIntegrationTest { +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Testcontainers +@Tag("integration") +@EnabledIfSystemProperty(named = "tests.integration", matches = "true") +class TenantDataIsolationTest { + + private static final String SECRET = "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="; @Container static PostgreSQLContainer postgres = new PostgreSQLContainer<>("postgres:16-alpine") @@ -28,6 +55,9 @@ class TenantDataIsolationTest extends AbstractIntegrationTest { .withUsername("testuser") .withPassword("testpass"); + @LocalServerPort + private int port; + @DynamicPropertySource static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.url", postgres::getJdbcUrl); @@ -35,28 +65,45 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", () -> SECRET); } private static final String TENANT_A = "tenant-a"; private static final String TENANT_B = "tenant-b"; - @AfterEach - void tearDown() { - // Each test asserts exact counts; remove connections created by prior tests - // under both tenants so counts don't accumulate across the shared DB. - for (var tenant : java.util.List.of(TENANT_A, TENANT_B)) { - given().header("X-Tenant-Id", tenant) - .when().get("/api/connections") - .then().statusCode(200) - .extract().jsonPath().getList("$", Map.class) - .forEach(row -> given().header("X-Tenant-Id", tenant) - .when().delete("/api/connections/{id}", row.get("id")) - .then().statusCode(204)); - } + private final JwtEncoder encoder = new NimbusJwtEncoder(new ImmutableJWKSet<>(new JWKSet( + new OctetSequenceKey.Builder(new SecretKeySpec(Base64.getDecoder().decode(SECRET), "HmacSHA256")) + .algorithm(JWSAlgorithm.HS256).build()))); + + @BeforeEach + void setUp() { + RestAssured.port = port; + } + + /** + * An admin bearer token scoped to the given tenant (subject 'admin' => full + * RBAC). + */ + private String tokenFor(String tenantId) { + var claims = JwtClaimsSet.builder() + .issuer("syncflow") + .subject("admin") + .issuedAt(Instant.now()) + .expiresAt(Instant.now().plusSeconds(3600)) + .claim("tid", tenantId) + .claim("scope", "ADMIN") + .build(); + return encoder.encode(JwtEncoderParameters.from(JwsHeader.with(MacAlgorithm.HS256).build(), claims)) + .getTokenValue(); + } + + private io.restassured.response.Response deleteConnection(String tenantId, String id) { + return given().header("Authorization", "Bearer " + tokenFor(tenantId)) + .when().delete("/api/connections/{id}", id); } private void createConnection(String tenantId, String name) { - given().header("X-Tenant-Id", tenantId) + given().header("Authorization", "Bearer " + tokenFor(tenantId)) .contentType(ContentType.JSON) .body(Map.of( "name", name, @@ -67,30 +114,43 @@ private void createConnection(String tenantId, String name) { .then().statusCode(201); } + private void cleanup(String tenantId) { + given().header("Authorization", "Bearer " + tokenFor(tenantId)) + .when().get("/api/connections") + .then().statusCode(200) + .extract().jsonPath().getList("$", Map.class) + .forEach(row -> deleteConnection(tenantId, (String) row.get("id")).then().statusCode(204)); + } + @Test @DisplayName("tenant A's connections are invisible to tenant B") void connectionListIsTenantScoped() { - createConnection(TENANT_A, "a-conn"); + try { + createConnection(TENANT_A, "a-conn"); - // Tenant B sees nothing created by tenant A. - int tenantB = given().header("X-Tenant-Id", TENANT_B) - .when().get("/api/connections") - .then().statusCode(200) - .extract().jsonPath().getList("$").size(); - assertEquals(0, tenantB, "tenant B must not see tenant A's connections"); + // Tenant B sees nothing created by tenant A. + int tenantB = given().header("Authorization", "Bearer " + tokenFor(TENANT_B)) + .when().get("/api/connections") + .then().statusCode(200) + .extract().jsonPath().getList("$").size(); + assertEquals(0, tenantB, "tenant B must not see tenant A's connections"); - // Tenant A sees its own. - int tenantA = given().header("X-Tenant-Id", TENANT_A) - .when().get("/api/connections") - .then().statusCode(200) - .extract().jsonPath().getList("$").size(); - assertEquals(1, tenantA, "tenant A must see its own connection"); + // Tenant A sees its own. + int tenantA = given().header("Authorization", "Bearer " + tokenFor(TENANT_A)) + .when().get("/api/connections") + .then().statusCode(200) + .extract().jsonPath().getList("$").size(); + assertEquals(1, tenantA, "tenant A must see its own connection"); + } finally { + cleanup(TENANT_A); + cleanup(TENANT_B); + } } @Test @DisplayName("tenant B cannot fetch tenant A's connection by id") void connectionGetIsTenantScoped() { - var id = given().header("X-Tenant-Id", TENANT_A) + var id = given().header("Authorization", "Bearer " + tokenFor(TENANT_A)) .contentType(ContentType.JSON) .body(Map.of( "name", "a-conn-2", @@ -100,15 +160,19 @@ void connectionGetIsTenantScoped() { .when().post("/api/connections") .then().statusCode(201) .extract().path("id"); - - // Tenant B gets a 404 for tenant A's connection. - given().header("X-Tenant-Id", TENANT_B) - .when().get("/api/connections/{id}", id) - .then().statusCode(404); - - // Tenant A can fetch it. - given().header("X-Tenant-Id", TENANT_A) - .when().get("/api/connections/{id}", id) - .then().statusCode(200); + try { + // Tenant B gets a 404 for tenant A's connection. + given().header("Authorization", "Bearer " + tokenFor(TENANT_B)) + .when().get("/api/connections/{id}", id) + .then().statusCode(404); + + // Tenant A can fetch it. + given().header("Authorization", "Bearer " + tokenFor(TENANT_A)) + .when().get("/api/connections/{id}", id) + .then().statusCode(200); + } finally { + cleanup(TENANT_A); + cleanup(TENANT_B); + } } } diff --git a/syncflow-api/src/test/java/com/syncflow/api/snapshot/SnapshotIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/snapshot/SnapshotIntegrationTest.java index e584a80..5601b48 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/snapshot/SnapshotIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/snapshot/SnapshotIntegrationTest.java @@ -55,6 +55,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @BeforeEach diff --git a/syncflow-api/src/test/java/com/syncflow/api/sse/SseIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/sse/SseIntegrationTest.java index 2cd1ab8..af543b9 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/sse/SseIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/sse/SseIntegrationTest.java @@ -44,6 +44,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @Autowired diff --git a/syncflow-api/src/test/java/com/syncflow/api/sync/SyncIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/sync/SyncIntegrationTest.java index 843a609..c8810fe 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/sync/SyncIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/sync/SyncIntegrationTest.java @@ -48,6 +48,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } @BeforeEach @@ -100,6 +102,7 @@ void deleteDlq() { var events = dlq.list("p-1"); if (!events.isEmpty()) { given() + .header("Authorization", "Bearer " + adminToken("default")) .when().delete("/api/dlq/{id}", events.getFirst().id()) .then() .statusCode(204); diff --git a/syncflow-api/src/test/java/com/syncflow/api/workflow/WorkflowApiIntegrationTest.java b/syncflow-api/src/test/java/com/syncflow/api/workflow/WorkflowApiIntegrationTest.java index 5ee3d05..03e5d0b 100644 --- a/syncflow-api/src/test/java/com/syncflow/api/workflow/WorkflowApiIntegrationTest.java +++ b/syncflow-api/src/test/java/com/syncflow/api/workflow/WorkflowApiIntegrationTest.java @@ -34,6 +34,8 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.password", postgres::getPassword); registry.add("spring.flyway.enabled", () -> "true"); registry.add("syncflow.encryption.key", () -> "MDEyMzQ1Njc4OWFiY2RlZg=="); + registry.add("syncflow.jwt.secret", + () -> "c3luY2Zsb3ctaHMyNTYtand0LXNlY3JldC1rZXktMjAyNi1jaGFuZ2UtaW4tcHJvZA=="); } // --- Sequential workflow --- @@ -41,6 +43,7 @@ static void properties(DynamicPropertyRegistry registry) { @Test void createAndStartSequentialWorkflow() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-1")) .when().post("/api/workflows") @@ -55,6 +58,7 @@ void createAndStartSequentialWorkflow() { assertEquals("p-1", wf.pipelineId()); var started = given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/start", wf.id().value()) .then() .statusCode(200) @@ -67,6 +71,7 @@ void createAndStartSequentialWorkflow() { @Test void sequentialWorkflowStatusTransitions() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-seq")) .post("/api/workflows") @@ -84,17 +89,20 @@ void sequentialWorkflowStatusTransitions() { @Test void cancelWorkflow() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-cancel")) .post("/api/workflows") .path("id"); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/start", wf) .then() .statusCode(200); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/cancel", wf) .then() .statusCode(200) @@ -117,6 +125,7 @@ void listWorkflows() { @Test void getWorkflowGraph() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-graph")) .post("/api/workflows") @@ -134,17 +143,20 @@ void getWorkflowGraph() { @Test void pauseAndResumeWorkflow() { var wf = given() + .header("Authorization", "Bearer " + adminToken("default")) .contentType(ContentType.JSON) .body(Map.of("pipelineId", "p-pr")) .post("/api/workflows") .path("id"); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/pause", wf) .then() .statusCode(200); given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/{id}/resume", wf) .then() .statusCode(200); @@ -163,6 +175,7 @@ void getNonExistentWorkflow() { @Test void cancelNonExistentWorkflow() { given() + .header("Authorization", "Bearer " + adminToken("default")) .when().post("/api/workflows/nonexistent/cancel") .then() .statusCode(500); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java index 9fdce63..333edcd 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/DebeziumCdcConnector.java @@ -55,6 +55,15 @@ public abstract class DebeziumCdcConnector implements CdcCapableConnector { protected abstract CDCEvent buildEvent(ChangeEvent event, ConnectorContext ctx); + /** + * JDBC URL for the source database, used by the durable offset store. + * Must be overridden by subclasses that support persistent offsets. + */ + protected String jdbcUrl(ConnectionConfiguration config) { + throw new UnsupportedOperationException( + connectorType() + " does not support a JDBC offset store"); + } + // ── Offset management ──────────────────────────────────────────────────── /** @@ -172,12 +181,16 @@ public void startCDC(ConnectorContext context, Consumer eventConsumer) debeziumProps.setProperty("name", "syncflow-" + connectorType().name().toLowerCase()); debeziumProps.setProperty("connector.class", connectorClassName()); - // use FileOffsetBackingStore so offsets survive JVM restarts - // each pipeline gets its own offset file keyed by pipeline id from context - var offsetFile = resolveOffsetFilePath(context); + // Durable offset store: Postgres-backed (survives pod restarts/reschedules). + // Plain JDBC — no JPA — so the connector module stays Spring-Data-free. + // The table is created by Flyway migration V13 (debezium_offsets). debeziumProps.setProperty("offset.storage", - "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - debeziumProps.setProperty("offset.storage.file.filename", offsetFile); + "com.syncflow.connector.cdc.JdbcOffsetBackingStore"); + debeziumProps.setProperty("offset.storage.jdbc.url", + jdbcUrl(config)); + debeziumProps.setProperty("offset.storage.jdbc.user", config.username()); + debeziumProps.setProperty("offset.storage.jdbc.password", config.password()); + debeziumProps.setProperty("offset.storage.jdbc.table.name", "debezium_offsets"); debeziumProps.setProperty("offset.flush.interval.ms", "5000"); debeziumProps.setProperty("topic.prefix", "syncflow"); @@ -294,20 +307,4 @@ private void handleSingleEvent(ChangeEvent event) { } } - /** - * Resolve a stable per-pipeline offset file path. - * Keyed by connector + host + database + PIPELINE id so multiple pipelines on - * the same database get their own offset file (shared files corrupt resume). - */ - private String resolveOffsetFilePath(ConnectorContext context) { - var config = context.config(); - var dir = System.getProperty("java.io.tmpdir"); - var pipelineKey = context.runtimeProperties().getOrDefault("pipelineId", "default"); - var safePipeline = pipelineKey.replaceAll("[^a-zA-Z0-9_-]", "_"); - var key = connectorType().name().toLowerCase() - + "_" + config.host().replace(".", "_") - + "_" + config.database() - + "_" + safePipeline; - return dir + "/syncflow_offset_" + key + ".dat"; - } } diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java new file mode 100644 index 0000000..8654a71 --- /dev/null +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/JdbcOffsetBackingStore.java @@ -0,0 +1,165 @@ +package com.syncflow.connector.cdc; + +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.storage.OffsetBackingStore; +import org.apache.kafka.connect.util.Callback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; + +/** + * Postgres-backed {@link OffsetBackingStore} for Debezium. + *

+ * Persists connector offsets in the {@code cdc_offsets} table (the same table + * the control-plane {@code OffsetStore} writes) instead of the ephemeral + * {@code /tmp} file used by {@code FileOffsetBackingStore}. Offsets therefore + * survive pod restarts and reschedules — no re-processing or missed events. + *

+ * Configured by the properties prefixed {@code offset.storage.jdbc.*} set in + * {@link DebeziumCdcConnector#startCDC}; the row key is the connector's own + * offset key (namespace + partition), stored JSON-encoded by Kafka Connect. + *

+ * This store is plain JDBC (no JPA) so the connector module keeps no Spring + * Data dependency; the JDBC driver is already on the module classpath. + */ +public class JdbcOffsetBackingStore implements OffsetBackingStore { + + private static final Logger log = LoggerFactory.getLogger(JdbcOffsetBackingStore.class); + + private String jdbcUrl; + private String jdbcUser; + private String jdbcPassword; + private String tableName = "cdc_offsets"; + + // In-memory cache of offsets read at start() so get() never hits the DB for + // already-loaded partitions; writes are batched into set() then flushed. + private final Map cache = new HashMap<>(); + + @Override + public void configure(WorkerConfig config) { + var originals = config.originalsWithPrefix("offset.storage.jdbc."); + jdbcUrl = stringValue(originals, "url", null); + jdbcUser = stringValue(originals, "user", ""); + jdbcPassword = stringValue(originals, "password", ""); + var table = stringValue(originals, "table.name", null); + if (table != null) { + tableName = table; + } + if (jdbcUrl == null) { + throw new IllegalStateException( + "offset.storage.jdbc.url is required for JdbcOffsetBackingStore"); + } + } + + @Override + public void start() { + // Load all persisted offsets into memory so get() resolves without a DB + // round trip on the CDC hot path. + try (var conn = connection(); + var stmt = conn.createStatement(); + var rs = stmt.executeQuery( + "SELECT offset_key, offset_data FROM " + tableName)) { + while (rs.next()) { + cache.put(fromDbBytes(rs.getBytes("offset_key")), fromDbBytes(rs.getBytes("offset_data"))); + } + log.info("Loaded {} persisted CDC offsets from {}", cache.size(), tableName); + } catch (SQLException e) { + // Table may not exist on a fresh database before Flyway migrates; the + // CDC engine treats a missing store as a cold start. + log.warn("Could not load persisted CDC offsets from {}: {}", tableName, e.getMessage()); + } + } + + @Override + public void stop() { + cache.clear(); + } + + @Override + public Future> get(Collection keys) { + var result = new HashMap(); + for (var key : keys) { + var value = cache.get(key); + if (value != null) { + result.put(key.duplicate(), value.duplicate()); + } + } + return CompletableFuture.completedFuture(result); + } + + @Override + public Future set(Map values, Callback callback) { + try { + try (var conn = connection(); + var upsert = conn.prepareStatement( + "INSERT INTO " + tableName + + " (offset_key, offset_data) VALUES (?, ?) " + + "ON CONFLICT (offset_key) DO UPDATE SET offset_data = EXCLUDED.offset_data")) { + for (var entry : values.entrySet()) { + var key = entry.getKey().duplicate(); + var value = entry.getValue() != null ? entry.getValue().duplicate() : null; + cache.put(key, value); + upsert.setBytes(1, toDbBytes(key)); + upsert.setBytes(2, value != null ? toDbBytes(value) : new byte[0]); + upsert.addBatch(); + } + upsert.executeBatch(); + } + if (callback != null) { + callback.onCompletion(null, null); + } + return CompletableFuture.completedFuture(null); + } catch (Exception e) { + log.error("Failed to persist CDC offsets", e); + if (callback != null) { + callback.onCompletion(e, null); + } + return CompletableFuture.failedFuture(e); + } + } + + @Override + public Set> connectorPartitions(String connectorName) { + return Set.of(); + } + + // ---- helpers ---- + + private Connection connection() throws SQLException { + var props = new Properties(); + if (jdbcUser != null) { + props.setProperty("user", jdbcUser); + } + if (jdbcPassword != null) { + props.setProperty("password", jdbcPassword); + } + return DriverManager.getConnection(jdbcUrl, props); + } + + private static String stringValue(Map m, String key, String def) { + var v = m.get(key); + return v != null ? String.valueOf(v) : def; + } + + private static ByteBuffer fromDbBytes(byte[] bytes) { + return bytes == null ? null : ByteBuffer.wrap(bytes); + } + + private static byte[] toDbBytes(ByteBuffer buf) { + var copy = buf.duplicate(); + var bytes = new byte[copy.remaining()]; + copy.get(bytes); + return bytes; + } +} diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java index ac2ce13..08efcb8 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/MySqlCdcConnector.java @@ -42,6 +42,12 @@ protected String connectorClassName() { return "io.debezium.connector.mysql.MySqlConnector"; } + @Override + protected String jdbcUrl(ConnectionConfiguration config) { + return "jdbc:mysql://" + config.host() + ":" + config.port() + + "/" + config.database(); + } + @Override protected Properties specificProperties(ConnectionConfiguration config) { var props = new Properties(); diff --git a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java index 559571e..d24d1ed 100644 --- a/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java +++ b/syncflow-connectors/src/main/java/com/syncflow/connector/cdc/PostgresCdcConnector.java @@ -45,6 +45,12 @@ protected String connectorClassName() { return "io.debezium.connector.postgresql.PostgresConnector"; } + @Override + protected String jdbcUrl(ConnectionConfiguration config) { + return "jdbc:postgresql://" + config.host() + ":" + config.port() + + "/" + config.database(); + } + /** * slot name and publication name are scoped per pipeline using the * database name so multiple pipelines pointing to different databases don't diff --git a/syncflow-core/src/main/java/com/syncflow/core/snapshot/SnapshotJob.java b/syncflow-core/src/main/java/com/syncflow/core/snapshot/SnapshotJob.java index 33d87bc..5f77112 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/snapshot/SnapshotJob.java +++ b/syncflow-core/src/main/java/com/syncflow/core/snapshot/SnapshotJob.java @@ -46,6 +46,15 @@ private SnapshotJob( this.createdAt = createdAt; } + /** Rebuild a persisted job. */ + public static SnapshotJob restore(SnapshotId id, String pipelineId, SnapshotStatus status, + SnapshotStatistics statistics, SnapshotProgress progress, + List errors, Instant createdAt) { + return new SnapshotJob(id, pipelineId, status, statistics, + progress != null ? progress : SnapshotProgress.starting(0), + errors != null ? List.copyOf(errors) : List.of(), createdAt); + } + public SnapshotJob withRunning() { return new SnapshotJob(id, pipelineId, SnapshotStatus.RUNNING, statistics, progress, errors, createdAt); } diff --git a/syncflow-core/src/main/java/com/syncflow/core/sync/SyncJob.java b/syncflow-core/src/main/java/com/syncflow/core/sync/SyncJob.java index 3b630f0..3c5da8d 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/sync/SyncJob.java +++ b/syncflow-core/src/main/java/com/syncflow/core/sync/SyncJob.java @@ -36,6 +36,15 @@ private SyncJob(String id, String pipelineId, SyncState state, this.createdAt = createdAt; } + /** + * Rebuild a job from persisted state; the processed-events buffer is not + * persisted. + */ + public static SyncJob restore(String id, String pipelineId, SyncState state, + SyncStatistics stats, Instant createdAt) { + return new SyncJob(id, pipelineId, state, stats, List.of(), createdAt); + } + public SyncJob withRunning() { return new SyncJob(id, pipelineId, SyncState.RUNNING, statistics, processed, createdAt); } diff --git a/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java b/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java index e249211..b8cc63b 100644 --- a/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java +++ b/syncflow-core/src/main/java/com/syncflow/core/workflow/WorkflowInstance.java @@ -26,4 +26,11 @@ public static WorkflowInstance create(String pipelineId, List task return new WorkflowInstance(WorkflowId.generate(), pipelineId, WorkflowStatus.PENDING, tasks, List.of(), Instant.now(), null); } + + /** Rebuild a persisted workflow instance. */ + public static WorkflowInstance restore(WorkflowId id, String pipelineId, WorkflowStatus status, + List tasks, List executions, + Instant createdAt, Instant completedAt) { + return new WorkflowInstance(id, pipelineId, status, tasks, executions, createdAt, completedAt); + } } diff --git a/syncflow-security/src/main/java/com/syncflow/security/SecurityConfig.java b/syncflow-security/src/main/java/com/syncflow/security/SecurityConfig.java index a01d82e..9f1c9e6 100644 --- a/syncflow-security/src/main/java/com/syncflow/security/SecurityConfig.java +++ b/syncflow-security/src/main/java/com/syncflow/security/SecurityConfig.java @@ -12,6 +12,13 @@ public final class SecurityConfig { private static final List PUBLIC_PATHS = List.of( "/api/health/**", "/api/auth/**", + // Fleet agent inbound endpoints: public here because the agent + // cannot do OAuth; gated fail-closed by AgentTokenFilter + // (X-Agent-Token) when syncflow.agent.token is configured. Control- + // plane agent ops (list/get/drain/restart) stay authenticated. + // mTLS is the upgrade path (see ADR-008/009). + "/api/agents/register", + "/api/agents/heartbeat", "/actuator/**", "/v3/api-docs/**", "/swagger-ui/**",