Skip to content

Commit a32b8e6

Browse files
authored
Merge pull request #23 from ProjectVG/feat/distribute-system
Feature : 분산 시스템 세션 개선 - Redis Pub/sub
2 parents 229695b + 095564f commit a32b8e6

File tree

50 files changed

+3313
-388
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+3313
-388
lines changed

.env.example

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,23 @@
33
# ===========================================
44
# Copy this file to .env and fill in your actual values
55

6+
ENVIRONMENT=development
7+
DEBUG_MODE=true
8+
LOG_LEVEL=INFO
9+
LOG_FORMAT=json
10+
DATA_PATH=../data
11+
612
# Database Configuration
713
DB_CONNECTION_STRING=Server=host.docker.internal,1433;Database=ProjectVG;User Id=sa;Password=YOUR_DB_PASSWORD;TrustServerCertificate=true;MultipleActiveResultSets=true
814
DB_PASSWORD=YOUR_DB_PASSWORD
915

1016
# Redis Configuration
1117
REDIS_CONNECTION_STRING=host.docker.internal:6380
1218

19+
# Distributed System Configuration
20+
DISTRIBUTED_MODE=false
21+
SERVER_ID=
22+
1323
# External Services
1424
LLM_BASE_URL=http://host.docker.internal:7930
1525
MEMORY_BASE_URL=http://host.docker.internal:7940
@@ -31,6 +41,11 @@ GOOGLE_OAUTH_REDIRECT_URI=http://localhost:7900/auth/oauth2/callback
3141
GOOGLE_OAUTH_AUTO_CREATE_USER=true
3242
GOOGLE_OAUTH_DEFAULT_ROLE=User
3343

44+
# WebSocket Configuration
45+
WEBSOCKET_KEEPALIVE_MINUTES=0
46+
WEBSOCKET_RECEIVE_BUFFER_SIZE=4096
47+
WEBSOCKET_SEND_BUFFER_SIZE=4096
48+
3449
# Application Configuration
3550
ASPNETCORE_ENVIRONMENT=Production
3651

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ _ReSharper*/
101101
# Docker
102102
**/Dockerfile.*
103103
docker-compose.override.yml
104+
.dockerignore.local
104105

105106
# Keep template files but ignore runtime files
106107
!docker-compose.prod.yml

ProjectVG.Api/ApiMiddlewareExtensions.cs

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ public static class ApiMiddlewareExtensions
1111
/// </summary>
1212
public static IApplicationBuilder UseApiMiddleware(this IApplicationBuilder app, IWebHostEnvironment environment)
1313
{
14+
var configuration = app.ApplicationServices.GetRequiredService<IConfiguration>();
1415
// 개발 환경 설정
1516
if (environment.IsDevelopment()) {
1617
app.UseSwagger();
@@ -23,8 +24,9 @@ public static IApplicationBuilder UseApiMiddleware(this IApplicationBuilder app,
2324
// 전역 예외 처리
2425
app.UseGlobalExceptionHandler();
2526

26-
// WebSocket 지원
27-
app.UseWebSockets();
27+
// WebSocket 지원 - 구성 가능한 옵션 사용
28+
var webSocketOptions = GetWebSocketOptions(configuration);
29+
app.UseWebSockets(webSocketOptions);
2830

2931
// WebSocket 미들웨어 등록
3032
app.UseMiddleware<WebSocketMiddleware>();
@@ -63,5 +65,56 @@ public static IApplicationBuilder UseDevelopmentFeatures(this IApplicationBuilde
6365

6466
return app;
6567
}
68+
69+
/// <summary>
70+
/// WebSocket 옵션을 구성 파일과 환경 변수에서 가져옵니다
71+
/// </summary>
72+
private static WebSocketOptions GetWebSocketOptions(IConfiguration configuration)
73+
{
74+
var options = new WebSocketOptions();
75+
76+
// KeepAliveInterval 설정 (환경 변수 > appsettings.json 순서)
77+
var keepAliveMinutes = Environment.GetEnvironmentVariable("WEBSOCKET_KEEPALIVE_MINUTES");
78+
if (string.IsNullOrEmpty(keepAliveMinutes))
79+
{
80+
keepAliveMinutes = configuration.GetValue<string>("WebSocket:KeepAliveIntervalMinutes");
81+
}
82+
83+
if (double.TryParse(keepAliveMinutes, out var minutes))
84+
{
85+
if (minutes <= 0)
86+
{
87+
options.KeepAliveInterval = TimeSpan.Zero; // KeepAlive 비활성화
88+
}
89+
else
90+
{
91+
options.KeepAliveInterval = TimeSpan.FromMinutes(minutes);
92+
}
93+
}
94+
else
95+
{
96+
// 기본값: KeepAlive 비활성화 (연결 안정성을 위해)
97+
options.KeepAliveInterval = TimeSpan.Zero;
98+
}
99+
100+
// 수신 버퍼 크기 설정
101+
var receiveBufferSize = Environment.GetEnvironmentVariable("WEBSOCKET_RECEIVE_BUFFER_SIZE") ??
102+
configuration.GetValue<string>("WebSocket:ReceiveBufferSize");
103+
if (int.TryParse(receiveBufferSize, out var recvSize) && recvSize > 0)
104+
{
105+
options.ReceiveBufferSize = recvSize;
106+
}
107+
108+
// 송신 버퍼 크기 설정 (WebSocketOptions에는 없으므로 로깅만)
109+
var sendBufferSize = Environment.GetEnvironmentVariable("WEBSOCKET_SEND_BUFFER_SIZE") ??
110+
configuration.GetValue<string>("WebSocket:SendBufferSize");
111+
112+
// 콘솔 로깅으로 설정 확인
113+
Console.WriteLine($"[WebSocket 설정] KeepAlive: {(options.KeepAliveInterval == TimeSpan.Zero ? "비활성화" : $"{options.KeepAliveInterval.TotalMinutes}분")}, " +
114+
$"ReceiveBuffer: {options.ReceiveBufferSize} bytes" +
115+
$"{(int.TryParse(sendBufferSize, out var sendSize) && sendSize > 0 ? $", SendBuffer: {sendSize} bytes (참고용)" : "")}");
116+
117+
return options;
118+
}
66119
}
67120
}

ProjectVG.Api/Middleware/WebSocketMiddleware.cs

Lines changed: 98 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
using ProjectVG.Application.Services.Session;
2-
using ProjectVG.Application.Services.WebSocket;
32
using ProjectVG.Infrastructure.Auth;
43
using ProjectVG.Infrastructure.Realtime.WebSocketConnection;
4+
using ProjectVG.Domain.Services.Server;
55
using System.Net.WebSockets;
66

77
namespace ProjectVG.Api.Middleware
@@ -10,22 +10,25 @@ public class WebSocketMiddleware
1010
{
1111
private readonly RequestDelegate _next;
1212
private readonly ILogger<WebSocketMiddleware> _logger;
13-
private readonly IWebSocketManager _webSocketService;
14-
private readonly IConnectionRegistry _connectionRegistry;
13+
private readonly ISessionManager _sessionManager;
14+
private readonly IWebSocketConnectionManager _connectionManager;
1515
private readonly IJwtProvider _jwtProvider;
16+
private readonly IServerRegistrationService? _serverRegistrationService;
1617

1718
public WebSocketMiddleware(
1819
RequestDelegate next,
1920
ILogger<WebSocketMiddleware> logger,
20-
IWebSocketManager webSocketService,
21-
IConnectionRegistry connectionRegistry,
22-
IJwtProvider jwtProvider)
21+
ISessionManager sessionManager,
22+
IWebSocketConnectionManager connectionManager,
23+
IJwtProvider jwtProvider,
24+
IServerRegistrationService? serverRegistrationService = null)
2325
{
2426
_next = next;
2527
_logger = logger;
26-
_webSocketService = webSocketService;
27-
_connectionRegistry = connectionRegistry;
28+
_sessionManager = sessionManager;
29+
_connectionManager = connectionManager;
2830
_jwtProvider = jwtProvider;
31+
_serverRegistrationService = serverRegistrationService;
2932
}
3033

3134
public async Task InvokeAsync(HttpContext context)
@@ -88,19 +91,59 @@ private string ExtractToken(HttpContext context)
8891
return string.Empty;
8992
}
9093

91-
/// <summary>
92-
/// 기존 연결 정리 후 새 연결 등록
94+
/// <summary>
95+
/// 새 아키텍처: 세션 관리와 WebSocket 연결 관리 분리
9396
/// </summary>
9497
private async Task RegisterConnection(Guid userId, WebSocket socket)
9598
{
96-
if (_connectionRegistry.TryGet(userId.ToString(), out var existing) && existing != null) {
97-
_logger.LogInformation("기존 연결 정리: {UserId}", userId);
98-
await _webSocketService.DisconnectAsync(userId.ToString());
99-
}
99+
var userIdString = userId.ToString();
100+
_logger.LogInformation("[WebSocketMiddleware] 연결 등록 시작: UserId={UserId}", userId);
101+
102+
try
103+
{
104+
// 기존 로컬 연결이 있으면 정리
105+
if (_connectionManager.HasLocalConnection(userIdString))
106+
{
107+
_logger.LogInformation("[WebSocketMiddleware] 기존 로컬 연결 발견 - 정리 중: UserId={UserId}", userId);
108+
_connectionManager.UnregisterConnection(userIdString);
109+
}
110+
111+
// 1. 세션 관리자에 세션 생성 (Redis 저장)
112+
await _sessionManager.CreateSessionAsync(userId);
113+
_logger.LogInformation("[WebSocketMiddleware] 세션 관리자에 세션 저장 완료: UserId={UserId}", userId);
114+
115+
// 2. WebSocket 연결 관리자에 로컬 연결 등록
116+
var connection = new WebSocketClientConnection(userIdString, socket);
117+
_connectionManager.RegisterConnection(userIdString, connection);
118+
_logger.LogInformation("[WebSocketMiddleware] 로컬 WebSocket 연결 등록 완료: UserId={UserId}", userId);
119+
120+
// 3. 분산 시스템: 사용자-서버 매핑 저장 (Redis)
121+
if (_serverRegistrationService != null)
122+
{
123+
try
124+
{
125+
var serverId = _serverRegistrationService.GetServerId();
126+
await _serverRegistrationService.SetUserServerAsync(userIdString, serverId);
127+
_logger.LogInformation("[WebSocketMiddleware] 사용자-서버 매핑 저장 완료: UserId={UserId}, ServerId={ServerId}", userId, serverId);
128+
}
129+
catch (Exception mapEx)
130+
{
131+
_logger.LogWarning(mapEx, "[WebSocketMiddleware] 사용자-서버 매핑 저장 실패: UserId={UserId}", userId);
132+
// 매핑 저장 실패는 로그만 남기고 연결은 계속 진행
133+
}
134+
}
100135

101-
var connection = new WebSocketClientConnection(userId.ToString(), socket);
102-
_connectionRegistry.Register(userId.ToString(), connection);
103-
await _webSocketService.ConnectAsync(userId.ToString());
136+
// [디버그] 등록 후 상태 확인
137+
var isSessionActive = await _sessionManager.IsSessionActiveAsync(userId);
138+
var hasLocalConnection = _connectionManager.HasLocalConnection(userIdString);
139+
_logger.LogInformation("[WebSocketMiddleware] 연결 등록 완료: UserId={UserId}, SessionActive={SessionActive}, LocalConnection={LocalConnection}",
140+
userId, isSessionActive, hasLocalConnection);
141+
}
142+
catch (Exception ex)
143+
{
144+
_logger.LogError(ex, "[WebSocketMiddleware] 연결 등록 실패: UserId={UserId}", userId);
145+
throw;
146+
}
104147
}
105148

106149
/// <summary>
@@ -160,6 +203,17 @@ await socket.SendAsync(
160203
WebSocketMessageType.Text,
161204
true,
162205
cancellationTokenSource.Token);
206+
207+
// 세션 하트비트 업데이트 (Redis TTL 갱신)
208+
try {
209+
if (Guid.TryParse(userId, out var userGuid))
210+
{
211+
await _sessionManager.UpdateSessionHeartbeatAsync(userGuid);
212+
}
213+
}
214+
catch (Exception heartbeatEx) {
215+
_logger.LogWarning(heartbeatEx, "세션 하트비트 업데이트 실패: {UserId}", userId);
216+
}
163217
}
164218
}
165219
}
@@ -177,14 +231,39 @@ await socket.SendAsync(
177231
_logger.LogInformation("WebSocket 연결 해제: {UserId}", userId);
178232

179233
try {
180-
await _webSocketService.DisconnectAsync(userId);
181-
_connectionRegistry.Unregister(userId);
234+
// 새 아키텍처: 세션과 로컬 연결 분리해서 정리
235+
if (Guid.TryParse(userId, out var userGuid))
236+
{
237+
// 1. 세션 관리자에서 세션 삭제 (Redis에서 제거)
238+
await _sessionManager.DeleteSessionAsync(userGuid);
239+
_logger.LogDebug("세션 관리자에서 세션 삭제 완료: {UserId}", userId);
240+
}
241+
242+
// 2. 분산 시스템: 사용자-서버 매핑 제거 (Redis)
243+
if (_serverRegistrationService != null)
244+
{
245+
try
246+
{
247+
await _serverRegistrationService.RemoveUserServerAsync(userId);
248+
_logger.LogDebug("사용자-서버 매핑 제거 완료: {UserId}", userId);
249+
}
250+
catch (Exception mapEx)
251+
{
252+
_logger.LogWarning(mapEx, "사용자-서버 매핑 제거 실패: {UserId}", userId);
253+
}
254+
}
255+
256+
// 3. 로컬 WebSocket 연결 해제
257+
_connectionManager.UnregisterConnection(userId);
258+
_logger.LogDebug("로컬 WebSocket 연결 해제 완료: {UserId}", userId);
182259

260+
// 4. WebSocket 소켓 정리
183261
if (socket.State == WebSocketState.Open || socket.State == WebSocketState.CloseReceived) {
184262
await socket.CloseAsync(
185263
WebSocketCloseStatus.NormalClosure,
186264
"Connection closed",
187265
CancellationToken.None);
266+
_logger.LogDebug("WebSocket 소켓 정리 완료: {UserId}", userId);
188267
}
189268
}
190269
catch (Exception ex) {

ProjectVG.Api/Program.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
}
3939

4040
builder.Services.AddInfrastructureServices(builder.Configuration);
41-
builder.Services.AddApplicationServices();
41+
builder.Services.AddApplicationServices(builder.Configuration);
4242
builder.Services.AddDevelopmentCors();
4343

4444
// 부하테스트 환경에서 성능 모니터링 서비스 추가

ProjectVG.Api/appsettings.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,30 @@
1010
"JWT": {
1111
"Issuer": "ProjectVG",
1212
"Audience": "ProjectVG"
13+
},
14+
"DistributedSystem": {
15+
"Enabled": true,
16+
"ServerId": "api-server-001",
17+
"HeartbeatIntervalSeconds": 30,
18+
"CleanupIntervalMinutes": 5,
19+
"ServerTimeoutMinutes": 2
20+
},
21+
"LLM": {
22+
"BaseUrl": "http://localhost:7930"
23+
},
24+
"MEMORY": {
25+
"BaseUrl": "http://localhost:7940"
26+
},
27+
"TTS": {
28+
"BaseUrl": "https://supertoneapi.com"
29+
},
30+
"WebSocket": {
31+
"KeepAliveIntervalMinutes": 10,
32+
"ReceiveBufferSize": 4096,
33+
"SendBufferSize": 4096
34+
},
35+
"ConnectionStrings": {
36+
"DefaultConnection": "Server=localhost,1433;Database=ProjectVG;User Id=sa;Password=ProjectVG123!;TrustServerCertificate=true;MultipleActiveResultSets=true",
37+
"Redis": "projectvg-redis:6379"
1338
}
1439
}

0 commit comments

Comments
 (0)