diff --git a/README.md b/README.md index c3684bb..59f7c24 100644 --- a/README.md +++ b/README.md @@ -1 +1,276 @@ -# API \ No newline at end of file +# Database Hosting Services (DBHS) API + +[![Go](https://img.shields.io/badge/Go-1.24.4-blue.svg)](https://golang.org/) +[![PostgreSQL](https://img.shields.io/badge/PostgreSQL-15+-blue.svg)](https://postgresql.org/) +[![Redis](https://img.shields.io/badge/Redis-7+-red.svg)](https://redis.io/) +[![Docker](https://img.shields.io/badge/Docker-Ready-blue.svg)](https://docker.com/) +[![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) + +A comprehensive, production-ready REST API for managing database hosting services. DBHS provides a complete solution for database-as-a-service operations, including user management, project creation, schema management, analytics, and AI-powered assistance. + +## ๐Ÿš€ Features + +### Core Functionality +- **๐Ÿ” User Authentication & Authorization**: JWT-based authentication with role-based access control +- **๐Ÿ“Š Project Management**: Create, manage, and organize database projects +- **๐Ÿ—„๏ธ Database Schema Management**: Dynamic schema creation, modification, and inspection +- **๐Ÿ“‹ Table Operations**: Full CRUD operations for database tables with real-time schema synchronization +- **๐Ÿ” SQL Editor**: Execute custom SQL queries with syntax validation and error handling +- **๐Ÿ“ˆ Analytics & Monitoring**: Real-time database usage statistics, storage monitoring, and cost analysis +- **๐Ÿค– AI Assistant**: Intelligent database assistant powered by advanced AI for query optimization and suggestions +- **โšก Caching**: Redis-based caching for improved performance +- **๐Ÿ”’ Security**: Rate limiting, input validation, and secure database connections + +### Advanced Features +- **๐Ÿ“Š Real-time Analytics**: Track database usage, query performance, and storage metrics +- **๐Ÿ”„ Background Workers**: Automated data collection and maintenance tasks +- **๐Ÿ“– API Documentation**: Auto-generated Swagger/OpenAPI documentation +- **๐Ÿณ Containerized Deployment**: Docker and Fly.io ready +- **๐Ÿ“ง Email Integration**: User verification and notification system +- **๐Ÿ›ก๏ธ Middleware Stack**: Comprehensive security and logging middleware + +## ๐Ÿ—๏ธ Architecture + +![Database Design](./public/database-design.png) + +The DBHS API follows a modular microservices-inspired architecture with the following components: + +### System Architecture +- **API Gateway Layer**: Gorilla Mux router with comprehensive middleware +- **Service Layer**: Business logic organized by domain (accounts, projects, tables, etc.) +- **Data Access Layer**: PostgreSQL with connection pooling and Redis caching +- **Background Processing**: Cron-based workers for analytics and maintenance +- **External Integrations**: AI services, email notifications, and monitoring + +### Database Strategy +- **Metadata Database**: Stores user accounts, project configurations, and system metadata +- **Dynamic User Databases**: Isolated databases created per user project for data isolation +- **Connection Pooling**: Efficient database connection management with pgxpool +- **Multi-tenancy**: Secure isolation between user projects and data + +## ๐Ÿ“ Project Structure + +``` +โ”œโ”€โ”€ main/ # Application entry point and routing +โ”œโ”€โ”€ config/ # Configuration management and database connections +โ”œโ”€โ”€ accounts/ # User authentication and profile management +โ”œโ”€โ”€ projects/ # Project creation and management +โ”œโ”€โ”€ tables/ # Database table operations and schema management +โ”œโ”€โ”€ schemas/ # Database schema inspection and metadata +โ”œโ”€โ”€ SqlEditor/ # SQL query execution and validation +โ”œโ”€โ”€ AI/ # AI assistant and intelligent features +โ”œโ”€โ”€ analytics/ # Usage analytics and monitoring +โ”œโ”€โ”€ indexes/ # Database indexing operations +โ”œโ”€โ”€ middleware/ # HTTP middleware (auth, rate limiting, CORS) +โ”œโ”€โ”€ response/ # Standardized API response handling +โ”œโ”€โ”€ utils/ # Shared utilities and helpers +โ”œโ”€โ”€ workers/ # Background processing and cron jobs +โ”œโ”€โ”€ caching/ # Redis caching implementation +โ”œโ”€โ”€ test/ # Comprehensive test suites +โ”œโ”€โ”€ docs/ # API documentation (Swagger/OpenAPI) +โ”œโ”€โ”€ public/ # Static assets and diagrams +โ””โ”€โ”€ templates/ # Email and notification templates +``` + +## ๐Ÿ› ๏ธ Technology Stack + +### Backend Technologies +- **Language**: Go 1.24.4 +- **Web Framework**: Gorilla Mux +- **Database**: PostgreSQL 15+ with pgx driver +- **Caching**: Redis 7+ +- **Authentication**: JWT with golang-jwt/jwt +- **Documentation**: Swagger/OpenAPI with go-swaggo +- **Background Jobs**: Robfig Cron +- **Email**: GoMail v2 + +### AI & Analytics +- **AI Integration**: Custom AI agent for database assistance +- **Monitoring**: Real-time analytics and usage tracking +- **Performance**: Query optimization and caching strategies + +## ๐Ÿš€ Quick Start + +### Prerequisites +- Go 1.24.4 or higher +- PostgreSQL 15+ +- Redis 7+ +- Docker (optional) + +### Installation + +1. **Clone the repository** + ```bash + git clone https://github.com/your-org/database-hosting-services-api.git + cd database-hosting-services-api + ``` + +2. **Install dependencies** + ```bash + go mod download + ``` + +3. **Set up environment variables** + ```bash + cp .env.example .env + # Edit .env with your configuration + ``` + +4. **Required Environment Variables** + ```env + # Database Configuration + DATABASE_URL=postgresql://user:password@localhost:5432/dbhs_main + DATABASE_ADMIN_URL=postgresql://admin:password@localhost:5432/postgres + TEST_DATABASE_URL=postgresql://user:password@localhost:5432/dbhs_test + + # Redis Configuration + REDIS_URL=redis://localhost:6379 + + # JWT Configuration + JWT_SECRET=your-super-secret-jwt-key + + # Email Configuration + SMTP_HOST=smtp.gmail.com + SMTP_PORT=587 + SMTP_USERNAME=your-email@gmail.com + SMTP_PASSWORD=your-app-password + + # AI Configuration + AI_API_KEY=your-ai-api-key + + # Application Configuration + API_PORT=8000 + ENV=development + ``` + +5. **Run database migrations** + ```bash + # Set up main database schema + psql -d $DATABASE_URL -f scripts/migrations/001_initial_schema.sql + ``` + +6. **Build and run the application** + ```bash + make build + make run + ``` + +## ๐Ÿ“– API Documentation + +### Interactive Documentation +- **Scalar API Reference**: https://orbix.fly.dev/reference +- **Swagger UI**: https://orbix.fly.dev/swagger/index.html +- **ReDoc**: https://orbix.fly.dev/redoc + +### Generate Documentation +```bash +make generate-docs +``` + +## ๐Ÿ”— API Endpoints + +### Authentication +- `POST /api/user/sign-up` - User registration +- `POST /api/user/sign-in` - User login +- `POST /api/user/verify` - Email verification +- `POST /api/user/resend-code` - Resend verification code +- `POST /api/user/forget-password` - Password reset request +- `POST /api/user/forget-password/verify` - Password reset verification + +### User Management +- `GET /api/users/me` - Get current user profile +- `POST /api/users/update-password` - Update user password +- `PATCH /api/users/{id}` - Update user profile + +### Projects +- `GET /api/projects` - List user projects +- `POST /api/projects` - Create new project +- `GET /api/projects/{project_id}` - Get project details +- `PATCH /api/projects/{project_id}` - Update project +- `DELETE /api/projects/{project_id}` - Delete project + +### Tables & Schema +- `GET /api/projects/{project_id}/tables` - List project tables +- `POST /api/projects/{project_id}/tables` - Create new table +- `GET /api/projects/{project_id}/tables/{table_id}` - Get table data (with pagination) +- `POST /api/projects/{project_id}/tables/{table_id}` - Insert row into table +- `PUT /api/projects/{project_id}/tables/{table_id}` - Update table schema +- `DELETE /api/projects/{project_id}/tables/{table_id}` - Delete table +- `GET /api/projects/{project_id}/tables/{table_id}/schema` - Get table schema + +### Database Schema +- `GET /api/projects/{project_id}/schema/tables` - Get database schema +- `GET /api/projects/{project_id}/schema/tables/{table_id}` - Get specific table schema + +### Indexes +- `GET /api/projects/{project_id}/indexes` - List project indexes +- `POST /api/projects/{project_id}/indexes` - Create new index +- `GET /api/projects/{project_id}/indexes/{index_oid}` - Get specific index +- `PUT /api/projects/{project_id}/indexes/{index_oid}` - Update index name +- `DELETE /api/projects/{project_id}/indexes/{index_oid}` - Delete index + +### SQL Editor +- `POST /api/projects/{project_id}/sqlEditor/run-query` - Execute SQL query + +### Analytics +- `GET /api/projects/{project_id}/analytics/storage` - Database storage analytics +- `GET /api/projects/{project_id}/analytics/execution-time` - Query execution time statistics +- `GET /api/projects/{project_id}/analytics/usage` - Database usage statistics + +### AI Assistant +- `GET /api/projects/{project_id}/ai/report` - Get AI-generated database report +- `POST /api/projects/{project_id}/ai/chatbot/ask` - Chat with AI assistant +- `POST /api/projects/{project_id}/ai/agent` - AI agent for database operations +- `POST /api/projects/{project_id}/ai/agent/accept` - Accept AI agent suggestions +- `POST /api/projects/{project_id}/ai/agent/cancel` - Cancel AI agent operations + + +## ๐Ÿ”ง Development + +## ๐Ÿงช Testing + +### Run Tests +```bash +# Run all tests +make test + +# Run specific test suite +go test ./test/accounts_test.go -v +go test ./test/projects_test.go -v +go test ./test/tables/ -v + +# Run tests with coverage +go test -cover ./... +``` + +### Test Structure +- **Unit Tests**: Individual component testing +- **Integration Tests**: End-to-end API testing +- **Service Tests**: Business logic validation +- **Database Tests**: Data access layer testing + +## ๐Ÿ“Š Monitoring & Analytics + +### Built-in Analytics +- **Database Usage**: Query counts, execution time, resource usage +- **Storage Metrics**: Database size, growth trends, optimization suggestions +- **Performance Monitoring**: Query performance, connection pooling metrics +- **User Analytics**: Usage patterns, feature adoption + +### External Integrations +- **Axiom**: Advanced logging and analytics +- **Health Checks**: Application and database health monitoring +- **Alerting**: Automated alerts for critical issues + +## ๐Ÿ”’ Security Features + +### Authentication & Authorization +- **JWT Tokens**: Secure, stateless authentication +- **Role-Based Access**: Fine-grained permission control +- **Project Isolation**: Secure multi-tenancy + +### Security Measures +- **Rate Limiting**: API abuse prevention +- **Input Validation**: SQL injection and XSS protection +- **CORS Configuration**: Cross-origin request security +- **Database Security**: Connection encryption and access control \ No newline at end of file diff --git a/analytics/handlers.go b/analytics/handlers.go index ddeded6..f184abf 100644 --- a/analytics/handlers.go +++ b/analytics/handlers.go @@ -5,6 +5,7 @@ import ( "DBHS/response" "DBHS/utils" "net/http" + "strings" "github.com/gorilla/mux" ) @@ -32,15 +33,17 @@ func CurrentStorage(app *config.Application) http.HandlerFunc { } storage, apiErr := GetALLDatabaseStorage(r.Context(), config.DB, projectOid) - if apiErr.Error() != nil { - utils.ResponseHandler(w, r, apiErr) + if len(storage) == 0 || (apiErr.Error() != nil && strings.Contains(apiErr.Error().Error(), "cannot scan NULL into")) { + response.NotFound(w, r, "No storage information found for the project", nil) return } - if len(storage) == 0 { - response.NotFound(w, r, "No storage information found for the project", nil) + + if apiErr.Error() != nil { + utils.ResponseHandler(w, r, apiErr) return } - response.OK(w, r, "Storage history retrieved successfully", StorageResponse) + + response.OK(w, r, "Storage history retrieved successfully", storage) } } @@ -66,13 +69,18 @@ func ExecutionTime(app *config.Application) http.HandlerFunc { return } - _, apiErr := GetALLExecutionTimeStats(r.Context(), config.DB, projectOid) + stats, apiErr := GetALLExecutionTimeStats(r.Context(), config.DB, projectOid) + if len(stats) == 0 || (apiErr.Error() != nil && strings.Contains(apiErr.Error().Error(), "cannot scan NULL into")) { + response.NotFound(w, r, "No execution time records found for the project", nil) + return + } + if apiErr.Error() != nil { utils.ResponseHandler(w, r, apiErr) return } - response.OK(w, r, "Execution time statistics retrieved successfully", DatabaseActivityResponse) + response.OK(w, r, "Execution time statistics retrieved successfully", stats) } } @@ -98,12 +106,17 @@ func DatabaseUsage(app *config.Application) http.HandlerFunc { return } - _, apiErr := GetALLDatabaseUsageStats(r.Context(), config.DB, projectOid) + stats, apiErr := GetALLDatabaseUsageStats(r.Context(), config.DB, projectOid) + if len(stats) == 0 || (apiErr.Error() != nil && strings.Contains(apiErr.Error().Error(), "cannot scan NULL into")) { + response.NotFound(w, r, "No database usage records found for the project", nil) + return + } + if apiErr.Error() != nil { utils.ResponseHandler(w, r, apiErr) return } - response.OK(w, r, "Database usage statistics retrieved successfully", DatabaseUsageStatsResponse) + response.OK(w, r, "Database usage statistics retrieved successfully", stats) } } diff --git a/analytics/models.go b/analytics/models.go index 7c021b2..0159b73 100644 --- a/analytics/models.go +++ b/analytics/models.go @@ -42,178 +42,4 @@ type DatabaseUsageCostWithDates struct { ReadWriteCost float64 `json:"read_write_cost"` CPUCost float64 `json:"cpu_cost"` TotalCost float64 `json:"total_cost"` -} - -var ( - // Storage dummy data - showing growth over time (all in KB, under 100) - StorageResponse = []StorageWithDates{ - { - Timestamp: "2025-01-01T00:00:00Z", - ManagementStorage: "12 kB", - ActualData: "45 kB", - }, - { - Timestamp: "2025-01-02T00:00:00Z", - ManagementStorage: "14 kB", - ActualData: "52 kB", - }, - { - Timestamp: "2025-01-03T00:00:00Z", - ManagementStorage: "16 kB", - ActualData: "58 kB", - }, - { - Timestamp: "2025-01-04T00:00:00Z", - ManagementStorage: "18 kB", - ActualData: "64 kB", - }, - { - Timestamp: "2025-01-05T00:00:00Z", - ManagementStorage: "21 kB", - ActualData: "71 kB", - }, - { - Timestamp: "2025-01-06T00:00:00Z", - ManagementStorage: "24 kB", - ActualData: "77 kB", - }, - { - Timestamp: "2025-01-07T00:00:00Z", - ManagementStorage: "27 kB", - ActualData: "84 kB", - }, - { - Timestamp: "2025-01-08T00:00:00Z", - ManagementStorage: "30 kB", - ActualData: "89 kB", - }, - { - Timestamp: "2025-01-09T00:00:00Z", - ManagementStorage: "33 kB", - ActualData: "95 kB", - }, - { - Timestamp: "2025-01-10T00:00:00Z", - ManagementStorage: "36 kB", - ActualData: "99 kB", - }, - } - - // Database Activity dummy data - showing realistic patterns with peaks and valleys (under 100) - DatabaseActivityResponse = []DatabaseActivityWithDates{ - { - Timestamp: "2025-01-01T00:00:00Z", - TotalTimeMs: 12.45, - TotalQueries: 15, - }, - { - Timestamp: "2025-01-02T00:00:00Z", - TotalTimeMs: 17.89, - TotalQueries: 21, - }, - { - Timestamp: "2025-01-03T00:00:00Z", - TotalTimeMs: 21.34, - TotalQueries: 28, - }, - { - Timestamp: "2025-01-04T00:00:00Z", - TotalTimeMs: 18.76, - TotalQueries: 24, - }, - { - Timestamp: "2025-01-05T00:00:00Z", - TotalTimeMs: 25.67, - TotalQueries: 32, - }, - { - Timestamp: "2025-01-06T00:00:00Z", - TotalTimeMs: 22.34, - TotalQueries: 29, - }, - { - Timestamp: "2025-01-07T00:00:00Z", - TotalTimeMs: 30.45, - TotalQueries: 37, - }, - { - Timestamp: "2025-01-08T00:00:00Z", - TotalTimeMs: 27.89, - TotalQueries: 34, - }, - { - Timestamp: "2025-01-09T00:00:00Z", - TotalTimeMs: 24.56, - TotalQueries: 31, - }, - { - Timestamp: "2025-01-10T00:00:00Z", - TotalTimeMs: 29.98, - TotalQueries: 36, - }, - } - - // Database Usage Stats dummy data - showing cost variations (under 100) - DatabaseUsageStatsResponse = []DatabaseUsageCostWithDates{ - { - ReadWriteCost: 25.45, - CPUCost: 17.23, - TotalCost: 42.68, - Timestamp: "2025-01-01T00:00:00Z", - }, - { - ReadWriteCost: 32.78, - CPUCost: 19.34, - TotalCost: 52.12, - Timestamp: "2025-01-02T00:00:00Z", - }, - { - ReadWriteCost: 38.56, - CPUCost: 22.67, - TotalCost: 61.23, - Timestamp: "2025-01-03T00:00:00Z", - }, - { - ReadWriteCost: 36.89, - CPUCost: 25.45, - TotalCost: 62.34, - Timestamp: "2025-01-04T00:00:00Z", - }, - { - ReadWriteCost: 44.67, - CPUCost: 34.23, - TotalCost: 78.90, - Timestamp: "2025-01-05T00:00:00Z", - }, - { - ReadWriteCost: 41.45, - CPUCost: 28.78, - TotalCost: 70.23, - Timestamp: "2025-01-06T00:00:00Z", - }, - { - ReadWriteCost: 47.34, - CPUCost: 36.89, - TotalCost: 84.23, - Timestamp: "2025-01-07T00:00:00Z", - }, - { - ReadWriteCost: 46.78, - CPUCost: 33.56, - TotalCost: 80.34, - Timestamp: "2025-01-08T00:00:00Z", - }, - { - ReadWriteCost: 43.45, - CPUCost: 35.67, - TotalCost: 79.12, - Timestamp: "2025-01-09T00:00:00Z", - }, - { - ReadWriteCost: 48.67, - CPUCost: 37.89, - TotalCost: 86.56, - Timestamp: "2025-01-10T00:00:00Z", - }, - } -) +} \ No newline at end of file diff --git a/analytics/utils.go b/analytics/utils.go index aa39947..0cb4f9b 100644 --- a/analytics/utils.go +++ b/analytics/utils.go @@ -12,8 +12,8 @@ import ( // CalculateCosts calculates the costs associated with database usage based on read/write queries and CPU time. func (d *DatabaseUsageStats) CalculateCosts() DatabaseUsageCost { Cost := DatabaseUsageCost{ - ReadWriteCost: float64(d.ReadQueries)/1_000_000*1.00 + float64(d.WriteQueries)/1_000_000*1.50, - CPUCost: (d.TotalCPUTimeMs / 1000 / 3600) * 0.000463, + ReadWriteCost: float64(d.ReadQueries)*1.00 + float64(d.WriteQueries)*1.50, + CPUCost: d.TotalCPUTimeMs * 1.50, } Cost.TotalCost = Cost.ReadWriteCost + Cost.CPUCost return Cost diff --git a/public/database-design.png b/public/database-design.png new file mode 100644 index 0000000..2dc5c87 Binary files /dev/null and b/public/database-design.png differ