chore: integrate MinIO S3-compatible storage for local development, including bucket auto-setup and app credentials

Author: MickaelCa

.docker/minio/setup.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# Simple script to set up MinIO bucket and user
+# Based on example from MinIO issues
+
+# Format bucket name to ensure compatibility
+BUCKET_NAME=$(echo "${S3_BUCKET_NAME}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
+
+# Configure MinIO client
+mc alias set myminio http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}
+
+# Remove bucket if it exists (for clean setup)
+mc rm -r --force myminio/${BUCKET_NAME} || true
+
+# Create bucket
+mc mb myminio/${BUCKET_NAME}
+
+# Set bucket policy to allow downloads
+mc anonymous set download myminio/${BUCKET_NAME}
+
+# Create user with access and secret keys
+mc admin user add myminio ${S3_ACCESS_KEY} ${S3_SECRET_KEY} || echo "User already exists"
+
+# Create policy for the bucket
+echo '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::'${BUCKET_NAME}'/*","arn:aws:s3:::'${BUCKET_NAME}'"]}]}' > /tmp/policy.json
+
+# Apply policy
+mc admin policy create myminio gitingest-policy /tmp/policy.json || echo "Policy already exists"
+mc admin policy attach myminio gitingest-policy --user ${S3_ACCESS_KEY}
+
+echo "MinIO setup completed successfully"
+echo "Bucket: ${BUCKET_NAME}"
+echo "Access via console: http://localhost:9001"

.env.example

@@ -33,3 +33,22 @@ GITINGEST_SENTRY_PROFILE_LIFECYCLE=trace
 GITINGEST_SENTRY_SEND_DEFAULT_PII=true
 # Environment name for Sentry (default: "")
 GITINGEST_SENTRY_ENVIRONMENT=development
+
+# MinIO Configuration (for development)
+# Root user credentials for MinIO admin access
+MINIO_ROOT_USER=minioadmin
+MINIO_ROOT_PASSWORD=minioadmin
+
+# S3 Configuration (for application)
+# Endpoint URL for the S3 service (MinIO in development)
+S3_ENDPOINT=http://minio:9000
+# Access key for the S3 bucket (created automatically in development)
+S3_ACCESS_KEY=gitingest
+# Secret key for the S3 bucket (created automatically in development)
+S3_SECRET_KEY=gitingest123
+# Name of the S3 bucket (created automatically in development)
+S3_BUCKET_NAME=gitingest-bucket
+# Region for the S3 bucket (default for MinIO)
+S3_REGION=us-east-1
+# Public URL/CDN for accessing S3 resources
+S3_ALIAS_HOST=127.0.0.1:9000/gitingest-bucket

README.md

@@ -233,6 +233,7 @@ The application can be configured using the following environment variables:
 - **GITINGEST_SENTRY_PROFILE_SESSION_SAMPLE_RATE**: Sampling rate for profile sessions (default: "1.0", range: 0.0-1.0)
 - **GITINGEST_SENTRY_PROFILE_LIFECYCLE**: Profile lifecycle mode (default: "trace")
 - **GITINGEST_SENTRY_SEND_DEFAULT_PII**: Send default personally identifiable information (default: "true")
+- **S3_ALIAS_HOST**: Public URL/CDN for accessing S3 resources (default: "127.0.0.1:9000/gitingest-bucket")
 
 ### Using Docker Compose
 
@@ -256,7 +257,7 @@ x-app-base: &app-base
 
 #### Services
 
-The file defines two services:
+The file defines three services:
 
 1. **app**: Production service configuration
    - Uses the `prod` profile
@@ -269,6 +270,31 @@ The file defines two services:
    - Mounts the source code for live development
    - Uses hot reloading for faster development
 
+3. **minio**: S3-compatible object storage for development
+   - Uses the `dev` profile (only available in development mode)
+   - Provides S3-compatible storage for local development
+   - Accessible via:
+     - API: Port 9000 (http://localhost:9000)
+     - Web Console: Port 9001 (http://localhost:9001)
+   - Default admin credentials:
+     - Username: `minioadmin`
+     - Password: `minioadmin`
+   - Configurable via environment variables:
+     - `MINIO_ROOT_USER`: Custom admin username (default: minioadmin)
+     - `MINIO_ROOT_PASSWORD`: Custom admin password (default: minioadmin)
+   - Includes persistent storage via Docker volume
+   - Auto-creates a bucket and application-specific credentials:
+     - Bucket name: `gitingest-bucket` (configurable via `S3_BUCKET_NAME`)
+     - Access key: `gitingest` (configurable via `S3_ACCESS_KEY`)
+     - Secret key: `gitingest123` (configurable via `S3_SECRET_KEY`)
+   - These credentials are automatically passed to the app-dev service via environment variables:
+     - `S3_ENDPOINT`: URL of the MinIO server
+     - `S3_ACCESS_KEY`: Access key for the S3 bucket
+     - `S3_SECRET_KEY`: Secret key for the S3 bucket
+     - `S3_BUCKET_NAME`: Name of the S3 bucket
+     - `S3_REGION`: Region for the S3 bucket (default: us-east-1)
+     - `S3_ALIAS_HOST`: Public URL/CDN for accessing S3 resources (default: "127.0.0.1:9000/gitingest-bucket")
+
 #### Usage Examples
 
 To run the application in development mode:

compose.yml

@@ -45,8 +45,65 @@ services:
     environment:
       - DEBUG=true
       - GITINGEST_SENTRY_ENVIRONMENT=${GITINGEST_SENTRY_ENVIRONMENT:-development}
+      # S3 Configuration
+      - S3_ENDPOINT=http://minio:9000
+      - S3_ACCESS_KEY=${S3_ACCESS_KEY:-gitingest}
+      - S3_SECRET_KEY=${S3_SECRET_KEY:-gitingest123}
+      # Use lowercase bucket name to ensure compatibility with MinIO
+      - S3_BUCKET_NAME=${S3_BUCKET_NAME:-gitingest-bucket}
+      - S3_REGION=${S3_REGION:-us-east-1}
+      # Public URL for S3 resources
+      - S3_ALIAS_HOST=${S3_ALIAS_HOST:-127.0.0.1:9000/${S3_BUCKET_NAME:-gitingest-bucket}}
     volumes:
       # Mount source code for live development
       - ./src:/app:ro
     # Use --reload flag for hot reloading during development
     command: ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
+    depends_on:
+      minio-setup:
+        condition: service_completed_successfully
+
+  # MinIO S3-compatible object storage for development
+  minio:
+    image: minio/minio:latest
+    profiles:
+      - dev
+    ports:
+      - "9000:9000"  # API port
+      - "9001:9001"  # Console port
+    environment:
+      - MINIO_ROOT_USER=${MINIO_ROOT_USER:-minioadmin}
+      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-minioadmin}
+    volumes:
+      - minio-data:/data
+    command: server /data --console-address ":9001"
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 30s
+      start_period: 30s
+      start_interval: 1s
+
+  # MinIO setup service to create bucket and user
+  minio-setup:
+    image: minio/mc
+    profiles:
+      - dev
+    depends_on:
+      minio:
+        condition: service_healthy
+    environment:
+      - MINIO_ROOT_USER=${MINIO_ROOT_USER:-minioadmin}
+      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-minioadmin}
+      - S3_ACCESS_KEY=${S3_ACCESS_KEY:-gitingest}
+      - S3_SECRET_KEY=${S3_SECRET_KEY:-gitingest123}
+      - S3_BUCKET_NAME=${S3_BUCKET_NAME:-gitingest-bucket}
+    volumes:
+      - ./.docker/minio/setup.sh:/setup.sh:ro
+    entrypoint: sh
+    command: -c /setup.sh
+
+volumes:
+  minio-data:
+    driver: local