33 configurable s3 bucket for multi deployments

deployment.yml

@@ -2,3 +2,7 @@ test:
   NAME: Climsoft Test
   DATABASE_URI: "mysql+mysqldb://root:password@mariadb/climsoft"
   AUTH_ENABLED: false
+  S3_BUCKET: climsoft-paper-archive
+  AWS_REGION: eu-west-2
+  AWS_ACCESS_KEY_ID: ACCESS_KEY_ID
+  AWS_SECRET_ACCESS_KEY: SECRET_ACCESS_KEY

src/climsoft_api/api/s3_files/router.py

@@ -1,18 +1,24 @@
 from climsoft_api.config import settings
 from climsoft_api.utils.s3 import get_s3_client
-from fastapi import APIRouter
+from fastapi import APIRouter, Request
 from fastapi.responses import Response
 from climsoft_api.utils.exception import handle_exceptions
+from climsoft_api.utils.deployment import override_settings
 
 router = APIRouter()
 
 
 @router.get("/s3/image/{object_key}")
 @handle_exceptions
-def get_s3_object(object_key):
-    s3_client = get_s3_client()
+def get_s3_object(object_key, request: Request):
+    try:
+        _settings = override_settings(request.state.settings_override)
+    except AttributeError:
+        _settings = settings
+
+    s3_client = get_s3_client(_settings)
     response = s3_client.get_object(
-        Bucket=settings.S3_BUCKET,
+        Bucket=_settings.S3_BUCKET,
         Key=object_key
     )
     return Response(response["Body"].read(),

src/climsoft_api/api/upload/router.py

@@ -1,14 +1,15 @@
+import json
 import logging
 from climsoft_api.api.upload.schema import (
     FileUploadedToDiskResponse,
     FileUploadedToS3Response
 )
-from climsoft_api.config import settings
 from climsoft_api.services import file_upload_service
 from climsoft_api.utils.response import get_success_response, get_error_response
-from fastapi import APIRouter, UploadFile, File
+from fastapi import APIRouter, UploadFile, File, Request
 from climsoft_api.utils.response import translate_schema
-
+from climsoft_api.utils.deployment import override_settings
+from climsoft_api.config import settings
 
 router = APIRouter()
 
@@ -19,15 +20,19 @@
 @router.post(
     "/file-upload/image"
 )
-async def upload_image(file: UploadFile = File(...)):
+async def upload_image(request: Request, file: UploadFile = File(...)):
+    try:
+        _settings = override_settings(request.state.settings_override)
+    except AttributeError:
+        _settings = settings
     try:
         contents = await file.read()
         file_type = file.content_type
         if not file_type.startswith("image"):
             raise TypeError(_("Only image files are supported."))
 
         filepath = file_upload_service.save_file(
-            settings.FILE_STORAGE,
+            _settings,
             contents,
             file_type
         )

src/climsoft_api/main.py

@@ -39,6 +39,7 @@ def get_app(config=None):
     async def db_session_middleware(request: Request, call_next):
         try:
             request.state.get_session = get_session_local(config)
+            request.state.settings_override = deployment_configs.get(config)
             response = await call_next(request)
         except Exception as exc:
             logging.exception(exc)

src/climsoft_api/services/file_upload_service.py

@@ -5,18 +5,18 @@
 from climsoft_api.utils.s3 import get_s3_client
 
 
-def save_file(storage, file, file_type):
+def save_file(settings, file, file_type):
     file_name = f"{uuid.uuid4().hex}.{file_type.split('/')[-1]}"
-    if storage == "disk":
+    if settings.storage == "disk":
         return save_file_to_disk(file, file_name)
-    elif storage == "s3":
-        return save_file_to_s3(file, file_name)
+    elif settings.storage == "s3":
+        return save_file_to_s3(settings, file, file_name)
     else:
         raise NotImplemented()
 
 
-def save_file_to_s3(file, file_name):
-    s3_client = get_s3_client()
+def save_file_to_s3(settings, file, file_name):
+    s3_client = get_s3_client(settings)
     s3_client.upload_fileobj(
         io.BytesIO(file),
         settings.S3_BUCKET,

src/climsoft_api/utils/deployment.py

@@ -1,6 +1,10 @@
+import copy
+
 import yaml
 from pathlib import Path
 from typing import Dict
+from pydantic import BaseSettings
+from climsoft_api.config import settings, Settings
 
 deployment_config_file = Path.resolve(Path("./deployment.yml"))
 
@@ -12,3 +16,12 @@ def load_deployment_configs() -> Dict[str, Dict[str, str]]:
         with open(deployment_config_file, "r") as stream:
             deployment_configs = yaml.safe_load(stream=stream)
     return deployment_configs
+
+
+def override_settings(overrides: Dict[str, str]) -> Settings:
+    if overrides.get("NAME"):
+        overrides.pop("NAME")
+    settings_copy = copy.deepcopy(settings)
+    for k, v in overrides.items():
+        setattr(settings_copy, k, v)
+    return settings_copy

src/climsoft_api/utils/s3.py

@@ -1,15 +1,13 @@
 import boto3
-from climsoft_api.config import settings
 
 
-def get_s3_client():
+def get_s3_client(settings):
     s3_client = boto3.client(
         's3',
         aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
         aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
         region_name=settings.AWS_REGION
     )
-
     return s3_client