feat: add centralized JSON logging and integrate into S3 utilities

- Implement `JSONFormatter` and methods for structured logging.
- Integrate logging into S3 client creation, uploads, and URL lookups.
- Enhance logging with extra fields for better traceability.

Author: MickaelCa

src/gitingest/utils/logging_config.py

@@ -0,0 +1,111 @@
+"""Centralized logging configuration for JSON logging in k8s environments."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+
+
+class JSONFormatter(logging.Formatter):
+    """Custom JSON formatter for structured logging."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format log record as JSON."""
+        log_entry = {
+            "timestamp": self.formatTime(record, self.datefmt),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "module": record.module,
+            "function": record.funcName,
+            "line": record.lineno,
+        }
+
+        # Add exception info if present
+        if record.exc_info:
+            log_entry["exception"] = self.formatException(record.exc_info)
+
+        # Add extra fields if present
+        if hasattr(record, "extra_fields"):
+            log_entry.update(record.extra_fields)
+
+        return json.dumps(log_entry)
+
+
+def configure_json_logging(level: str = "INFO") -> None:
+    """Configure JSON logging for the application.
+
+    Parameters
+    ----------
+    level : str
+        Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+
+    """
+    # Convert string level to logging constant
+    numeric_level = getattr(logging, level.upper(), logging.INFO)
+
+    # Create JSON formatter
+    formatter = JSONFormatter(datefmt="%Y-%m-%dT%H:%M:%S")
+
+    # Configure root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(numeric_level)
+
+    # Remove existing handlers to avoid duplicates
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+
+    # Create console handler for stdout
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(numeric_level)
+    console_handler.setFormatter(formatter)
+
+    # Add handler to root logger
+    root_logger.addHandler(console_handler)
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Get a logger instance with the given name.
+
+    Parameters
+    ----------
+    name : str
+        Logger name (typically __name__)
+
+    Returns
+    -------
+    logging.Logger
+        Configured logger instance
+
+    """
+    return logging.getLogger(name)
+
+
+def log_with_extra(logger: logging.Logger, level: str, message: str, **extra_fields: str | int | bool | None) -> None:
+    """Log a message with extra fields.
+
+    Parameters
+    ----------
+    logger : logging.Logger
+        Logger instance
+    level : str
+        Log level (debug, info, warning, error, critical)
+    message : str
+        Log message
+    **extra_fields : str | int | bool | None
+        Additional fields to include in the log entry
+
+    """
+    # Create a LogRecord with extra fields
+    record = logger.makeRecord(
+        logger.name,
+        getattr(logging, level.upper()),
+        "",
+        0,
+        message,
+        (),
+        None,
+    )
+    record.extra_fields = extra_fields
+    logger.handle(record)

src/gitingest/utils/s3_utils.py

@@ -10,6 +10,11 @@
 from boto3 import client as boto_client
 from botocore.exceptions import ClientError
 
+from gitingest.utils.logging_config import get_logger, log_with_extra
+
+# Initialize logger for this module
+logger = get_logger(__name__)
+
 
 class S3UploadError(Exception):
     """Custom exception for S3 upload failures."""
@@ -131,6 +136,17 @@ def generate_s3_file_path(
 def create_s3_client() -> boto_client:  # type: ignore[name-defined]
     """Create and return an S3 client with configuration from environment."""
     config = get_s3_config()
+
+    # Log S3 client creation with configuration details (excluding sensitive info)
+    log_config = {k: v for k, v in config.items() if k not in ["aws_access_key_id", "aws_secret_access_key"]}
+    log_with_extra(
+        logger,
+        "debug",
+        "Creating S3 client",
+        s3_config=log_config,
+        has_credentials=bool(config.get("aws_access_key_id")),
+    )
+
     return boto_client("s3", **config)
 
 
@@ -166,10 +182,21 @@ def upload_to_s3(content: str, s3_file_path: str, ingest_id: UUID) -> str:
         msg = "S3 is not enabled"
         raise ValueError(msg)
 
-    try:
-        s3_client = create_s3_client()
-        bucket_name = get_s3_bucket_name()
+    s3_client = create_s3_client()
+    bucket_name = get_s3_bucket_name()
+
+    # Log upload attempt
+    log_with_extra(
+        logger,
+        "debug",
+        "Starting S3 upload",
+        bucket_name=bucket_name,
+        s3_file_path=s3_file_path,
+        ingest_id=str(ingest_id),
+        content_size=len(content),
+    )
 
+    try:
         # Upload the content with ingest_id as tag
         s3_client.put_object(
             Bucket=bucket_name,
@@ -178,21 +205,46 @@ def upload_to_s3(content: str, s3_file_path: str, ingest_id: UUID) -> str:
             ContentType="text/plain",
             Tagging=f"ingest_id={ingest_id!s}",
         )
+    except ClientError as e:
+        # Log upload failure
+        log_with_extra(
+            logger,
+            "error",
+            "S3 upload failed",
+            bucket_name=bucket_name,
+            s3_file_path=s3_file_path,
+            ingest_id=str(ingest_id),
+            error_code=e.response.get("Error", {}).get("Code"),
+            error_message=str(e),
+        )
+        msg = f"Failed to upload to S3: {e}"
+        raise S3UploadError(msg) from e
 
-        # Generate public URL
-        alias_host = get_s3_alias_host()
-        if alias_host:
-            # Use alias host if configured
-            return f"{alias_host.rstrip('/')}/{s3_file_path}"
+    # Generate public URL
+    alias_host = get_s3_alias_host()
+    if alias_host:
+        # Use alias host if configured
+        public_url = f"{alias_host.rstrip('/')}/{s3_file_path}"
+    else:
         # Fallback to direct S3 URL
-        endpoint = get_s3_config()["endpoint_url"]
+        endpoint = get_s3_config().get("endpoint_url")
         if endpoint:
-            return f"{endpoint.rstrip('/')}/{bucket_name}/{s3_file_path}"
-        return f"https://{bucket_name}.s3.{get_s3_config()['region_name']}.amazonaws.com/{s3_file_path}"
+            public_url = f"{endpoint.rstrip('/')}/{bucket_name}/{s3_file_path}"
+        else:
+            public_url = f"https://{bucket_name}.s3.{get_s3_config()['region_name']}.amazonaws.com/{s3_file_path}"
 
-    except ClientError as e:
-        msg = f"Failed to upload to S3: {e}"
-        raise S3UploadError(msg) from e
+    # Log successful upload
+    log_with_extra(
+        logger,
+        "debug",
+        "S3 upload completed successfully",
+        bucket_name=bucket_name,
+        s3_file_path=s3_file_path,
+        ingest_id=str(ingest_id),
+        public_url=public_url,
+    )
+
+    return public_url
 
 
 def _build_s3_url(key: str) -> str:
@@ -241,8 +293,16 @@ def get_s3_url_for_ingest_id(ingest_id: UUID) -> str | None:
 
     """
     if not is_s3_enabled():
+        logger.debug("S3 not enabled, skipping URL lookup for ingest_id: %s", ingest_id)
         return None
 
+    log_with_extra(
+        logger,
+        "debug",
+        "Starting S3 URL lookup for ingest ID",
+        ingest_id=str(ingest_id),
+    )
+
     try:
         s3_client = create_s3_client()
         bucket_name = get_s3_bucket_name()
@@ -254,16 +314,43 @@ def get_s3_url_for_ingest_id(ingest_id: UUID) -> str | None:
             Prefix="ingest/",
         )
 
+        objects_checked = 0
         for page in page_iterator:
             if "Contents" not in page:
                 continue
 
             for obj in page["Contents"]:
                 key = obj["Key"]
+                objects_checked += 1
                 if _check_object_tags(s3_client, bucket_name, key, ingest_id):
-                    return _build_s3_url(key)
+                    s3_url = _build_s3_url(key)
+                    log_with_extra(
+                        logger,
+                        "debug",
+                        "Found S3 object for ingest ID",
+                        ingest_id=str(ingest_id),
+                        s3_key=key,
+                        s3_url=s3_url,
+                        objects_checked=objects_checked,
+                    )
+                    return s3_url
+
+        log_with_extra(
+            logger,
+            "debug",
+            "No S3 object found for ingest ID",
+            ingest_id=str(ingest_id),
+            objects_checked=objects_checked,
+        )
 
-    except ClientError:
-        pass
+    except ClientError as e:
+        log_with_extra(
+            logger,
+            "error",
+            "Error during S3 URL lookup",
+            ingest_id=str(ingest_id),
+            error_code=e.response.get("Error", {}).get("Code"),
+            error_message=str(e),
+        )
 
     return None