v0.238.024 - Development into Staging

.devcontainer/Dockerfile

@@ -0,0 +1,3 @@
+FROM mcr.microsoft.com/devcontainers/base:ubuntu
+# Install the xz-utils package
+RUN apt-get update && apt-get install -y xz-utils

.devcontainer/devcontainer.json

@@ -3,7 +3,14 @@
 {
 	"name": "Python 3",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/python:0-3.11",
+	//"image": "mcr.microsoft.com/devcontainers/python:0-3.11",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"args": {
+			"VARIANT": "3.11"
+		}
+	},
+	// Features to add to the dev container. More info: https://containers.dev/features.
 	"features": {
 		"ghcr.io/devcontainers/features/docker-in-docker:2": {
 			"version": "latest"
@@ -14,18 +21,16 @@
 			"installBicep": true,
 			"installUsingPython": true,
 			"version": "2.72.0",
-			"bicepVersion": "latest"			
+			"bicepVersion": "latest"
 		},
 		"ghcr.io/devcontainers/features/terraform:1": {},
 		"ghcr.io/devcontainers/features/powershell:1": {},
 		"ghcr.io/azure/azure-dev/azd:latest": {}
 	},
-	// Features to add to the dev container. More info: https://containers.dev/features.
-	// "features": {},
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pip3 install --user -r application/single_app/requirements.txt",
+	"postCreateCommand": "python3 -m venv .venv && .venv/bin/pip install -r application/single_app/requirements.txt",
 	// Configure tool-specific properties.
 	"customizations": {
 		"vscode": {
@@ -36,7 +41,10 @@
 				"ms-vscode.azurecli",
 				"HashiCorp.terraform",
 				"ms-vscode.powershell"
-			]
+			],
+			"settings": {
+				"python.defaultInterpreterPath": "${containerWorkspaceFolder}/.venv/bin/python"
+			}
 		}
 	}
 	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.

.gitignore

@@ -42,3 +42,6 @@ flask_session
 **/my_chart.png
 **/sample_pie.csv
 **/sample_stacked_column.csv
+tmp**cwd
+tmp_images
+nul

CLAUDE.md

@@ -0,0 +1,160 @@
+# SimpleChat — Project Instructions
+
+SimpleChat is a Flask web application using Azure Cosmos DB, Azure AI Search, and Azure OpenAI. It supports personal, group, and public workspaces for document management and AI-powered chat.
+
+## Code Style — Python
+
+- Start every file with a filename comment: `# filename.py`
+- Place imports at the top, after the module docstring (exceptions must be documented)
+- Use 4-space indentation, never tabs
+- Use `log_event` from `functions_appinsights.py` for logging instead of `print()`
+
+## Code Style — JavaScript
+
+- Start every file with a filename comment: `// filename.js`
+- Group imports at the top of the file (exceptions must be documented)
+- Use 4-space indentation, never tabs
+- Use camelCase for variables and functions: `myVariable`, `getUserData()`
+- Use PascalCase for classes: `MyClass`
+- Never use `display:none` in JavaScript; use Bootstrap's `d-none` class instead
+- Use Bootstrap alert classes for notifications, not `alert()` calls
+
+## Route Decorators — Swagger Security
+
+**Every Flask route MUST include the `@swagger_route(security=get_auth_security())` decorator.**
+
+- Import `swagger_route` and `get_auth_security` from `swagger_wrapper`
+- Place `@swagger_route(security=get_auth_security())` immediately after the `@app.route(...)` decorator and before any authentication decorators (`@login_required`, `@user_required`, etc.)
+- This applies to all new and existing routes — no exceptions
+
+Correct pattern:
+```python
+from swagger_wrapper import swagger_route, get_auth_security
+
+@app.route("/api/example", methods=["GET"])
+@swagger_route(security=get_auth_security())
+@login_required
+@user_required
+def example_route():
+    ...
+```
+
+## Security — Settings Sanitization
+
+**NEVER send raw settings or configuration data to the frontend without sanitization.**
+
+- Always use `sanitize_settings_for_user()` from `functions_settings.py` before passing settings to `render_template()` or `jsonify()`
+- **Exception**: Admin routes should NOT be sanitized (breaks admin features)
+- Sanitization strips: API keys, Cosmos DB connection strings, Azure Search admin keys, Document Intelligence keys, authentication secrets, internal endpoint URLs, database credentials, and any field containing "key", "secret", "password", or "connection"
+
+Correct pattern:
+```python
+from functions_settings import get_settings, sanitize_settings_for_user
+
+settings = get_settings()
+public_settings = sanitize_settings_for_user(settings)
+return render_template('page.html', settings=public_settings)
+```
+
+## Version Management
+
+- Version is stored in `config.py`: `VERSION = "X.XXX.XXX"`
+- When incrementing, only change the third segment (e.g., `0.238.024` -> `0.238.025`)
+- Include the current version in functional test file headers and documentation files
+
+## Documentation Locations
+
+- **Feature documentation**: `docs/explanation/features/[FEATURE_NAME].md` (uppercase with underscores)
+- **Fix documentation**: `docs/explanation/fixes/[ISSUE_NAME]_FIX.md` (uppercase with underscores)
+- **Release notes**: `docs/explanation/release_notes.md`
+
+### Feature Documentation Structure
+
+1. Header: title, overview, version, dependencies
+2. Technical specifications: architecture, APIs, configuration, file structure
+3. Usage instructions: enable/configure, workflows, examples
+4. Testing and validation: coverage, performance, limitations
+
+### Fix Documentation Structure
+
+1. Header: title, issue description, root cause, version
+2. Technical details: files modified, code changes, testing, impact
+3. Validation: test results, before/after comparison
+
+## Release Notes
+
+After completing code changes, offer to update `docs/explanation/release_notes.md`.
+
+- Add entries under the current version from `config.py`
+- If the version was bumped, create a new section at the top: `### **(vX.XXX.XXX)**`
+- Entry categories: **New Features**, **Bug Fixes**, **User Interface Enhancements**, **Breaking Changes**
+- Format each entry with a bold title, bullet-point details, and a `(Ref: ...)` line referencing relevant files/concepts
+
+## Functional Tests
+
+- **Location**: `functional_tests/`
+- **Naming**: `test_{feature_area}_{specific_test}.py` or `.js`
+- **When to create**: bug fixes, new features, API changes, database migration, UI/UX changes, authentication/security changes
+
+Every test file must include a version header:
+```python
+#!/usr/bin/env python3
+"""
+Functional test for [feature/fix name].
+Version: [current version from config.py]
+Implemented in: [version when fix/feature was added]
+
+This test ensures that [description of what is being tested].
+"""
+```
+
+Test template pattern:
+```python
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+def test_primary_functionality():
+    """Test the main functionality."""
+    print("Testing [Feature Name]...")
+    try:
+        # Setup, execute, validate, cleanup
+        print("Test passed!")
+        return True
+    except Exception as e:
+        print(f"Test failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+if __name__ == "__main__":
+    success = test_primary_functionality()
+    sys.exit(0 if success else 1)
+```
+
+
+
+## Key Project Files
+
+| File | Purpose |
+|------|---------|
+| `application/single_app/config.py` | App configuration and `VERSION` |
+| `application/single_app/functions_settings.py` | `get_settings()`, `sanitize_settings_for_user()` |
+| `application/single_app/functions_appinsights.py` | `log_event()` for logging |
+| `application/single_app/functions_documents.py` | Document CRUD, chunk operations, tag management |
+| `application/single_app/functions_group.py` | Group workspace operations |
+| `application/single_app/functions_public_workspaces.py` | Public workspace operations |
+| `application/single_app/route_backend_documents.py` | Personal document API routes |
+| `application/single_app/route_backend_group_documents.py` | Group document API routes |
+| `application/single_app/route_external_public_documents.py` | Public document API routes |
+| `application/single_app/route_backend_chats.py` | Chat API routes and AI search integration |
+
+## Frontend Architecture
+
+- Templates: `application/single_app/templates/` (Jinja2 HTML)
+- Static JS: `application/single_app/static/js/`
+  - `chat/` — Chat interface modules (chat-messages.js, chat-documents.js, chat-citations.js, chat-streaming.js)
+  - `workspace/` — Personal workspace (workspace-documents.js, workspace-tags.js)
+  - `public/` — Public workspace (public_workspace.js)
+- Group workspace JS is inline in `templates/group_workspaces.html`
+- Uses Bootstrap 5 for UI components and styling

application/single_app/config.py

@@ -88,7 +88,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.237.009"
+VERSION = "0.238.024"
 
 SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production')
 

application/single_app/functions_activity_logging.py

@@ -170,6 +170,60 @@ def log_web_search_consent_acceptance(
         debug_print(f"Error logging web search consent acceptance for user {user_id}: {str(e)}")
 
 
+def log_index_auto_fix(
+    index_type: str,
+    missing_fields: list,
+    user_id: str = 'system',
+    admin_email: Optional[str] = None
+) -> None:
+    """
+    Log automatic Azure AI Search index field fixes to activity_logs and App Insights.
+
+    Args:
+        index_type (str): Type of index fixed ('user', 'group', or 'public').
+        missing_fields (list): List of field names that were added.
+        user_id (str, optional): User ID triggering the fix. Defaults to 'system'.
+        admin_email (str, optional): Admin email if triggered by admin.
+    """
+    try:
+        activity_record = {
+            'id': str(uuid.uuid4()),
+            'activity_type': 'index_auto_fix',
+            'user_id': user_id,
+            'timestamp': datetime.utcnow().isoformat(),
+            'created_at': datetime.utcnow().isoformat(),
+            'index_type': index_type,
+            'missing_fields': missing_fields,
+            'fields_added': len(missing_fields),
+            'trigger': 'automatic',
+            'description': f"Automatically added {len(missing_fields)} missing field(s) to {index_type} index: {', '.join(missing_fields)}"
+        }
+
+        if admin_email:
+            activity_record['admin_email'] = admin_email
+
+        cosmos_activity_logs_container.create_item(body=activity_record)
+
+        log_event(
+            message=f"Auto-fixed {index_type} index: added {len(missing_fields)} field(s)",
+            extra=activity_record,
+            level=logging.INFO
+        )
+        debug_print(f"Logged index auto-fix for {index_type} index: {', '.join(missing_fields)}")
+
+    except Exception as e:
+        log_event(
+            message=f"Error logging index auto-fix: {str(e)}",
+            extra={
+                'user_id': user_id,
+                'index_type': index_type,
+                'error': str(e)
+            },
+            level=logging.ERROR
+        )
+        debug_print(f"Error logging index auto-fix for {index_type}: {str(e)}")
+
+
 def log_document_upload(
     user_id: str,
     container_type: str,

application/single_app/functions_conversation_metadata.py

@@ -464,7 +464,30 @@ def collect_conversation_metadata(user_message, conversation_id, user_id, active
             }
             current_tags[semantic_key] = semantic_tag    # Update the tags array
     conversation_item['tags'] = list(current_tags.values())
-
+
+    # --- Scope Lock Logic ---
+    current_scope_locked = conversation_item.get('scope_locked')
+
+    if document_map:
+        # Always update locked_contexts when search results exist (even if unlocked)
+        # This ensures re-locking uses the most up-to-date workspace list
+        locked_set = set()
+        for ctx in conversation_item.get('context', []):
+            if ctx.get('scope') != 'model_knowledge' and ctx.get('type') in ('primary', 'secondary'):
+                locked_set.add((ctx['scope'], ctx.get('id')))
+
+        # Merge with existing locked_contexts
+        for ctx in conversation_item.get('locked_contexts', []):
+            locked_set.add((ctx.get('scope'), ctx.get('id')))
+
+        conversation_item['locked_contexts'] = [
+            {"scope": s, "id": i} for s, i in locked_set if s and i
+        ]
+
+        # Only auto-lock the FIRST time (from null state)
+        if current_scope_locked is None:
+            conversation_item['scope_locked'] = True
+
     return conversation_item