[wip] mcp poc

.github/workflows/api-deployer.yml

@@ -237,6 +237,11 @@ jobs:
           DOCKER_IMAGE_VERSION=$EXTRACTED_VERSION.$FEED_API_IMAGE_VERSION
           scripts/docker-build-push.sh -project_id $PROJECT_ID -repo_name feeds-$ENVIRONMENT -service feed-api -region $REGION -version $DOCKER_IMAGE_VERSION
 
+      - name: Build & Publish MCP Server Docker Image
+        run: |
+          MCP_IMAGE_VERSION=$EXTRACTED_VERSION.$FEED_API_IMAGE_VERSION
+          scripts/docker-build-push.sh -project_id $PROJECT_ID -repo_name feeds-$ENVIRONMENT -service mcp-server -region $REGION -version $MCP_IMAGE_VERSION -dockerfile mcp/Dockerfile -context .
+
   terraform-deploy:
     runs-on: ubuntu-latest
     permissions: write-all
@@ -293,6 +298,7 @@ jobs:
           echo "ENVIRONMENT=${{ inputs.ENVIRONMENT }}" >> $GITHUB_ENV
           echo "DEPLOYER_SERVICE_ACCOUNT=${{ inputs.DEPLOYER_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
           echo "FEED_API_IMAGE_VERSION=$EXTRACTED_VERSION.${{ inputs.FEED_API_IMAGE_VERSION }}" >> $GITHUB_ENV
+          echo "MCP_IMAGE_VERSION=$EXTRACTED_VERSION.${{ inputs.FEED_API_IMAGE_VERSION }}" >> $GITHUB_ENV
           echo "OAUTH2_CLIENT_ID=${{ secrets.OAUTH2_CLIENT_ID }}" >> $GITHUB_ENV
           echo "OAUTH2_CLIENT_SECRET=${{ secrets.OAUTH2_CLIENT_SECRET }}" >> $GITHUB_ENV
           echo "GLOBAL_RATE_LIMIT_REQ_PER_MINUTE=${{ inputs.GLOBAL_RATE_LIMIT_REQ_PER_MINUTE }}" >> $GITHUB_ENV
@@ -321,7 +327,7 @@ jobs:
       - name: Populate Variables
         run: |
           scripts/replace-variables.sh -in_file infra/backend.conf.rename_me -out_file infra/backend.conf -variables BUCKET_NAME,OBJECT_PREFIX
-          scripts/replace-variables.sh -in_file infra/vars.tfvars.rename_me -out_file infra/vars.tfvars -variables PROJECT_ID,REGION,ENVIRONMENT,DEPLOYER_SERVICE_ACCOUNT,FEED_API_IMAGE_VERSION,OAUTH2_CLIENT_ID,OAUTH2_CLIENT_SECRET,GLOBAL_RATE_LIMIT_REQ_PER_MINUTE,ARTIFACT_REPO_NAME,VALIDATOR_ENDPOINT,TRANSITLAND_API_KEY,OPERATIONS_OAUTH2_CLIENT_ID,TDG_API_TOKEN,WEB_APP_REVALIDATE_URL,WEB_APP_REVALIDATE_SECRET
+          scripts/replace-variables.sh -in_file infra/vars.tfvars.rename_me -out_file infra/vars.tfvars -variables PROJECT_ID,REGION,ENVIRONMENT,DEPLOYER_SERVICE_ACCOUNT,FEED_API_IMAGE_VERSION,MCP_IMAGE_VERSION,OAUTH2_CLIENT_ID,OAUTH2_CLIENT_SECRET,GLOBAL_RATE_LIMIT_REQ_PER_MINUTE,ARTIFACT_REPO_NAME,VALIDATOR_ENDPOINT,TRANSITLAND_API_KEY,OPERATIONS_OAUTH2_CLIENT_ID,TDG_API_TOKEN,WEB_APP_REVALIDATE_URL,WEB_APP_REVALIDATE_SECRET
 
       - uses: hashicorp/setup-terraform@v3
         with:

.github/workflows/db-deployer.yml

@@ -15,6 +15,12 @@ on:
       POSTGRE_USER_PASSWORD:
         description: PostgreSQL User Password
         required: true
+      POSTGRE_READONLY_USER_NAME:
+        description: PostgreSQL Read-Only User Name (used by MCP server)
+        required: true
+      POSTGRE_READONLY_USER_PASSWORD:
+        description: PostgreSQL Read-Only User Password
+        required: true
       POSTGRE_SQL_INSTANCE_NAME:
         description: PostgreSQL Instance Name
         required: true
@@ -93,13 +99,15 @@ jobs:
           echo "POSTGRE_SQL_DB_NAME=${{ inputs.POSTGRE_SQL_DB_NAME }}" >> $GITHUB_ENV
           echo "POSTGRE_USER_NAME=${{ secrets.POSTGRE_USER_NAME }}" >> $GITHUB_ENV
           echo "POSTGRE_USER_PASSWORD=${{ secrets.POSTGRE_USER_PASSWORD }}" >> $GITHUB_ENV
+          echo "POSTGRE_READONLY_USER_NAME=${{ secrets.POSTGRE_READONLY_USER_NAME }}" >> $GITHUB_ENV
+          echo "POSTGRE_READONLY_USER_PASSWORD=${{ secrets.POSTGRE_READONLY_USER_PASSWORD }}" >> $GITHUB_ENV
           echo "POSTGRE_INSTANCE_TIER=${{ inputs.POSTGRE_INSTANCE_TIER }}" >> $GITHUB_ENV
           echo "MAX_CONNECTIONS=${{ inputs.MAX_CONNECTIONS }}" >> $GITHUB_ENV
 
       - name: Populate Variables
         run: |
           scripts/replace-variables.sh -in_file infra/backend.conf.rename_me -out_file infra/postgresql/backend.conf -variables BUCKET_NAME,OBJECT_PREFIX
-          scripts/replace-variables.sh -in_file infra/postgresql/vars.tfvars.rename_me -out_file infra/postgresql/vars.tfvars -variables ENVIRONMENT,PROJECT_ID,REGION,DEPLOYER_SERVICE_ACCOUNT,POSTGRE_SQL_INSTANCE_NAME,POSTGRE_SQL_DB_NAME,POSTGRE_USER_NAME,POSTGRE_USER_PASSWORD,POSTGRE_INSTANCE_TIER,MAX_CONNECTIONS
+          scripts/replace-variables.sh -in_file infra/postgresql/vars.tfvars.rename_me -out_file infra/postgresql/vars.tfvars -variables ENVIRONMENT,PROJECT_ID,REGION,DEPLOYER_SERVICE_ACCOUNT,POSTGRE_SQL_INSTANCE_NAME,POSTGRE_SQL_DB_NAME,POSTGRE_USER_NAME,POSTGRE_USER_PASSWORD,POSTGRE_READONLY_USER_NAME,POSTGRE_READONLY_USER_PASSWORD,POSTGRE_INSTANCE_TIER,MAX_CONNECTIONS
 
       - name: Install Terraform
         uses: hashicorp/setup-terraform@v3
@@ -149,6 +157,8 @@ jobs:
     env:
       POSTGRE_USER_NAME: ${{ secrets.POSTGRE_USER_NAME }}
       POSTGRE_USER_PASSWORD: ${{ secrets.POSTGRE_USER_PASSWORD }}
+      POSTGRE_READONLY_USER_NAME: ${{ secrets.POSTGRE_READONLY_USER_NAME }}
+      POSTGRE_READONLY_USER_PASSWORD: ${{ secrets.POSTGRE_READONLY_USER_PASSWORD }}
       POSTGRE_SQL_DB_NAME: ${{ inputs.POSTGRE_SQL_DB_NAME }}
       DB_INSTANCE_HOST: ${{ needs.terraform.outputs.db_instance_host }}
     steps:
@@ -165,7 +175,20 @@ jobs:
           SECRET_NAME="DEV_FEEDS_DATABASE_URL"
           SECRET_VALUE="postgresql://${{ env.POSTGRE_USER_NAME }}:${{ env.POSTGRE_USER_PASSWORD }}@${{ env.DB_INSTANCE_HOST }}/${{ env.POSTGRE_SQL_DB_NAME }}DEV"
           echo $SECRET_VALUE
-
+
+          if gcloud secrets describe $SECRET_NAME --project=mobility-feeds-dev; then
+            echo "Secret $SECRET_NAME already exists, updating..."
+            echo -n "$SECRET_VALUE" | gcloud secrets versions add $SECRET_NAME --data-file=- --project=mobility-feeds-dev
+          else
+            echo "Secret $SECRET_NAME does not exist, creating..."
+            echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=mobility-feeds-dev
+          fi
+
+      - name: Create or Update Readonly Secret in DEV
+        run: |
+          SECRET_NAME="DEV_FEEDS_DATABASE_URL_READONLY"
+          SECRET_VALUE="postgresql://${{ env.POSTGRE_READONLY_USER_NAME }}:${{ env.POSTGRE_READONLY_USER_PASSWORD }}@${{ env.DB_INSTANCE_HOST }}/${{ env.POSTGRE_SQL_DB_NAME }}DEV"
+
           if gcloud secrets describe $SECRET_NAME --project=mobility-feeds-dev; then
             echo "Secret $SECRET_NAME already exists, updating..."
             echo -n "$SECRET_VALUE" | gcloud secrets versions add $SECRET_NAME --data-file=- --project=mobility-feeds-dev

.github/workflows/db-prod.yml

@@ -21,6 +21,8 @@ jobs:
     secrets:
       POSTGRE_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }}
       POSTGRE_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }}
+      POSTGRE_READONLY_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_READONLY_USER_PASSWORD }}
+      POSTGRE_READONLY_USER_NAME: ${{ secrets.PROD_POSTGRE_READONLY_USER_NAME }}
       POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
       DEV_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }}

.github/workflows/db-qa.yml

@@ -21,6 +21,8 @@ jobs:
     secrets:
       POSTGRE_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }}
       POSTGRE_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }}
+      POSTGRE_READONLY_USER_PASSWORD: ${{ secrets.QA_POSTGRE_READONLY_USER_PASSWORD }}
+      POSTGRE_READONLY_USER_NAME: ${{ secrets.QA_POSTGRE_READONLY_USER_NAME }}
       POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
       DEV_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }}

.gitignore

@@ -86,4 +86,7 @@ functions-python/**/*.csv
 *.code-workspace
 
 # Ignore OpenApi local backup files
-*.yaml.bak
+*.yaml.bak
+
+# Claude folder
+.claude

infra/main.tf

@@ -100,6 +100,16 @@ module "feed-api" {
 }
 
 
+module "mcp" {
+  source = "./mcp"
+
+  project_id             = var.project_id
+  gcp_region             = var.gcp_region
+  environment            = var.environment
+  docker_repository_name = "${var.artifact_repo_name}-${var.environment}"
+  mcp_image_version      = var.mcp_image_version
+}
+
 module "functions-python" {
   source = "./functions-python"
   project_id  = var.project_id

infra/mcp/main.tf

@@ -0,0 +1,142 @@
+#
+# MobilityData 2024
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script deploys the MCP Server cloud run service.
+# The cloud run service is created with name: mcp-server-${var.environment}
+# Module output:
+#   mcp_server_uri: URI of the MCP Server Cloud Run service
+
+data "google_project" "project" {}
+
+locals {
+  vpc_connector_name    = lower(var.environment) == "dev" ? "vpc-connector-qa" : "vpc-connector-${lower(var.environment)}"
+  vpc_connector_project = lower(var.environment) == "dev" ? "mobility-feeds-qa" : var.project_id
+
+  service_account_roles = [
+    # Cloud Logging: allows writing logs to GCP
+    "roles/logging.logWriter",
+    # Cloud Trace: allows writing trace and span data
+    "roles/cloudtrace.agent",
+    # Cloud Monitoring: allows publishing custom metrics
+    "roles/monitoring.metricWriter",
+    # Serverless VPC Access: required to use a VPC connector
+    "roles/vpcaccess.user",
+  ]
+
+  service_account_role_bindings = {
+    for role in local.service_account_roles :
+    "${role}" => {
+      role    = role
+      project = var.project_id
+    }
+  }
+}
+
+data "google_vpc_access_connector" "vpc_connector" {
+  name    = local.vpc_connector_name
+  region  = var.gcp_region
+  project = local.vpc_connector_project
+}
+
+resource "google_service_account" "mcp_service_account" {
+  account_id   = "mcp-service-account"
+  display_name = "MCP Server Service Account"
+}
+
+resource "google_cloud_run_v2_service" "mcp_server" {
+  name     = "mcp-server-${var.environment}"
+  location = var.gcp_region
+  ingress  = "INGRESS_TRAFFIC_ALL"
+
+  template {
+    service_account = google_service_account.mcp_service_account.email
+
+    # Use annotations for max instances on older provider(<6.0.0)
+    annotations = {
+      "run.googleapis.com/max-instances" = "10"
+    }
+
+    vpc_access {
+      connector = data.google_vpc_access_connector.vpc_connector.id
+      egress    = "ALL_TRAFFIC"
+    }
+
+    containers {
+      image = "${var.gcp_region}-docker.pkg.dev/${var.project_id}/${var.docker_repository_name}/mcp-server:${var.mcp_image_version}"
+
+      env {
+        name = "FEEDS_DATABASE_URL"
+        value_source {
+          secret_key_ref {
+            secret  = "${upper(var.environment)}_FEEDS_DATABASE_URL_READONLY"
+            version = "latest"
+          }
+        }
+      }
+
+      env {
+        name  = "DATASETS_BUCKET_URL"
+        value = "https://storage.googleapis.com/mobilitydata-datasets-${lower(var.environment)}"
+      }
+
+      resources {
+        limits = {
+          cpu    = "1"
+          memory = "512Mi"
+        }
+      }
+    }
+  }
+}
+
+data "google_iam_policy" "noauth" {
+  binding {
+    role    = "roles/run.invoker"
+    members = ["allUsers"]
+  }
+}
+
+resource "google_cloud_run_service_iam_policy" "noauth" {
+  location    = google_cloud_run_v2_service.mcp_server.location
+  project     = google_cloud_run_v2_service.mcp_server.project
+  service     = google_cloud_run_v2_service.mcp_server.name
+  policy_data = data.google_iam_policy.noauth.policy_data
+}
+
+resource "google_secret_manager_secret_iam_member" "feeds_db_url_access" {
+  project   = var.project_id
+  secret_id = "${upper(var.environment)}_FEEDS_DATABASE_URL_READONLY"
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:${google_service_account.mcp_service_account.email}"
+}
+
+resource "google_project_iam_member" "mcp_service_account_roles" {
+  for_each = local.service_account_role_bindings
+
+  project = each.value.project
+  role    = each.value.role
+  member  = "serviceAccount:${google_service_account.mcp_service_account.email}"
+}
+
+output "mcp_server_uri" {
+  value       = google_cloud_run_v2_service.mcp_server.uri
+  description = "URI of the MCP Server Cloud Run service"
+}
+
+output "mcp_server_name" {
+  value       = google_cloud_run_v2_service.mcp_server.name
+  description = "Name of the MCP Server Cloud Run service"
+}

infra/mcp/vars.tf

@@ -0,0 +1,40 @@
+#
+# MobilityData 2024
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+variable "project_id" {
+  type        = string
+  description = "GCP project ID"
+}
+
+variable "gcp_region" {
+  type        = string
+  description = "GCP region"
+}
+
+variable "environment" {
+  type        = string
+  description = "Deployment environment (prod, staging, dev)"
+}
+
+variable "docker_repository_name" {
+  type        = string
+  description = "Artifact Registry Docker repository name"
+}
+
+variable "mcp_image_version" {
+  type        = string
+  description = "Docker image version/tag for the MCP server"
+}

infra/postgresql/main.tf

@@ -95,6 +95,12 @@ resource "google_sql_user" "users" {
   password = var.postgresql_user_password
 }
 
+resource "google_sql_user" "readonly_user" {
+  name     = var.postgresql_readonly_user_name
+  instance = google_sql_database_instance.db.name
+  password = var.postgresql_readonly_user_password
+}
+
 resource "google_secret_manager_secret" "secret_db_url" {
   project   = var.project_id
   secret_id = "${upper(var.environment)}_FEEDS_DATABASE_URL"
@@ -108,6 +114,19 @@ resource "google_secret_manager_secret_version" "secret_version" {
   secret_data = "postgresql+psycopg2://${var.postgresql_user_name}:${var.postgresql_user_password}@${google_sql_database_instance.db.private_ip_address}/${var.postgresql_database_name}"
 }
 
+resource "google_secret_manager_secret" "secret_db_url_readonly" {
+  project   = var.project_id
+  secret_id = "${upper(var.environment)}_FEEDS_DATABASE_URL_READONLY"
+  replication {
+     auto {}
+  }
+}
+
+resource "google_secret_manager_secret_version" "secret_version_readonly" {
+  secret = google_secret_manager_secret.secret_db_url_readonly.id
+  secret_data = "postgresql+psycopg2://${var.postgresql_readonly_user_name}:${var.postgresql_readonly_user_password}@${google_sql_database_instance.db.private_ip_address}/${var.postgresql_database_name}"
+}
+
 output "instance_address" {
   description = "The first public IPv4 address of the SQL instance"
   value       = google_sql_database_instance.db.private_ip_address

infra/postgresql/vars.tf

@@ -52,4 +52,14 @@ variable "postgresql_db_instance" {
 variable "max_db_connections" {
   type        = string
   description = "Maximum number of database connections"
+}
+
+variable "postgresql_readonly_user_name" {
+  type        = string
+  description = "The name of the read-only PostgreSQL user (used by MCP server)"
+}
+
+variable "postgresql_readonly_user_password" {
+  type        = string
+  description = "The password for the read-only PostgreSQL user"
 }

infra/postgresql/vars.tfvars.rename_me

@@ -14,5 +14,7 @@ postgresql_instance_name = {{POSTGRE_SQL_INSTANCE_NAME}}
 postgresql_database_name = {{POSTGRE_SQL_DB_NAME}}
 postgresql_user_name     = {{POSTGRE_USER_NAME}}
 postgresql_user_password = {{POSTGRE_USER_PASSWORD}}
+postgresql_readonly_user_name     = {{POSTGRE_READONLY_USER_NAME}}
+postgresql_readonly_user_password = {{POSTGRE_READONLY_USER_PASSWORD}}
 postgresql_db_instance = {{POSTGRE_INSTANCE_TIER}}
 max_db_connections = {{MAX_CONNECTIONS}}

infra/vars.tf

@@ -89,4 +89,9 @@ variable "web_app_revalidate_secret" {
   description = "Secret token used to authenticate requests to the website revalidation endpoint"
     sensitive   = true
     default     = ""
+}
+
+variable "mcp_image_version" {
+  type        = string
+  description = "Docker image version/tag for the MCP server"
 }

infra/vars.tfvars.rename_me

@@ -11,6 +11,7 @@ artifact_repo_name = {{ARTIFACT_REPO_NAME}}
 
 deployer_service_account = {{DEPLOYER_SERVICE_ACCOUNT}}
 feed_api_image_version      = {{FEED_API_IMAGE_VERSION}}
+mcp_image_version           = {{MCP_IMAGE_VERSION}}
 
 oauth2_client_id = {{OAUTH2_CLIENT_ID}}
 oauth2_client_secret = {{OAUTH2_CLIENT_SECRET}}