NPA-6546: fix end to end tests

.github/actions/setup-proxygen-prod/action.yaml

@@ -0,0 +1,48 @@
+name: "Setup Proxygen CLI (Prod)"
+description: "Setup Proxygen CLI for production deployments"
+
+inputs:
+  ENVIRONMENT:
+    description: "The environment to configure Proxygen for"
+    required: true
+  PROXYGEN_CLIENT_ID:
+    description: "Client ID for Proxygen CLI (Prod)"
+    required: true
+  PROXYGEN_KEY_ID:
+    description: "Key ID for Proxygen CLI (Prod)"
+    required: true
+  PROXYGEN_PRIVATE_KEY:
+    description: "Private key for Proxygen CLI (Prod)"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Proxygen CLI
+      run: pip install proxygen-cli
+      shell: bash
+
+    - name: Create Proxygen configuration directory
+      run: mkdir -p ~/.proxygen
+      shell: bash
+
+    - name: Copy Proxygen credentials
+      run: cp proxygen/credentials-prod.yaml ~/.proxygen/credentials.yaml
+      shell: bash
+
+    - name: Create Proxygen private key file
+      run: |
+        echo "${{ inputs.PROXYGEN_PRIVATE_KEY }}" > ~/.proxygen/private_key.pem
+        chmod 600 ~/.proxygen/private_key.pem
+      shell: bash
+
+    - name: Update Proxygen Credentials
+      run: |
+        sed -i "s|CLIENT_ID_TO_BE_REPLACED|${{ inputs.PROXYGEN_CLIENT_ID }}|" ~/.proxygen/credentials.yaml
+        sed -i "s|KEY_ID_TO_BE_REPLACED|${{ inputs.PROXYGEN_KEY_ID }}|" ~/.proxygen/credentials.yaml
+        sed -i "s|PRIVATE_KEY_PATH_TO_BE_REPLACED|private_key.pem|" ~/.proxygen/credentials.yaml
+      shell: bash
+
+    - name: Copy Proxygen settings
+      run: cp proxygen/settings-prod.yaml ~/.proxygen/settings.yaml
+      shell: bash

.github/actions/setup-proxygen/action.yaml

@@ -24,7 +24,7 @@ runs:
       shell: bash
 
     - name: Copy Proxygen credentials
-      run: cp proxygen/credentials.yaml ~/.proxygen/credentials.yaml
+      run: cp proxygen/credentials-ptl.yaml ~/.proxygen/credentials.yaml
       shell: bash
 
     - name: Create Proxygen private key file
@@ -41,5 +41,5 @@ runs:
       shell: bash
 
     - name: Copy Proxygen settings
-      run: cp proxygen/settings.yaml ~/.proxygen/settings.yaml
+      run: cp proxygen/settings-ptl.yaml ~/.proxygen/settings.yaml
       shell: bash

.github/workflows/cicd-stage-3-deploy-to-prod.yml

@@ -20,8 +20,9 @@ jobs:
       - name: Setup Python Dependencies
         uses: ./.github/actions/setup-python-dependencies
       - name: Setup proxygen
-        uses: ./.github/actions/setup-proxygen
+        uses: ./.github/actions/setup-proxygen-prod
         with:
+          ENVIRONMENT: "prod"
           PROXYGEN_CLIENT_ID: ${{ secrets.PROXYGEN_CLIENT_ID }}
           PROXYGEN_KEY_ID: ${{ secrets.PROXYGEN_KEY_ID }}
           PROXYGEN_PRIVATE_KEY: ${{ secrets.PROXYGEN_PRIVATE_KEY }}

Makefile

@@ -56,10 +56,12 @@ deploy-ci:
 
 # Deploy spec to uat
 deploy-spec-uat:
+	cp -f specification/x-nhsd-apim/x-nhsd-apim-int.yaml specification/x-nhsd-apim/x-nhsd-apim.generated.yaml
 	proxygen spec publish --uat $(PROXYGEN_ARGS) specification/im1-pfs-auth-api.yaml
 
 # Deploy spec to prod
 deploy-spec-prod:
+	cp -f specification/x-nhsd-apim/x-nhsd-apim-prod.yaml specification/x-nhsd-apim/x-nhsd-apim.generated.yaml
 	proxygen spec publish $(PROXYGEN_ARGS) specification/im1-pfs-auth-api.yaml
 
 # Deploy spec to uat in CI

app/api/domain/forward_response_model.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, ConfigDict, field_validator
+from pydantic import BaseModel, ConfigDict, SerializeAsAny, field_validator
 from pydantic.alias_generators import to_camel
 
 
@@ -26,8 +26,8 @@ class ForwardResponse(BaseModel):
     session_id: str
     supplier: str
     ods_code: str
-    user: Demographics
-    patients: list[Demographics]
+    user: SerializeAsAny[Demographics]
+    patients: list[SerializeAsAny[Demographics]]
 
     @field_validator("patients")
     def patients_must_not_be_empty(cls, v: list) -> list:  # noqa: N805

app/api/infrastructure/emis/client.py

@@ -14,7 +14,7 @@
     EffectiveServices,
     Identifier,
     MedicalRecordPermissions,
-    Patient,
+    Person,
     SessionRequestData,
     SessionRequestHeaders,
     SessionResponse,
@@ -114,7 +114,7 @@ def transform_response(self, response: dict) -> SessionResponse:
             endUserSessionId=response.get("EndUserSessionId"),
             supplier=self.supplier,
             odsCode=self.request.patient_ods_code,
-            user=Patient(
+            user=Person(
                 firstName=response.get("FirstName"),
                 surname=response.get("Surname"),
                 title=response.get("Title"),
@@ -145,20 +145,20 @@ def _mock_response(self) -> dict:
         with Path((BASE_DIR) / "data" / "mocked_response.json").open("r") as f:
             return load(f)
 
-    def _parse_patients(self, patient_links: list) -> list[Patient]:
-        """Parsing raw data from Client into structual model.
+    def _parse_patients(self, patient_links: list) -> list[Person]:
+        """Parsing raw data from Client into structural model.
 
         Args:
-            patient_links (dict): Raw data containing information about patients
+            patient_links (list[dict]): Raw data containing information about patients
 
         Returns:
-            list[Patient]: Parsed information about patients
+            list[Person]: Parsed information about patients
         """
         parsed_patients = []
         for patient in patient_links:
             raw_permissions = patient.get("EffectiveServices", {})
             parsed_patients.append(
-                Patient(
+                Person(
                     firstName=patient.get("FirstName"),
                     surname=patient.get("Surname"),
                     title=patient.get("Title"),

app/api/infrastructure/emis/models.py

@@ -83,7 +83,7 @@ class EffectiveServices(Permissions):
     medical_record: MedicalRecordPermissions
 
 
-class Patient(Demographics):
+class Person(Demographics):
     """Base Model for User and Patient."""
 
     model_config = ConfigDict(alias_generator=to_camel)
@@ -99,5 +99,5 @@ class SessionResponse(ForwardResponse):
     model_config = ConfigDict(alias_generator=to_camel)
 
     end_user_session_id: str
-    user: Patient
-    patients: list[Patient]
+    user: Person
+    patients: list[Person]

app/api/infrastructure/emis/tests/test_client.py

@@ -18,7 +18,7 @@
     EffectiveServices,
     Identifier,
     MedicalRecordPermissions,
-    Patient,
+    Person,
     SessionResponse,
 )
 
@@ -140,7 +140,7 @@ def test_emis_client_transform_response(client: EmisClient) -> None:
         endUserSessionId="SESS_mDq6nE2b8R7KQ0v",
         supplier="EMIS",
         odsCode="some patient ods code",
-        user=Patient(
+        user=Person(
             firstName="Alex",
             surname="Taylor",
             title="Mr",
@@ -170,7 +170,7 @@ def test_emis_client_transform_response(client: EmisClient) -> None:
             ),
         ),
         patients=[
-            Patient(
+            Person(
                 firstName="Jane",
                 surname="Doe",
                 title="Mrs",
@@ -201,7 +201,7 @@ def test_emis_client_transform_response(client: EmisClient) -> None:
                     ),
                 ),
             ),
-            Patient(
+            Person(
                 firstName="Ella",
                 surname="Taylor",
                 title="Ms",

app/api/infrastructure/tpp/client.py

@@ -136,7 +136,7 @@ def _mock_response(self) -> dict:
         return xmltodict.parse(mocked_response)
 
     def _parse_patients(self, data: dict) -> list[Person]:
-        """Parsing raw data from Client into structual model.
+        """Parsing raw data from Client into structural model.
 
         Args:
             data (dict): Raw data containing information about multiple patients

docs/user-guides/Glossary.md

@@ -0,0 +1,140 @@
+# Glossary
+
+A reference guide for key terms, tools, and concepts used in the IM1 PFS Auth project.
+
+---
+
+## APIM (API Management Platform)
+
+The **NHS API Management Platform**, built and operated by NHS England. It provides the infrastructure for publishing, securing, monitoring, and managing APIs across the NHS. APIM is built on top of **Apigee** and is the platform that `im1-pfs-auth` is deployed to. It is accessible via `*.api.service.nhs.uk` URLs (e.g. `https://int.api.service.nhs.uk/im1-pfs-auth`).
+
+---
+
+## Apigee
+
+**Apigee** is Google's API gateway and management product, which underpins the NHS APIM platform. It handles:
+
+- Routing inbound API requests to backend containers
+- Authentication and authorisation enforcement (e.g. composite token validation)
+- Analytics and monitoring
+
+In this project, Apigee proxies are deployed via the **Proxygen CLI** using the OpenAPI specification in the `specification/` directory. The Apigee UI is accessible at [https://apigee.com/edge](https://apigee.com/edge) under the `nhsd-nonprod` (non-production) and `nhsd-prod` (production) organisations.
+
+---
+
+## Proxygen (Service)
+
+**Proxygen** (short for Proxy Generator) is an NHS England-built service that sits in front of Apigee and acts as the control plane for API producers. It abstracts the complexity of directly interacting with Apigee by accepting an OpenAPI specification and handling the creation and management of Apigee API proxies on your behalf.
+
+The Proxygen service is accessible at:
+
+```text
+https://proxygen.prod.api.platform.nhs.uk
+```
+
+It is also responsible for managing the **AWS ECR container registry** that holds the Docker images that back deployed API proxies.
+
+---
+
+## Proxygen CLI
+
+The **Proxygen CLI** (`proxygen-cli`) is a Python command-line tool provided by NHS England that allows API producer teams to interact with the Proxygen service. In this project it is used to:
+
+- **Deploy API proxy instances** to APIM environments (e.g. `internal-dev`, `int`, `prod`) via `proxygen instance deploy`
+- **Publish the OpenAPI spec** to the NHS developer portal via `proxygen spec publish`
+- **Obtain Docker credentials** for pushing container images to the NHS ECR registry via `proxygen docker get-login`
+- **Obtain test tokens** for running end-to-end tests via `proxygen pytest-nhsd-apim get-token`
+
+The CLI authenticates against the NHS identity service at:
+
+```text
+https://identity.prod.api.platform.nhs.uk/realms/api-producers
+```
+
+This is a **Keycloak** identity provider used for machine-to-machine authentication for API producer teams. Authentication requires three credentials: a `client_id`, a `key_id`, and a `private_key` (PEM file). In CI/CD these are stored as GitHub secrets (`PROXYGEN_CLIENT_ID`, `PROXYGEN_KEY_ID`, `PROXYGEN_PRIVATE_KEY`) and also in the VRS AWS Prod Secrets Manager under the prefix `im1-pfs-auth/proxygen/<secret>`.
+
+See the [Proxygen CLI guide](./Proxygen_CLI.md) for installation and configuration instructions.
+
+---
+
+## IM1
+
+**IM1** (Interface Mechanism 1) is a GP system integration standard used in the NHS. It defines how third-party applications (such as patient-facing services) can integrate with GP clinical systems (supplied by GPIT suppliers such as EMIS and TPP/SystmOne). IM1 allows authorised applications to act on behalf of patients, interacting with their GP practice's system.
+
+---
+
+## IM1 PFS Auth (`im1-pfs-auth`)
+
+**IM1 Patient Facing Service Auth** is this project. It is an intermediary API service that enables patient-facing applications to authenticate and establish sessions with GP practice systems via the IM1 interface. It sits between a patient-facing application and a GPIT supplier system (e.g. EMIS, TPP), handling:
+
+- Validation of NHS login proxy tokens
+- Session initiation with the appropriate supplier system based on ODS code
+- Transformation of supplier responses
+
+It is deployed as an Apigee API proxy backed by a Docker container, and is accessible at `*.api.service.nhs.uk/im1-pfs-auth`.
+
+---
+
+## IM1 PFS Auth Developer Test App
+
+The **IM1 PFS Auth Developer Test App** is a registered application in the Apigee developer portal (under the `nhsd-nonprod` organisation) used specifically for running end-to-end tests. It has access to the `mock-jwks` service, which is required for generating composite authentication tokens in the `internal-dev` environment.
+
+When a new ephemeral deployment is created (e.g. from a pull request), it must be manually associated with this test app in the Apigee UI before end-to-end tests can be run against it. See the [Setup end to end tests guide](./Setup_end_to_end_tests.md) for instructions.
+
+---
+
+## Composite Token / mock-jwks
+
+A **composite token** is a development-environment authentication token used in APIM's `internal-dev` environment to simulate authenticated requests without requiring real NHS login credentials. It is obtained from APIM's `mock-jwks` service.
+
+The `mock-jwks` service is only enabled in the `internal-dev` environment. The **IM1 PFS Auth Developer Test App** must be associated with a deployment before composite tokens can be used against it.
+
+---
+
+## GPIT Supplier
+
+A **GPIT supplier** is a provider of GP IT systems in the NHS — primarily **EMIS Health** and **TPP (The Phoenix Partnership)**, who make SystmOne. These are the systems that `im1-pfs-auth` communicates with when establishing patient sessions. Their base URLs are configured at build time via the `EMIS_BASE_URL` and `TPP_BASE_URL` environment variables.
+
+---
+
+## ODS Code
+
+An **ODS (Organisation Data Service) code** is a unique identifier assigned to NHS organisations, including GP practices. `im1-pfs-auth` uses the ODS code of a patient's GP practice to determine which GPIT supplier system to route a session request to.
+
+---
+
+## ECR (Elastic Container Registry)
+
+**AWS ECR** is the container image registry used to store the Docker images for `im1-pfs-auth`. The NHS-managed registry is at:
+
+```text
+958002497996.dkr.ecr.eu-west-2.amazonaws.com/im1-pfs-auth
+```
+
+Docker credentials to push to this registry are obtained via `proxygen docker get-login`. Apigee pulls the container from this registry when serving API requests.
+
+---
+
+## NHS Developer Hub / Developer Portal
+
+The **NHS Internal Developer Hub** (accessible at `https://dos-internal.ptl.api.platform.nhs.uk`) is the portal where API producer teams manage their applications, API keys, and key pairs used for testing. It is also where the **IM1 PFS Auth Developer Test App** API key (`TEST_APP_API_KEY`) and private key (`TEST_APP_PRIVATE_KEY`) are registered and managed.
+
+Access to the developer hub for end-to-end testing requires membership of the `Proxy Dev Team`. See the [NHS Developer Hub guide](./NHS_developer_hub.md) for more detail.
+
+---
+
+## Keycloak
+
+**Keycloak** is an open-source identity and access management solution. NHS England uses a Keycloak instance at `https://identity.prod.api.platform.nhs.uk/realms/api-producers` as the identity provider for authenticating API producer machine users (i.e. the Proxygen CLI). It is also used in the test setup — `TEST_APP_KEYCLOAK_CLIENT_ID` and `TEST_APP_KEYCLOAK_CLIENT_SECRET` are credentials for a mocked authorisation provider client used in end-to-end tests.
+
+---
+
+## Sandbox
+
+The **sandbox** is a simulated version of the `im1-pfs-auth` API that returns mock responses without connecting to real GPIT supplier systems. It is deployed alongside the main app in certain environments (e.g. `internal-dev-sandbox`, `sandbox`) and does not require authentication. It allows developers and API consumers to explore the API without needing to onboard or hold real credentials.
+
+---
+
+## Ephemeral Deployment
+
+An **ephemeral deployment** is a temporary deployment of `im1-pfs-auth` created automatically when a pull request is opened. It is deployed to the `internal-dev` environment with a URL path following the pattern `im1-pfs-auth-pr-<pr_number>`, resulting in an Apigee proxy named `im1-pfs-auth--internal-dev--im1-pfs-auth-pr-<pr_number>`. These deployments are used to run end-to-end tests against code changes before they are merged to `main`.

docs/user-guides/Proxygen_CLI.md

@@ -87,3 +87,5 @@ Secrets used for machine access are stored in Validated Relationships Service's
 
 As well secrets are held in GitHub Secrets for the project. The secrets are used to authenticate the workflows to deploy the API to the NHS API Platform. The secrets are:
 the private key is available in GitHub Secrets under the names `PROXYGEN_CLIENT_ID`, `PROXYGEN_KEY_ID`, and `PROXYGEN_PRIVATE_KEY`.
+
+For production, these are stored as environment level secrets, and for PTL these are stored as repository level secrets - this way the default is PTL, and if it's a prod specific workflow/action, it will use the production variables.

proxygen/credentials-ptl.yaml

@@ -0,0 +1,4 @@
+base_url: https://identity.ptl.api.platform.nhs.uk/realms/api-producers
+client_id: CLIENT_ID_TO_BE_REPLACED
+key_id: KEY_ID_TO_BE_REPLACED
+private_key_path: PRIVATE_KEY_PATH_TO_BE_REPLACED

proxygen/settings-ptl.yaml

@@ -0,0 +1,5 @@
+{
+  "endpoint_url": "https://proxygen.ptl.api.platform.nhs.uk",
+  "spec_output_format": "json",
+  "api": "im1-pfs-auth",
+}

specification/im1-pfs-auth-api.yaml

@@ -96,6 +96,7 @@ info:
 servers:
   - url: https://sandbox.api.service.nhs.uk/im1-pfs-auth/
   - url: https://int.api.service.nhs.uk/im1-pfs-auth/
+  - url: https://internal-dev.api.service.nhs.uk/im1-pfs-auth
 
 paths:
   /authenticate: